rahmad7/hermes-openmodel
0
1#!/usr/bin/env python32"""3Hermes Agent HF Spaces Persistence — Full Directory Sync4=========================================================5 6Simplified persistence: upload/download the entire /opt/data directory7as-is to/from a Hugging Face Dataset repo.8 9- Startup: snapshot_download → /opt/data10- Periodic: upload_folder → dataset hermes_data/11- Shutdown: final upload_folder → dataset hermes_data/12"""13 14import os15import sys16import time17import threading18import subprocess19import signal20import shutil21import tempfile22import traceback23from pathlib import Path24from datetime import datetime25# Set timeout BEFORE importing huggingface_hub26os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "300")27os.environ.setdefault("HF_HUB_UPLOAD_TIMEOUT", "600")28os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")29os.environ.setdefault("HF_HUB_VERBOSITY", "warning")30 31import logging as _logging32_logging.getLogger("huggingface_hub").setLevel(_logging.WARNING)33_logging.getLogger("huggingface_hub.utils").setLevel(_logging.WARNING)34_logging.getLogger("filelock").setLevel(_logging.WARNING)35 36from huggingface_hub import HfApi, snapshot_download37 38# ── Logging helper ──────────────────────────────────────────────────────────39 40class TeeLogger:41 """Duplicate output to stream and file."""42 def __init__(self, filename, stream):43 self.stream = stream44 self.file = open(filename, "a", encoding="utf-8")45 def write(self, message):46 self.stream.write(message)47 self.file.write(message)48 self.flush()49 def flush(self):50 self.stream.flush()51 self.file.flush()52 def fileno(self):53 return self.stream.fileno()54 55# ── Configuration ───────────────────────────────────────────────────────────56 57HF_TOKEN = os.environ.get("HF_TOKEN")58HERMES_DATA = Path("/opt/data")59APP_DIR = Path("/opt/hermes")60DATASET_PATH = "hermes_data"61 62AGENT_NAME = os.environ.get("AGENT_NAME", "HermesFace")63 64# HF Spaces built-in env vars (auto-set by HF runtime)65SPACE_HOST = os.environ.get("SPACE_HOST", "")66SPACE_ID = os.environ.get("SPACE_ID", "")67 68SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", "60"))69AUTO_CREATE_DATASET = os.environ.get("AUTO_CREATE_DATASET", "true").lower() in ("true", "1", "yes")70 71# Dataset repo: auto-derive from SPACE_ID when not explicitly set.72# Format: {username}/{SpaceName}-data73HF_REPO_ID = os.environ.get("HERMES_DATASET_REPO", "")74if not HF_REPO_ID and SPACE_ID:75 HF_REPO_ID = f"{SPACE_ID}-data"76 print(f"[SYNC] HERMES_DATASET_REPO not set — auto-derived from SPACE_ID: {HF_REPO_ID}")77elif not HF_REPO_ID and HF_TOKEN:78 try:79 _api = HfApi(token=HF_TOKEN)80 _username = _api.whoami()["name"]81 HF_REPO_ID = f"{_username}/HermesFace-data"82 print(f"[SYNC] HERMES_DATASET_REPO not set — auto-derived from HF_TOKEN: {HF_REPO_ID}")83 del _api, _username84 except Exception as e:85 print(f"[SYNC] WARNING: Could not derive username from HF_TOKEN: {e}")86 HF_REPO_ID = ""87 88# Setup logging89log_dir = HERMES_DATA / "logs"90log_dir.mkdir(parents=True, exist_ok=True)91sys.stdout = TeeLogger(log_dir / "sync.log", sys.stdout)92sys.stderr = sys.stdout93 94 95# ── Sync Manager ────────────────────────────────────────────────────────────96 97class HermesFullSync:98 """Upload/download the entire /opt/data directory to HF Dataset."""99 100 def __init__(self):101 self.enabled = False102 self.dataset_exists = False103 self.api = None104 105 if not HF_TOKEN:106 print("[SYNC] WARNING: HF_TOKEN not set. Persistence disabled.")107 return108 if not HF_REPO_ID:109 print("[SYNC] WARNING: Could not determine dataset repo (no SPACE_ID or HERMES_DATASET_REPO).")110 print("[SYNC] Persistence disabled.")111 return112 113 self.enabled = True114 self.api = HfApi(token=HF_TOKEN)115 self.dataset_exists = self._ensure_repo_exists()116 117 # ── Repo management ────────────────────────────────────────────────118 119 def _ensure_repo_exists(self):120 """Check if dataset repo exists; auto-create only when AUTO_CREATE_DATASET=true."""121 try:122 self.api.repo_info(repo_id=HF_REPO_ID, repo_type="dataset")123 print(f"[SYNC] Dataset repo found: {HF_REPO_ID}")124 return True125 except Exception:126 if not AUTO_CREATE_DATASET:127 print(f"[SYNC] Dataset repo NOT found: {HF_REPO_ID}")128 print("[SYNC] Set AUTO_CREATE_DATASET=true to auto-create.")129 print("[SYNC] Persistence disabled (app will still run normally).")130 return False131 print(f"[SYNC] Dataset repo NOT found: {HF_REPO_ID} — creating...")132 try:133 self.api.create_repo(134 repo_id=HF_REPO_ID,135 repo_type="dataset",136 private=True,137 )138 print(f"[SYNC] Dataset repo created: {HF_REPO_ID}")139 return True140 except Exception as e:141 print(f"[SYNC] Failed to create dataset repo: {e}")142 return False143 144 # ── Restore (startup) ─────────────────────────────────────────────145 146 def load_from_repo(self):147 """Download from dataset → /opt/data"""148 if not self.enabled:149 print("[SYNC] Persistence disabled - skipping restore")150 self._ensure_default_config()151 return152 153 if not self.dataset_exists:154 print(f"[SYNC] Dataset {HF_REPO_ID} does not exist - starting fresh")155 self._ensure_default_config()156 return157 158 print(f"[SYNC] Restoring /opt/data from dataset {HF_REPO_ID} ...")159 HERMES_DATA.mkdir(parents=True, exist_ok=True)160 161 try:162 files = self.api.list_repo_files(repo_id=HF_REPO_ID, repo_type="dataset")163 data_files = [f for f in files if f.startswith(f"{DATASET_PATH}/")]164 if not data_files:165 print(f"[SYNC] No {DATASET_PATH}/ folder in dataset. Starting fresh.")166 self._ensure_default_config()167 return168 169 print(f"[SYNC] Found {len(data_files)} files under {DATASET_PATH}/ in dataset")170 171 with tempfile.TemporaryDirectory() as tmpdir:172 snapshot_download(173 repo_id=HF_REPO_ID,174 repo_type="dataset",175 allow_patterns=f"{DATASET_PATH}/**",176 local_dir=tmpdir,177 token=HF_TOKEN,178 )179 downloaded_root = Path(tmpdir) / DATASET_PATH180 if downloaded_root.exists():181 for item in downloaded_root.rglob("*"):182 if item.is_file():183 rel = item.relative_to(downloaded_root)184 # NEVER restore scripts or assets — they come from Docker image, not dataset185 if rel.parts and rel.parts[0] in ("scripts", "assets"):186 continue187 dest = HERMES_DATA / rel188 dest.parent.mkdir(parents=True, exist_ok=True)189 shutil.copy2(str(item), str(dest))190 print("[SYNC] Restore completed (scripts/assets skipped — from Docker image).")191 else:192 print("[SYNC] Downloaded snapshot but dir not found. Starting fresh.")193 194 except Exception as e:195 print(f"[SYNC] Restore failed: {e}")196 traceback.print_exc()197 198 self._ensure_default_config()199 self._debug_list_files()200 201 # ── Save (periodic + shutdown) ─────────────────────────────────────202 203 def save_to_repo(self):204 """Upload entire /opt/data directory → dataset (all files, no filtering)"""205 if not self.enabled:206 return207 if not HERMES_DATA.exists():208 print("[SYNC] /opt/data does not exist, nothing to save.")209 return210 211 if not self._ensure_repo_exists():212 print(f"[SYNC] Dataset {HF_REPO_ID} unavailable - skipping save")213 return214 215 print(f"[SYNC] Uploading /opt/data → dataset {HF_REPO_ID}/{DATASET_PATH}/ ...")216 217 try:218 total_size = 0219 file_count = 0220 for root, dirs, fls in os.walk(HERMES_DATA):221 for fn in fls:222 fp = os.path.join(root, fn)223 total_size += os.path.getsize(fp)224 file_count += 1225 print(f"[SYNC] Uploading: {file_count} files, {total_size} bytes total")226 227 if file_count == 0:228 print("[SYNC] Nothing to upload.")229 return230 231 self.api.upload_folder(232 folder_path=str(HERMES_DATA),233 path_in_repo=DATASET_PATH,234 repo_id=HF_REPO_ID,235 repo_type="dataset",236 token=HF_TOKEN,237 commit_message=f"Sync hermes_data — {datetime.now().isoformat()}",238 ignore_patterns=[239 "*.log", # Log files — regenerated on boot240 "*.lock", # Lock files — stale after restart241 "*.tmp", # Temp files242 "*.pid", # PID files243 "*.env", # NEVER upload env files — contain secrets244 ".env", # NEVER upload .env — contain secrets245 "__pycache__", # Python cache246 ".cache/", # Cache dirs (uv, pip, etc.)247 "scripts/*", # HermesFace scripts — from git, not data248 "assets/*", # Static assets — from git, not data249 ],250 )251 print(f"[SYNC] Upload completed at {datetime.now().isoformat()}")252 253 try:254 files = self.api.list_repo_files(repo_id=HF_REPO_ID, repo_type="dataset")255 data_files = [f for f in files if f.startswith(f"{DATASET_PATH}/")]256 print(f"[SYNC] Dataset now has {len(data_files)} files under {DATASET_PATH}/")257 except Exception:258 pass259 260 except Exception as e:261 print(f"[SYNC] Upload failed: {e}")262 traceback.print_exc()263 264 # ── Config helpers ─────────────────────────────────────────────────265 266 def _ensure_default_config(self):267 """Ensure Hermes has config.yaml and .env for HF Spaces."""268 config_path = HERMES_DATA / "config.yaml"269 env_path = HERMES_DATA / ".env"270 soul_path = HERMES_DATA / "SOUL.md"271 272 # Bootstrap from Hermes templates if available273 if not config_path.exists():274 template = APP_DIR / "cli-config.yaml.example"275 if template.exists():276 shutil.copy2(str(template), str(config_path))277 print("[SYNC] Created config.yaml from Hermes template")278 else:279 # Minimal fallback config280 import yaml281 config = {282 "agent": {"name": AGENT_NAME},283 "server": {"host": "0.0.0.0", "port": 7860},284 }285 with open(config_path, "w") as f:286 yaml.dump(config, f, default_flow_style=False)287 print(f"[SYNC] Created minimal config.yaml (agent={AGENT_NAME}, port=7860)")288 289 if not env_path.exists():290 template = APP_DIR / ".env.example"291 if template.exists():292 shutil.copy2(str(template), str(env_path))293 print("[SYNC] Created .env from Hermes template")294 else:295 env_lines = []296 for key in [297 "OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY",298 "NOUS_API_KEY", "GOOGLE_API_KEY", "MISTRAL_API_KEY",299 "TELEGRAM_BOT_TOKEN", "DISCORD_BOT_TOKEN", "SLACK_BOT_TOKEN",300 ]:301 val = os.environ.get(key, "")302 if val:303 env_lines.append(f"{key}={val}")304 if env_lines:305 with open(env_path, "w") as f:306 f.write("\n".join(env_lines) + "\n")307 print(f"[SYNC] Created .env with {len(env_lines)} keys")308 309 if not soul_path.exists():310 template = APP_DIR / "docker" / "SOUL.md"311 if template.exists():312 shutil.copy2(str(template), str(soul_path))313 print("[SYNC] Created SOUL.md from Hermes template")314 else:315 with open(soul_path, "w") as f:316 f.write(f"# {AGENT_NAME}\n\nI am {AGENT_NAME}, a self-improving AI assistant powered by Hermes Agent.\n")317 print("[SYNC] Created default SOUL.md")318 319 def _debug_list_files(self):320 try:321 count = sum(1 for _, _, files in os.walk(HERMES_DATA) for _ in files)322 print(f"[SYNC] Local /opt/data: {count} files")323 except Exception as e:324 print(f"[SYNC] listing failed: {e}")325 326 # ── Background sync loop ──────────────────────────────────────────327 328 def background_sync_loop(self, stop_event):329 print(f"[SYNC] Background sync started (interval={SYNC_INTERVAL}s)")330 while not stop_event.is_set():331 if stop_event.wait(timeout=SYNC_INTERVAL):332 break333 print(f"[SYNC] Periodic sync triggered at {datetime.now().isoformat()}")334 self.save_to_repo()335 336 # ── Application runner ─────────────────────────────────────────────337 338 def _patch_web_server_cors(self):339 """Patch Hermes web_server.py:340 - Allow any origin (HF Spaces iframe, custom domains)341 - Allow iframe embedding in huggingface.co + *.hf.space342 """343 ws_path = APP_DIR / "hermes_cli" / "web_server.py"344 if not ws_path.exists():345 return346 try:347 code = ws_path.read_text()348 changed = False349 350 old_cors = 'allow_origin_regex=r"^https?://(localhost|127\\.0\\.0\\.1)(:\\d+)?$"'351 new_cors = 'allow_origins=["*"]'352 if old_cors in code:353 code = code.replace(old_cors, new_cors)354 changed = True355 print("[SYNC] Patched web_server.py CORS for HF Spaces")356 357 # Neutralise X-Frame-Options so HF Spaces can embed the dashboard.358 for pat in ('X-Frame-Options", "DENY"', 'X-Frame-Options", "SAMEORIGIN"'):359 if pat in code:360 code = code.replace(pat, 'X-Frame-Options", "ALLOWALL"')361 changed = True362 print("[SYNC] Relaxed X-Frame-Options for HF Spaces")363 364 # Relax CSP frame-ancestors if present.365 csp_old = 'frame-ancestors \'none\''366 csp_new = "frame-ancestors 'self' https://huggingface.co https://*.hf.space"367 if csp_old in code:368 code = code.replace(csp_old, csp_new)369 changed = True370 print("[SYNC] Relaxed CSP frame-ancestors for HF Spaces")371 372 if changed:373 ws_path.write_text(code)374 except Exception as e:375 print(f"[SYNC] web_server patch failed (non-fatal): {e}")376 377 def _start_process(self, cmd, label, env, log_path):378 """Helper to start a subprocess with output logging."""379 log_fh = open(log_path, "a")380 try:381 process = subprocess.Popen(382 cmd,383 cwd=str(APP_DIR),384 stdout=subprocess.PIPE,385 stderr=subprocess.STDOUT,386 text=True,387 bufsize=1,388 env=env,389 )390 391 def copy_output():392 try:393 for line in process.stdout:394 log_fh.write(line)395 log_fh.flush()396 stripped = line.strip()397 if not stripped:398 continue399 if any(skip in stripped for skip in [400 'Downloading', 'Fetching', '%|', '━', '───',401 'Already cached', 'Using cache', 'tokenizer',402 '.safetensors', 'model-', 'shard',403 ]):404 continue405 print(line, end='')406 except Exception as e:407 print(f"[SYNC] {label} output error: {e}")408 finally:409 log_fh.close()410 411 threading.Thread(target=copy_output, daemon=True).start()412 print(f"[SYNC] {label} started (PID {process.pid})")413 return process414 except Exception as e:415 log_fh.close()416 print(f"[SYNC] ERROR starting {label}: {e}")417 traceback.print_exc()418 return None419 420 def run_hermes(self):421 """Start Hermes gateway on port 7860 — auth gate on web UI, Telegram bridge outbound."""422 log_dir = HERMES_DATA / "logs"423 log_dir.mkdir(parents=True, exist_ok=True)424 425 if not APP_DIR.exists():426 print(f"[SYNC] ERROR: App directory does not exist: {APP_DIR}")427 return None428 429 hermes_bin = shutil.which("hermes") or str(APP_DIR / ".venv" / "bin" / "hermes")430 if not Path(hermes_bin).exists():431 print("[SYNC] ERROR: hermes CLI not found")432 return None433 434 env = os.environ.copy()435 env["HERMES_HOME"] = str(HERMES_DATA)436 env["GATEWAY_ALLOW_ALL_USERS"] = "true"437 env["PORT"] = "7860"438 439 # ── 1. Patch web dashboard CORS for HF Spaces ────────────────440 self._patch_web_server_cors()441 442 # ── 2. Start hermes gateway on port 7860 ─────────────────────443 gateway_cmd = [hermes_bin, "gateway", "run"]444 print("[SYNC] Starting gateway on port 7860...")445 self.gateway_proc = self._start_process(446 gateway_cmd, "Gateway", env, log_dir / "gateway.log"447 )448 449 return self.gateway_proc450 451 452# ── Main ────────────────────────────────────────────────────────────────────453 454def main():455 try:456 t_main_start = time.time()457 458 t0 = time.time()459 sync = HermesFullSync()460 print(f"[TIMER] sync_hf init: {time.time() - t0:.1f}s")461 462 # 1. Restore463 t0 = time.time()464 sync.load_from_repo()465 print(f"[TIMER] load_from_repo (restore): {time.time() - t0:.1f}s")466 467 # 1b. Build config (OpenModel provider setup)468 print("[SYNC] Building config via build_config.py...")469 t_bc = time.time()470 bc_path = HERMES_DATA / "scripts" / "build_config.py"471 if bc_path.exists():472 subprocess.run([sys.executable, str(bc_path)], check=False)473 print(f"[TIMER] build_config: {time.time() - t_bc:.1f}s")474 else:475 print("[SYNC] WARNING: build_config.py not found, skipping")476 477 # 2. Background sync478 stop_event = threading.Event()479 t = threading.Thread(target=sync.background_sync_loop, args=(stop_event,), daemon=True)480 t.start()481 482 # 3. Start application (Hermes API server will bind port 7860)483 t0 = time.time()484 process = sync.run_hermes()485 print(f"[TIMER] run_hermes launch: {time.time() - t0:.1f}s")486 print(f"[TIMER] Total startup (init → app launched): {time.time() - t_main_start:.1f}s")487 488 # Signal handler489 def handle_signal(sig, frame):490 print(f"\n[SYNC] Signal {sig} received. Shutting down...")491 stop_event.set()492 t.join(timeout=10)493 # Stop gateway494 if hasattr(sync, 'gateway_proc') and sync.gateway_proc:495 sync.gateway_proc.terminate()496 try:497 sync.gateway_proc.wait(timeout=5)498 except subprocess.TimeoutExpired:499 sync.gateway_proc.kill()500 # Stop dashboard501 if process:502 process.terminate()503 try:504 process.wait(timeout=5)505 except subprocess.TimeoutExpired:506 process.kill()507 print("[SYNC] Final sync...")508 sync.save_to_repo()509 sys.exit(0)510 511 signal.signal(signal.SIGINT, handle_signal)512 signal.signal(signal.SIGTERM, handle_signal)513 514 # Wait515 if process is None:516 print("[SYNC] ERROR: Failed to start Hermes process. Exiting.")517 stop_event.set()518 t.join(timeout=5)519 sys.exit(1)520 521 exit_code = process.wait()522 print(f"[SYNC] Hermes exited with code {exit_code}")523 stop_event.set()524 t.join(timeout=10)525 print("[SYNC] Final sync...")526 sync.save_to_repo()527 sys.exit(exit_code)528 529 except Exception as e:530 print(f"[SYNC] FATAL ERROR in main: {e}")531 traceback.print_exc()532 sys.exit(1)533 534 535if __name__ == "__main__":536 main()537