Kustt/mbappe_hermes_hf2
0
1#!/usr/bin/env python32"""3Hermes Agent HF Spaces Persistence — Full Directory Sync4=========================================================5 6Simplified persistence: upload/download the entire /opt/data directory7as-is to/from a Hugging Face Dataset repo.8 9- Startup: per-file hf_hub_download → /opt/data (skip failures)10- Periodic: upload_folder → dataset hermes_data/11- Shutdown: final upload_folder → dataset hermes_data/12"""13 14import os15import sys16import time17import threading18import subprocess19import signal20import shutil21import tempfile22import traceback23from pathlib import Path24from datetime import datetime25# Set timeout BEFORE importing huggingface_hub26os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "300")27os.environ.setdefault("HF_HUB_UPLOAD_TIMEOUT", "600")28os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")29os.environ.setdefault("HF_HUB_VERBOSITY", "warning")30 31import logging as _logging32_logging.getLogger("huggingface_hub").setLevel(_logging.WARNING)33_logging.getLogger("huggingface_hub.utils").setLevel(_logging.WARNING)34_logging.getLogger("filelock").setLevel(_logging.WARNING)35 36from huggingface_hub import HfApi, hf_hub_download37 38# ── Logging helper ──────────────────────────────────────────────────────────39 40class TeeLogger:41 """Duplicate output to stream and file."""42 def __init__(self, filename, stream):43 self.stream = stream44 self.file = open(filename, "a", encoding="utf-8")45 def write(self, message):46 self.stream.write(message)47 self.file.write(message)48 self.flush()49 def flush(self):50 self.stream.flush()51 self.file.flush()52 def fileno(self):53 return self.stream.fileno()54 55# ── Configuration ───────────────────────────────────────────────────────────56 57HF_TOKEN = os.environ.get("HF_TOKEN")58HERMES_DATA = Path("/opt/data")59APP_DIR = Path("/opt/hermes")60DATASET_PATH = "hermes_data"61 62AGENT_NAME = os.environ.get("AGENT_NAME", "HermesFace")63 64# HF Spaces built-in env vars (auto-set by HF runtime)65SPACE_HOST = os.environ.get("SPACE_HOST", "")66SPACE_ID = os.environ.get("SPACE_ID", "")67 68SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", "60"))69AUTO_CREATE_DATASET = os.environ.get("AUTO_CREATE_DATASET", "true").lower() in ("true", "1", "yes")70 71# Dataset repo: auto-derive from SPACE_ID when not explicitly set.72# Format: {username}/{SpaceName}-data73HF_REPO_ID = os.environ.get("HERMES_DATASET_REPO", "")74if not HF_REPO_ID and SPACE_ID:75 HF_REPO_ID = f"{SPACE_ID}-data"76 print(f"[SYNC] HERMES_DATASET_REPO not set — auto-derived from SPACE_ID: {HF_REPO_ID}")77elif not HF_REPO_ID and HF_TOKEN:78 try:79 _api = HfApi(token=HF_TOKEN)80 _username = _api.whoami()["name"]81 HF_REPO_ID = f"{_username}/HermesFace-data"82 print(f"[SYNC] HERMES_DATASET_REPO not set — auto-derived from HF_TOKEN: {HF_REPO_ID}")83 del _api, _username84 except Exception as e:85 print(f"[SYNC] WARNING: Could not derive username from HF_TOKEN: {e}")86 HF_REPO_ID = ""87 88# Setup logging89log_dir = HERMES_DATA / "logs"90log_dir.mkdir(parents=True, exist_ok=True)91sys.stdout = TeeLogger(log_dir / "sync.log", sys.stdout)92sys.stderr = sys.stdout93 94 95# ── Sync Manager ────────────────────────────────────────────────────────────96 97class HermesFullSync:98 """Upload/download the entire /opt/data directory to HF Dataset."""99 100 def __init__(self):101 self.enabled = False102 self.dataset_exists = False103 self.api = None104 105 if not HF_TOKEN:106 print("[SYNC] WARNING: HF_TOKEN not set. Persistence disabled.")107 return108 if not HF_REPO_ID:109 print("[SYNC] WARNING: Could not determine dataset repo (no SPACE_ID or HERMES_DATASET_REPO).")110 print("[SYNC] Persistence disabled.")111 return112 113 self.enabled = True114 self.api = HfApi(token=HF_TOKEN)115 self.dataset_exists = self._ensure_repo_exists()116 117 # ── Repo management ────────────────────────────────────────────────118 119 def _ensure_repo_exists(self):120 """Check if dataset repo exists; auto-create only when AUTO_CREATE_DATASET=true."""121 try:122 self.api.repo_info(repo_id=HF_REPO_ID, repo_type="dataset")123 print(f"[SYNC] Dataset repo found: {HF_REPO_ID}")124 return True125 except Exception:126 if not AUTO_CREATE_DATASET:127 print(f"[SYNC] Dataset repo NOT found: {HF_REPO_ID}")128 print("[SYNC] Set AUTO_CREATE_DATASET=true to auto-create.")129 print("[SYNC] Persistence disabled (app will still run normally).")130 return False131 print(f"[SYNC] Dataset repo NOT found: {HF_REPO_ID} — creating...")132 try:133 self.api.create_repo(134 repo_id=HF_REPO_ID,135 repo_type="dataset",136 private=True,137 )138 print(f"[SYNC] Dataset repo created: {HF_REPO_ID}")139 return True140 except Exception as e:141 print(f"[SYNC] Failed to create dataset repo: {e}")142 return False143 144 # ── Restore (startup) ─────────────────────────────────────────────145 146 def load_from_repo(self):147 """Download from dataset → /opt/data, per-file resilient.148 149 Downloads files one-by-one using hf_hub_download, skipping150 any file that fails (instead of aborting the entire restore).151 Applies the same ignore patterns as upload to avoid downloading152 cache files that were uploaded before the filter was fixed.153 """154 if not self.enabled:155 print("[SYNC] Persistence disabled - skipping restore")156 self._ensure_default_config()157 self._ensure_agents_md_link()158 return159 160 if not self.dataset_exists:161 print(f"[SYNC] Dataset {HF_REPO_ID} does not exist - starting fresh")162 self._ensure_default_config()163 self._ensure_agents_md_link()164 return165 166 print(f"[SYNC] Restoring /opt/data from dataset {HF_REPO_ID} ...")167 HERMES_DATA.mkdir(parents=True, exist_ok=True)168 169 try:170 # List all files in the dataset under DATASET_PATH/171 all_files = self.api.list_repo_files(repo_id=HF_REPO_ID, repo_type="dataset")172 data_files = [f for f in all_files if f.startswith(f"{DATASET_PATH}/")]173 174 if not data_files:175 print(f"[SYNC] No {DATASET_PATH}/ folder in dataset. Starting fresh.")176 self._ensure_default_config()177 self._ensure_agents_md_link()178 return179 180 # Apply ignore filter to skip cache files181 ignore_pats = self._get_ignore_patterns()182 essential_files = []183 skipped_count = 0184 for f in data_files:185 rel = f[len(DATASET_PATH) + 1:] # strip "hermes_data/"186 if any(self._match_ignore(rel, pat) for pat in ignore_pats):187 skipped_count += 1188 continue189 essential_files.append(f)190 191 print(f"[SYNC] Dataset has {len(data_files)} total files under {DATASET_PATH}/")192 print(f"[SYNC] Skipping {skipped_count} cache/derived files")193 print(f"[SYNC] Downloading {len(essential_files)} essential files...")194 195 # Download each file independently into a temp dir, then copy into196 # /opt/data/<rel>. Using local_dir=HERMES_DATA with filename197 # "hermes_data/foo" would nest as /opt/data/hermes_data/foo.198 success_count = 0199 fail_count = 0200 with tempfile.TemporaryDirectory(prefix="hermes_restore_") as tmp_dir:201 for i, f in enumerate(essential_files):202 rel = f[len(DATASET_PATH) + 1:]203 dest = HERMES_DATA / rel204 try:205 dest.parent.mkdir(parents=True, exist_ok=True)206 downloaded = hf_hub_download(207 repo_id=HF_REPO_ID,208 repo_type="dataset",209 filename=f,210 local_dir=tmp_dir,211 token=HF_TOKEN,212 )213 if not os.path.exists(downloaded):214 raise FileNotFoundError(f"download missing: {downloaded}")215 shutil.copy2(downloaded, str(dest))216 success_count += 1217 # Brief yield every 10 files to keep CPU load manageable218 # on constrained HF Spaces (cpu-basic = 2 vCPU).219 if (i + 1) % 10 == 0:220 time.sleep(0.05)221 if (i + 1) % 50 == 0:222 print(f"[SYNC] Progress: {i + 1}/{len(essential_files)} files")223 except Exception as e:224 fail_count += 1225 print(f"[SYNC] WARNING: Failed to download {rel}: {e}")226 continue227 228 print(f"[SYNC] Restore completed: {success_count} files downloaded, {fail_count} failed")229 230 except Exception as e:231 print(f"[SYNC] Restore error (list/download phase): {e}")232 traceback.print_exc()233 234 self._ensure_default_config()235 self._ensure_agents_md_link()236 self._debug_list_files()237 238 # ── Shared ignore patterns ─────────────────────────────────────────239 240 @staticmethod241 def _get_ignore_patterns():242 """Return the canonical ignore list used by BOTH upload and download.243 244 Everything here is either:245 - Regenerated on boot (caches, logs, locks, bytecode)246 - Platform-specific derived data (home/, lsp/)247 - Ephemeral SQLite sidecar files (*.db-wal, *.db-shm, *.db-journal)248 """249 return [250 # ── SQLite ephemera (binary, HF git rejects these) ─────251 "*.db-wal",252 "*.db-shm",253 "*.db-journal",254 255 # ── Caches (regenerated on boot) ───────────────────────256 "home/", # user home: uv/pip/npm caches (~14K files)257 "home/**", # nested variant for safety258 "lsp/", # TypeScript/Python LSP cache (~7K files)259 "lsp/**",260 "cache/", # local hermes cache dir261 "cache/**",262 ".cache/", # huggingface xet cache + uv cache263 ".cache/**",264 ".local/", # pip user installs, tirith state265 ".local/**",266 ".npm/", # npm cache267 ".npm/**",268 ".bash_history",269 ".python_history",270 ".node_repl_history",271 ".wget-hsts",272 ".lesshst",273 274 # ── Logs + lock files ──────────────────────────────────275 "*.log",276 "*.jsonl", # tirith audit logs — may contain tokens277 "*.lock",278 "*.tmp",279 "*.pid",280 "*.pyc",281 "__pycache__/",282 "**/__pycache__/",283 284 # ── Shell/dotfiles (regenerated / not needed) ──────────285 # NOTE: scripts/ IS backed up so Space-side fixes to286 # sync_hf.py / entrypoint survive restart.287 "assets/*",288 "bin/*",289 ".bash_logout",290 ".bashrc",291 ".profile",292 ".bash_profile",293 ".hermes_history",294 ".skills_prompt_snapshot.json",295 "channel_directory.json",296 "feishu_seen_message_ids.json",297 "fix_config.py",298 "verification_evidence.db",299 "verification_evidence.db-wal",300 "verification_evidence.db-shm",301 "verification_evidence.db-journal",302 # SQLite WAL/SHM for any db name (HF git rejects binaries)303 "*.db-wal",304 "*.db-shm",305 "*.db-journal",306 "kanban.db-wal",307 "kanban.db-shm",308 "state.db-wal",309 "state.db-shm",310 311 # ── Ephemeral runtime files ────────────────────────────312 "audio_cache/",313 "audio_cache/**",314 "image_cache/",315 "image_cache/**",316 "pastes/",317 "pastes/**",318 "workspace/",319 "workspace/**",320 "gateway.lock",321 "gateway.pid",322 "gateway_state.json",323 "processes.json",324 "models_dev_cache.json",325 "ollama_cloud_models_cache.json",326 "provider_models_cache.json",327 "auth.lock",328 ".update_check",329 ".clean_shutdown",330 "ticker_heartbeat",331 "ticker_last_success",332 333 # ── Cron output + runtime DBs (regenerated on boot) ────334 "cron/output/",335 "cron/output/**",336 "cron/executions.db",337 338 # ── Bundled skills (reinstalled from Hermes on boot) ──339 "skills/",340 "skills/**",341 342 # ── Large runtime databases ────────────────────────────343 "state.db",344 "kanban.db",345 "projects.db",346 347 # ── Runtime state files ────────────────────────────────348 "logs/curator/",349 "logs/curator/**",350 "tui-theme-boot.json",351 "web-ui-build-stamp.json",352 ".startup_boot_marker",353 ".update_exit_code",354 ".update_output.txt",355 ".update_pending.json",356 ".update_prompt.json",357 ".restart_last_processed.json",358 ".restart_pending.json",359 "sessions/",360 "sessions/**",361 "gateway/restart_loop.json",362 "state/gateway.heartbeat",363 ]364 365 # ── Save (periodic + shutdown) ─────────────────────────────────────366 367 def save_to_repo(self):368 """Upload /opt/data → dataset, skipping caches and ephemera.369 370 Stages only the kept files into a temp directory before upload.371 huggingface_hub's ignore_patterns matching differs from our372 ``_match_ignore`` (e.g. ``*.db-wal`` / top-level filenames), so we373 never rely on upload_folder ignore alone.374 """375 if not self.enabled:376 return377 if not HERMES_DATA.exists():378 print("[SYNC] /opt/data does not exist, nothing to save.")379 return380 381 if not self._ensure_repo_exists():382 print(f"[SYNC] Dataset {HF_REPO_ID} unavailable - skipping save")383 return384 385 print(f"[SYNC] Uploading /opt/data → dataset {HF_REPO_ID}/{DATASET_PATH}/ ...")386 387 try:388 ignore_pats = self._get_ignore_patterns()389 kept = []390 total_size = 0391 for root, dirs, fls in os.walk(HERMES_DATA):392 # prune ignored dirs early393 rel_root = os.path.relpath(root, HERMES_DATA)394 if rel_root == ".":395 rel_root = ""396 dirs[:] = [397 d for d in dirs398 if not any(399 self._match_ignore(400 f"{rel_root}/{d}".lstrip("/") if rel_root else d, pat401 ) or self._match_ignore(402 f"{rel_root}/{d}/".lstrip("/") if rel_root else f"{d}/", pat403 )404 for pat in ignore_pats405 )406 ]407 for fn in fls:408 fp = os.path.join(root, fn)409 rel = os.path.relpath(fp, HERMES_DATA)410 if any(self._match_ignore(rel, pat) for pat in ignore_pats):411 continue412 try:413 sz = os.path.getsize(fp)414 except OSError:415 continue416 kept.append((fp, rel, sz))417 total_size += sz418 419 print(f"[SYNC] Uploading: {len(kept)} files, {total_size} bytes total")420 if not kept:421 print("[SYNC] Nothing to upload.")422 return423 424 with tempfile.TemporaryDirectory(prefix="hermes_upload_") as stage:425 stage_path = Path(stage)426 for src, rel, _sz in kept:427 dest = stage_path / rel428 dest.parent.mkdir(parents=True, exist_ok=True)429 try:430 shutil.copy2(src, dest)431 except Exception as e:432 print(f"[SYNC] WARNING: skip copy {rel}: {e}")433 434 self.api.upload_folder(435 folder_path=str(stage_path),436 path_in_repo=DATASET_PATH,437 repo_id=HF_REPO_ID,438 repo_type="dataset",439 token=HF_TOKEN,440 commit_message=f"Sync hermes_data — {datetime.now().isoformat()}",441 )442 443 print(f"[SYNC] Upload completed at {datetime.now().isoformat()}")444 445 try:446 files = self.api.list_repo_files(repo_id=HF_REPO_ID, repo_type="dataset")447 data_files = [f for f in files if f.startswith(f"{DATASET_PATH}/")]448 print(f"[SYNC] Dataset now has {len(data_files)} files under {DATASET_PATH}/")449 except Exception:450 pass451 452 except Exception as e:453 print(f"[SYNC] Upload failed: {e}")454 traceback.print_exc()455 456 @staticmethod457 def _match_ignore(rel_path, pattern):458 """Simple glob match for a relative path against an ignore pattern."""459 import fnmatch460 # Match basename461 if fnmatch.fnmatch(os.path.basename(rel_path), pattern):462 return True463 # Match full relative path464 if fnmatch.fnmatch(rel_path, pattern):465 return True466 # Match path prefix (for directory patterns like "home/")467 if pattern.endswith("/") and rel_path.startswith(pattern):468 return True469 if pattern.endswith("/**") and rel_path.startswith(pattern[:-3]):470 return True471 # Match anywhere in path (for patterns like "**/__pycache__/")472 if "**" in pattern:473 flat = rel_path.replace(os.sep, "/")474 pat_flat = pattern.replace(os.sep, "/")475 if fnmatch.fnmatch(flat, pat_flat):476 return True477 return False478 479 def cleanup_old_cache_files(self):480 """Delete cache/derived files from the dataset that are no longer backed up.481 482 Call this ONCE after updating the ignore patterns. Uses glob patterns483 to delete entire cache directories (home/, lsp/, .cache/, etc.) that484 were uploaded by the old (pre-fix) script.485 """486 if not self.enabled:487 print("[SYNC] Persistence disabled - skipping cleanup")488 return489 490 # Directories and file patterns to delete from the dataset.491 # These match what the new ignore patterns exclude.492 cleanup_patterns = [493 f"{DATASET_PATH}/home/**",494 f"{DATASET_PATH}/lsp/**",495 f"{DATASET_PATH}/.cache/**",496 f"{DATASET_PATH}/.local/**",497 f"{DATASET_PATH}/.npm/**",498 f"{DATASET_PATH}/*.db-wal",499 f"{DATASET_PATH}/*.db-shm",500 f"{DATASET_PATH}/*.db-journal",501 f"{DATASET_PATH}/audio_cache/**",502 f"{DATASET_PATH}/image_cache/**",503 f"{DATASET_PATH}/pastes/**",504 f"{DATASET_PATH}/workspace/**",505 f"{DATASET_PATH}/gateway.lock",506 f"{DATASET_PATH}/gateway.pid",507 f"{DATASET_PATH}/gateway_state.json",508 f"{DATASET_PATH}/processes.json",509 f"{DATASET_PATH}/models_dev_cache.json",510 f"{DATASET_PATH}/ollama_cloud_models_cache.json",511 f"{DATASET_PATH}/provider_models_cache.json",512 f"{DATASET_PATH}/auth.lock",513 f"{DATASET_PATH}/.update_check",514 f"{DATASET_PATH}/.clean_shutdown",515 f"{DATASET_PATH}/*.log",516 f"{DATASET_PATH}/*.lock",517 f"{DATASET_PATH}/*.pid",518 f"{DATASET_PATH}/*.pyc",519 f"{DATASET_PATH}/__pycache__/**",520 f"{DATASET_PATH}/**/__pycache__/**",521 f"{DATASET_PATH}/ticker_heartbeat",522 f"{DATASET_PATH}/ticker_last_success",523 # ── New patterns from 2026-07-30 optimization ──────────524 f"{DATASET_PATH}/cron/output/**",525 f"{DATASET_PATH}/cron/executions.db",526 f"{DATASET_PATH}/skills/**",527 f"{DATASET_PATH}/state.db",528 f"{DATASET_PATH}/kanban.db",529 f"{DATASET_PATH}/projects.db",530 f"{DATASET_PATH}/logs/curator/**",531 f"{DATASET_PATH}/tui-theme-boot.json",532 f"{DATASET_PATH}/web-ui-build-stamp.json",533 f"{DATASET_PATH}/.startup_boot_marker",534 f"{DATASET_PATH}/.update_exit_code",535 f"{DATASET_PATH}/.update_output.txt",536 f"{DATASET_PATH}/.update_pending.json",537 f"{DATASET_PATH}/.update_prompt.json",538 f"{DATASET_PATH}/.restart_last_processed.json",539 f"{DATASET_PATH}/.restart_pending.json",540 f"{DATASET_PATH}/sessions/**",541 f"{DATASET_PATH}/gateway/restart_loop.json",542 f"{DATASET_PATH}/state/gateway.heartbeat",543 ]544 545 print(f"[SYNC] Deleting old cache files from dataset {HF_REPO_ID}...")546 print(f"[SYNC] Patterns: {cleanup_patterns}")547 548 for pattern in cleanup_patterns:549 try:550 self.api.delete_files(551 repo_id=HF_REPO_ID,552 repo_type="dataset",553 delete_patterns=[pattern],554 token=HF_TOKEN,555 commit_message=f"Cleanup: delete {pattern}",556 )557 print(f"[SYNC] Deleted: {pattern}")558 except Exception as e:559 # "nothing to delete" is not an error560 err_msg = str(e)561 if "nothing to delete" in err_msg.lower() or "no files" in err_msg.lower():562 print(f"[SYNC] Skipped (nothing to delete): {pattern}")563 else:564 print(f"[SYNC] WARNING deleting {pattern}: {e}")565 566 print(f"[SYNC] Cleanup completed")567 568 # ── Config helpers ─────────────────────────────────────────────────569 570 def _ensure_agents_md_link(self):571 """Point /opt/hermes/AGENTS.md at the persistent /opt/data copy.572 573 AGENTS.md is project-context loaded from TERMINAL_CWD (/opt/hermes).574 The real file must live under HERMES_HOME (/opt/data) so sync keeps it;575 recreate the symlink after every restore / boot.576 """577 agents_data = HERMES_DATA / "AGENTS.md"578 agents_link = APP_DIR / "AGENTS.md"579 if not agents_data.exists():580 # First boot: seed from image copy if present581 if agents_link.is_file() and not agents_link.is_symlink():582 try:583 shutil.copy2(str(agents_link), str(agents_data))584 print("[SYNC] Seeded AGENTS.md into /opt/data from /opt/hermes")585 except Exception as e:586 print(f"[SYNC] WARNING: could not seed AGENTS.md: {e}")587 return588 else:589 return590 try:591 if agents_link.is_symlink():592 if agents_link.resolve() == agents_data.resolve():593 return594 agents_link.unlink()595 elif agents_link.exists():596 # Prefer persistent copy; replace image file with symlink597 agents_link.unlink()598 agents_link.symlink_to(agents_data)599 print(f"[SYNC] Linked {agents_link} -> {agents_data}")600 except Exception as e:601 print(f"[SYNC] WARNING: could not link AGENTS.md: {e}")602 603 def _ensure_default_config(self):604 """Ensure Hermes has config.yaml and .env for HF Spaces."""605 config_path = HERMES_DATA / "config.yaml"606 env_path = HERMES_DATA / ".env"607 soul_path = HERMES_DATA / "SOUL.md"608 609 # Bootstrap from Hermes templates if available610 if not config_path.exists():611 template = APP_DIR / "cli-config.yaml.example"612 if template.exists():613 shutil.copy2(str(template), str(config_path))614 print("[SYNC] Created config.yaml from Hermes template")615 else:616 # Minimal fallback config617 import yaml618 config = {619 "agent": {"name": AGENT_NAME},620 "server": {"host": "0.0.0.0", "port": 7860},621 }622 with open(config_path, "w") as f:623 yaml.dump(config, f, default_flow_style=False)624 print(f"[SYNC] Created minimal config.yaml (agent={AGENT_NAME}, port=7860)")625 626 if not env_path.exists():627 template = APP_DIR / ".env.example"628 if template.exists():629 shutil.copy2(str(template), str(env_path))630 print("[SYNC] Created .env from Hermes template")631 else:632 env_lines = []633 for key in [634 "OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY",635 "NOUS_API_KEY", "GOOGLE_API_KEY", "MISTRAL_API_KEY",636 "TELEGRAM_BOT_TOKEN", "DISCORD_BOT_TOKEN", "SLACK_BOT_TOKEN",637 ]:638 val = os.environ.get(key, "")639 if val:640 env_lines.append(f"{key}={val}")641 if env_lines:642 with open(env_path, "w") as f:643 f.write("\n".join(env_lines) + "\n")644 print(f"[SYNC] Created .env with {len(env_lines)} keys")645 646 if not soul_path.exists():647 template = APP_DIR / "docker" / "SOUL.md"648 if template.exists():649 shutil.copy2(str(template), str(soul_path))650 print("[SYNC] Created SOUL.md from Hermes template")651 else:652 with open(soul_path, "w") as f:653 f.write(f"# {AGENT_NAME}\n\nI am {AGENT_NAME}, a self-improving AI assistant powered by Hermes Agent.\n")654 print("[SYNC] Created default SOUL.md")655 656 def _debug_list_files(self):657 try:658 count = sum(1 for _, _, files in os.walk(HERMES_DATA) for _ in files)659 print(f"[SYNC] Local /opt/data: {count} files")660 except Exception as e:661 print(f"[SYNC] listing failed: {e}")662 663 # ── Background sync loop ──────────────────────────────────────────664 665 def _ensure_dashboard_auth(self, env):666 """Ensure dashboard has basic auth configured so it can bind 0.0.0.0.667 668 Hermes v0.15.1+ requires an auth provider when binding to non-loopback.669 We configure a simple username/password from HF secrets or defaults.670 """671 import yaml672 config_path = HERMES_DATA / "config.yaml"673 674 # Read existing config or create empty675 config = {}676 if config_path.exists():677 with open(config_path, "r") as f:678 config = yaml.safe_load(f) or {}679 680 # Only configure if not already set681 if not config.get("dashboard", {}).get("basic_auth"):682 import hashlib, secrets683 username = os.environ.get("DASHBOARD_USERNAME", "admin")684 password = os.environ.get("DASHBOARD_PASSWORD", secrets.token_urlsafe(12))685 password_hash = hashlib.sha256(password.encode()).hexdigest()686 secret = secrets.token_hex(32)687 688 config.setdefault("dashboard", {})["basic_auth"] = {689 "username": username,690 "password_hash": password_hash,691 "secret": secret,692 }693 694 with open(config_path, "w") as f:695 yaml.dump(config, f, default_flow_style=False)696 697 # Write password to a file so user can retrieve it698 creds_path = HERMES_DATA / "logs" / "dashboard_credentials.txt"699 with open(creds_path, "w") as f:700 f.write(f"Username: {username}\nPassword: {password}\n")701 702 print(f"[SYNC] Dashboard auth configured: user={username}, password saved to logs/dashboard_credentials.txt")703 else:704 print("[SYNC] Dashboard auth already configured")705 706 # ── Background sync loop ──────────────────────────────────────────707 708 def background_sync_loop(self, stop_event):709 print(f"[SYNC] Background sync started (interval={SYNC_INTERVAL}s)")710 while not stop_event.is_set():711 if stop_event.wait(timeout=SYNC_INTERVAL):712 break713 print(f"[SYNC] Periodic sync triggered at {datetime.now().isoformat()}")714 self.save_to_repo()715 716 # ── Application runner ─────────────────────────────────────────────717 718 def _ensure_startup_script(self):719 """Create the startup_check.sh script if missing (lost on Docker rebuild)."""720 script_path = HERMES_DATA / "scripts" / "startup_check.sh"721 script_path.parent.mkdir(parents=True, exist_ok=True)722 723 content = """#!/bin/bash724# Auto-generated by sync_hf.py — do not edit manually.725# Detects new Space boot and delivers STARTUP.md once.726BOOT_ID_FILE="/proc/sys/kernel/random/boot_id"727MARKER_FILE="/opt/data/.startup_boot_marker"728STARTUP_MD="/opt/data/STARTUP.md"729 730CURRENT_BOOT=$(cat "$BOOT_ID_FILE" 2>/dev/null)731STORED_BOOT=$(cat "$MARKER_FILE" 2>/dev/null)732 733if [ "$CURRENT_BOOT" != "$STORED_BOOT" ]; then734 echo "$CURRENT_BOOT" > "$MARKER_FILE"735 if [ -f "$STARTUP_MD" ]; then736 echo "🔄 **Space 重启检测** (boot $(echo $CURRENT_BOOT | cut -c1-8)...)"737 echo ""738 cat "$STARTUP_MD"739 else740 echo "🔄 Space 重启检测完成 (STARTUP.md 不存在)"741 fi742fi743"""744 current = script_path.read_text() if script_path.exists() else ""745 if current.strip() != content.strip():746 script_path.write_text(content)747 script_path.chmod(0o755)748 print("[SYNC] Created/updated startup_check.sh")749 750 def _ensure_startup_cron(self):751 """Register/repair the startup cron job with Hermes-compatible schema.752 753 Hermes expects schedule as a structured dict (kind/minutes/display),754 job primary key as ``id`` (not ``job_id``), and repeat as755 ``{times, completed}``. Older bootstrap code wrote a broken flat756 string schedule which crashed the ticker with:757 AttributeError: 'str' object has no attribute 'get'758 """759 import json as _json760 from datetime import timezone as _tz, timedelta as _td761 762 jobs_file = HERMES_DATA / "cron" / "jobs.json"763 jobs_file.parent.mkdir(parents=True, exist_ok=True)764 765 existing: dict = {}766 if jobs_file.exists():767 try:768 existing = _json.loads(jobs_file.read_text())769 except Exception:770 existing = {}771 772 jobs: list = existing.get("jobs", []) if isinstance(existing, dict) else []773 if not isinstance(jobs, list):774 jobs = []775 776 job_id = "startup-bootstrap"777 now = datetime.now(_tz.utc)778 next_run = (now + _td(minutes=1)).isoformat()779 780 def _is_valid_startup_job(j: dict) -> bool:781 if not isinstance(j, dict):782 return False783 if j.get("id") != job_id and j.get("job_id") != job_id:784 return False785 schedule = j.get("schedule")786 if not isinstance(schedule, dict):787 return False788 if schedule.get("kind") not in {"interval", "cron"}:789 return False790 if j.get("id") != job_id:791 return False792 if not isinstance(j.get("repeat"), dict):793 return False794 return True795 796 # Repair existing broken entry in place if present797 for idx, j in enumerate(jobs):798 if not isinstance(j, dict):799 continue800 if j.get("id") == job_id or j.get("job_id") == job_id:801 if _is_valid_startup_job(j):802 print(f"[SYNC] Startup cron job already registered ({job_id})")803 return804 # Broken schema — rewrite with correct Hermes shape805 jobs[idx] = {806 "id": job_id,807 "name": j.get("name") or "Space 启动检测",808 "prompt": "",809 "skills": [],810 "skill": None,811 "model": None,812 "provider": None,813 "provider_snapshot": None,814 "model_snapshot": None,815 "base_url": None,816 "script": j.get("script") or "startup_check.sh",817 "no_agent": True,818 "context_from": None,819 "schedule": {820 "kind": "interval",821 "minutes": 1,822 "display": "every 1m",823 },824 "schedule_display": "every 1m",825 "repeat": {"times": None, "completed": 0},826 "enabled": True,827 "state": "scheduled",828 "paused_at": None,829 "paused_reason": None,830 "created_at": j.get("created_at") or now.isoformat(),831 "next_run_at": j.get("next_run_at") if isinstance(j.get("next_run_at"), str) else next_run,832 "last_run_at": j.get("last_run_at"),833 "last_status": j.get("last_status"),834 "last_error": None,835 "last_delivery_error": None,836 "deliver": j.get("deliver") or "origin",837 "origin": j.get("origin"),838 "enabled_toolsets": None,839 "workdir": None,840 }841 existing["jobs"] = jobs842 existing["updated_at"] = now.isoformat()843 jobs_file.write_text(_json.dumps(existing, indent=2, ensure_ascii=False))844 print(f"[SYNC] Startup cron job repaired to Hermes schema ({job_id})")845 return846 847 new_job = {848 "id": job_id,849 "name": "Space 启动检测",850 "prompt": "",851 "skills": [],852 "skill": None,853 "model": None,854 "provider": None,855 "provider_snapshot": None,856 "model_snapshot": None,857 "base_url": None,858 "script": "startup_check.sh",859 "no_agent": True,860 "context_from": None,861 "schedule": {862 "kind": "interval",863 "minutes": 1,864 "display": "every 1m",865 },866 "schedule_display": "every 1m",867 "repeat": {"times": None, "completed": 0},868 "enabled": True,869 "state": "scheduled",870 "paused_at": None,871 "paused_reason": None,872 "created_at": now.isoformat(),873 "next_run_at": next_run,874 "last_run_at": None,875 "last_status": None,876 "last_error": None,877 "last_delivery_error": None,878 "deliver": "origin",879 "origin": None,880 "enabled_toolsets": None,881 "workdir": None,882 }883 jobs.append(new_job)884 existing["jobs"] = jobs885 existing["updated_at"] = now.isoformat()886 jobs_file.write_text(_json.dumps(existing, indent=2, ensure_ascii=False))887 print(f"[SYNC] Startup cron job registered ({job_id})")888 889 def _patch_web_server_cors(self):890 """Patch Hermes web_server.py:891 - Allow any origin (HF Spaces iframe, custom domains)892 - Allow iframe embedding in huggingface.co + *.hf.space893 - Patch auto-SSO redirect to avoid NotImplementedError with basic_auth894 """895 ws_path = APP_DIR / "hermes_cli" / "web_server.py"896 if not ws_path.exists():897 return898 try:899 code = ws_path.read_text()900 changed = False901 902 old_cors = 'allow_origin_regex=r"^https?://(localhost|127\\.0\\.0\\.1)(:\\d+)?$"'903 new_cors = 'allow_origins=["*"]'904 if old_cors in code:905 code = code.replace(old_cors, new_cors)906 changed = True907 print("[SYNC] Patched web_server.py CORS for HF Spaces")908 909 # Neutralise X-Frame-Options so HF Spaces can embed the dashboard.910 for pat in ('X-Frame-Options", "DENY"', 'X-Frame-Options", "SAMEORIGIN"'):911 if pat in code:912 code = code.replace(pat, 'X-Frame-Options", "ALLOWALL"')913 changed = True914 print("[SYNC] Relaxed X-Frame-Options for HF Spaces")915 916 # Relax CSP frame-ancestors if present.917 csp_old = 'frame-ancestors \'none\''918 csp_new = "frame-ancestors 'self' https://huggingface.co https://*.hf.space"919 if csp_old in code:920 code = code.replace(csp_old, csp_new)921 changed = True922 print("[SYNC] Relaxed CSP frame-ancestors for HF Spaces")923 924 if changed:925 ws_path.write_text(code)926 except Exception as e:927 print(f"[SYNC] web_server patch failed (non-fatal): {e}")928 929 def _patch_web_server_accept_all_hosts(self):930 """Patch Hermes _is_accepted_host to accept any host (HF Spaces proxy).931 932 When proxying through HF Spaces nginx, the Host header is the public hostname,933 not 127.0.0.1. Hermes rejects these with 400 'Invalid Host header'.934 """935 ws_path = APP_DIR / "hermes_cli" / "web_server.py"936 if not ws_path.exists():937 return938 try:939 code = ws_path.read_text()940 changed = False941 942 # Patch _is_accepted_host: accept any host943 old_marker = "def _is_accepted_host("944 if old_marker in code and "def _is_accepted_host(\n " in code:945 # Find the function and prepend unconditional return True946 idx = code.find(old_marker)947 # Insert " return True # Patched for HF Spaces\n" at start of function body948 line_start = code.find("\n", idx) + 1 # end of def line949 insert_at = code.find("\n", line_start) + 1 # skip docstring opening line950 # Simpler: insert right after def line951 def_end = code.find(":\n", idx) + 2952 code = code[:def_end] + " return True # Patched: accept all hosts (HF Spaces)\n" + code[def_end:]953 changed = True954 print("[SYNC] Patched _is_accepted_host to accept all hosts")955 956 # Same for WS check (_ws_host_origin_reason in v0.17+)957 ws_marker = "def _ws_host_origin_reason("958 if ws_marker in code:959 idx2 = code.find(ws_marker)960 def_end2 = code.find(":\n", idx2) + 2961 code = code[:def_end2] + " return None # Patched: accept all WS clients (HF Spaces)\n" + code[def_end2:]962 changed = True963 print("[SYNC] Patched _ws_client_is_allowed to accept all clients")964 965 if changed:966 ws_path.write_text(code)967 except Exception as e:968 print(f"[SYNC] accept_all_hosts patch failed (non-fatal): {e}")969 970 def _patch_middleware_auto_sso(self):971 """Patch middleware.py to skip auto-SSO redirect when only basic_auth is configured.972 973 Hermes v0.17+ has a bug: _auto_sso_response() redirects / to /auth/login?provider=basic974 which crashes BasicAuthProvider (NotImplementedError). We skip auto-SSO when the only975 provider is password-only.976 """977 mw_path = APP_DIR / "hermes_cli" / "middleware.py"978 if not mw_path.exists():979 print("[SYNC] middleware.py not found, skipping auto-SSO patch")980 return981 try:982 code = mw_path.read_text()983 changed = False984 985 # Patch: skip auto-SSO redirect for password-only providers986 # Find the _auto_sso_response function and add a guard987 old_pattern = "providers = list_session_providers()"988 if old_pattern in code:989 guard = '''providers = list_session_providers()990 # Patched: skip auto-SSO redirect when the only provider is password-only (basic_auth)991 if len(providers) == 1 and getattr(providers[0], "supports_password", False):992 return None'''993 code = code.replace(old_pattern, guard)994 changed = True995 print("[SYNC] Patched middleware.py auto-SSO for basic_auth")996 997 if changed:998 mw_path.write_text(code)999 except Exception as e:1000 print(f"[SYNC] middleware patch failed (non-fatal): {e}")1001 1002 def _start_process(self, cmd, label, env, log_path):1003 """Helper to start a subprocess with output logging."""1004 log_fh = open(log_path, "a")1005 try:1006 process = subprocess.Popen(1007 cmd,1008 cwd=str(APP_DIR),1009 stdout=subprocess.PIPE,1010 stderr=subprocess.STDOUT,1011 text=True,1012 bufsize=1,1013 env=env,1014 )1015 1016 def copy_output():1017 try:1018 for line in process.stdout:1019 log_fh.write(line)1020 log_fh.flush()1021 stripped = line.strip()1022 if not stripped:1023 continue1024 if any(skip in stripped for skip in [1025 'Downloading', 'Fetching', '%|', '━', '───',1026 'Already cached', 'Using cache', 'tokenizer',1027 '.safetensors', 'model-', 'shard',1028 ]):1029 continue1030 print(line, end='')1031 except Exception as e:1032 print(f"[SYNC] {label} output error: {e}")1033 finally:1034 log_fh.close()1035 1036 threading.Thread(target=copy_output, daemon=True).start()1037 print(f"[SYNC] {label} started (PID {process.pid})")1038 return process1039 except Exception as e:1040 log_fh.close()1041 print(f"[SYNC] ERROR starting {label}: {e}")1042 traceback.print_exc()1043 return None1044 1045 def start_dashboard_and_proxy(self):1046 """Start dashboard + reverse proxy immediately (3 s) for HF health check.1047 Returns (dashboard_proc, proxy_proc)."""1048 log_dir = HERMES_DATA / "logs"1049 log_dir.mkdir(parents=True, exist_ok=True)1050 1051 hermes_bin = shutil.which("hermes") or str(APP_DIR / ".venv" / "bin" / "hermes")1052 1053 env = os.environ.copy()1054 env["HERMES_HOME"] = str(HERMES_DATA)1055 env["GATEWAY_ALLOW_ALL_USERS"] = "true"1056 env.pop("API_SERVER_ENABLED", None)1057 env.pop("API_SERVER_PORT", None)1058 1059 # Patch Hermes to accept all hosts1060 self._patch_web_server_accept_all_hosts()1061 1062 # ── Start dashboard ──1063 dashboard_cmd = [hermes_bin, "dashboard", "--host", "127.0.0.1", "--port", "7861", "--no-open"]1064 print("[SYNC] Starting web dashboard on 127.0.0.1:7861...")1065 dashboard_proc = self._start_process(dashboard_cmd, "Dashboard", env, log_dir / "dashboard.log")1066 1067 # ── Start reverse proxy ──1068 time.sleep(2) # Brief pause for dashboard to bind1069 proxy_script = HERMES_DATA / "scripts" / "proxy_server.py"1070 proxy_cmd = [sys.executable, "-u", str(proxy_script)]1071 print("[SYNC] Starting reverse proxy on 0.0.0.0:7860 → 127.0.0.1:7861...")1072 proxy_proc = self._start_process(proxy_cmd, "Proxy", env, log_dir / "proxy.log")1073 1074 # Store for signal handler1075 self._early_proxy_proc = proxy_proc1076 return dashboard_proc, proxy_proc1077 1078 def start_gateway_and_cron(self):1079 """Start llm-all proxy + gateway + cron bootstrap (called AFTER restore)."""1080 log_dir = HERMES_DATA / "logs"1081 env = os.environ.copy()1082 env["HERMES_HOME"] = str(HERMES_DATA)1083 env["GATEWAY_ALLOW_ALL_USERS"] = "true"1084 1085 # ── llm-all.pro proxy ──1086 time.sleep(1)1087 llm_all_script = HERMES_DATA / "scripts" / "llm_all_proxy.py"1088 if llm_all_script.exists():1089 llm_all_cmd = [sys.executable, "-u", str(llm_all_script), "--port", "8787"]1090 print("[SYNC] Starting llm-all.pro proxy on 127.0.0.1:8787...")1091 self.llm_all_proxy_proc = self._start_process(llm_all_cmd, "llm-all-proxy", env, log_dir / "llm_all_proxy.log")1092 else:1093 print("[SYNC] llm_all_proxy.py not found, skipping")1094 1095 # ── Gateway ──1096 time.sleep(2)1097 gateway_env = env.copy()1098 gateway_env["GATEWAY_ALLOW_ALL_USERS"] = "true"1099 gateway_cmd = [shutil.which("hermes") or str(APP_DIR / ".venv" / "bin" / "hermes"), "gateway"]1100 print("[SYNC] Starting gateway (messaging platforms)...")1101 self.gateway_proc = self._start_process(gateway_cmd, "Gateway", gateway_env, log_dir / "gateway.log")1102 1103 # ── Cron bootstrap ──1104 time.sleep(5)1105 self._ensure_startup_script()1106 self._ensure_startup_cron()1107 1108 1109# ── Main ────────────────────────────────────────────────────────────────────1110 1111def main():1112 try:1113 t_main_start = time.time()1114 1115 t0 = time.time()1116 sync = HermesFullSync()1117 print(f"[TIMER] sync_hf init: {time.time() - t0:.1f}s")1118 1119 # 0. Pre-create essential dirs so dashboard can start immediately1120 for d in ["sessions", "logs", "hooks", "memories", "skills", "skins", "plans", "workspace"]:1121 (HERMES_DATA / d).mkdir(parents=True, exist_ok=True)1122 1123 # 1. Start dashboard + proxy IMMEDIATELY (3 s) so HF health check passes1124 t0 = time.time()1125 dashboard_proc, proxy_proc = sync.start_dashboard_and_proxy()1126 print(f"[TIMER] Dashboard+proxy launch: {time.time() - t0:.1f}s")1127 1128 # 2. Restore in background, then start gateway + cron1129 restore_done = threading.Event()1130 1131 def _restore_then_gateway():1132 t0 = time.time()1133 sync.load_from_repo()1134 print(f"[TIMER] load_from_repo (restore): {time.time() - t0:.1f}s")1135 restore_done.set()1136 # Background sync loop1137 stop_event = threading.Event()1138 threading.Thread(target=sync.background_sync_loop, args=(stop_event,), daemon=True).start()1139 # Gateway1140 t0 = time.time()1141 sync.start_gateway_and_cron()1142 print(f"[TIMER] Gateway+cron launch: {time.time() - t0:.1f}s")1143 1144 bg = threading.Thread(target=_restore_then_gateway, daemon=True)1145 bg.start()1146 1147 print(f"[TIMER] Total startup (dashboard+proxy ready): {time.time() - t_main_start:.1f}s")1148 1149 # Signal handler1150 def handle_signal(sig, frame):1151 print(f"\n[SYNC] Signal {sig} received. Shutting down...")1152 # Stop gateway1153 if hasattr(sync, 'gateway_proc') and sync.gateway_proc:1154 sync.gateway_proc.terminate()1155 try:1156 sync.gateway_proc.wait(timeout=5)1157 except subprocess.TimeoutExpired:1158 sync.gateway_proc.kill()1159 # Stop dashboard1160 if dashboard_proc:1161 dashboard_proc.terminate()1162 try:1163 dashboard_proc.wait(timeout=5)1164 except subprocess.TimeoutExpired:1165 dashboard_proc.kill()1166 # Stop proxy1167 if proxy_proc:1168 proxy_proc.terminate()1169 try:1170 proxy_proc.wait(timeout=3)1171 except subprocess.TimeoutExpired:1172 proxy_proc.kill()1173 print("[SYNC] Final sync...")1174 sync.save_to_repo()1175 sys.exit(0)1176 1177 signal.signal(signal.SIGINT, handle_signal)1178 signal.signal(signal.SIGTERM, handle_signal)1179 1180 # Wait for dashboard (foreground, keeps container alive)1181 if dashboard_proc is None:1182 print("[SYNC] ERROR: Failed to start dashboard. Exiting.")1183 sys.exit(1)1184 1185 exit_code = dashboard_proc.wait()1186 print(f"[SYNC] Hermes exited with code {exit_code}")1187 # Stop gateway (may be running in background thread)1188 if hasattr(sync, 'gateway_proc') and sync.gateway_proc:1189 sync.gateway_proc.terminate()1190 print("[SYNC] Final sync...")1191 sync.save_to_repo()1192 sys.exit(exit_code)1193 1194 except Exception as e:1195 print(f"[SYNC] FATAL ERROR in main: {e}")1196 traceback.print_exc()1197 sys.exit(1)1198 1199 1200if __name__ == "__main__":