Byakk1/hermes333
0
1#!/usr/bin/env python32"""3Hermes Agent Data Sync Service4Handles data persistence to/from Hugging Face Dataset5"""6 7import os8import sys9import time10import json11import shutil12import tarfile13import argparse14from pathlib import Path15from datetime import datetime16from typing import Optional, Dict, List17 18from huggingface_hub import HfApi, hf_hub_download, upload_folder19from loguru import logger20import re21 22# 文件监控（可选）23try:24 from watchdog.observers import Observer25 from watchdog.events import FileSystemEventHandler26 WATCHDOG_AVAILABLE = True27except ImportError:28 WATCHDOG_AVAILABLE = False29 logger.warning("watchdog not installed, file change detection disabled")30 31 32class DatasetManager:33 """Manages data synchronization with Hugging Face Dataset"""34 35 def __init__(self, dataset_repo: Optional[str] = None, token: Optional[str] = None):36 self.dataset_repo = dataset_repo or os.environ.get('HF_DATASET_REPO')37 self.token = token or os.environ.get('HF_TOKEN') or os.environ.get('HUGGING_FACE_HUB_TOKEN')38 self.api = HfApi(token=self.token)39 self.hermes_home = Path(os.environ.get('HERMES_HOME', '/data/.hermes'))40 self.temp_dir = Path('/tmp/hermes_sync')41 42 # 数据路径映射43 self.path_mapping = {44 'config': self.hermes_home / 'config.yaml',45 'env': self.hermes_home / '.env',46 'auth': self.hermes_home / 'auth.json',47 'soul': self.hermes_home / 'SOUL.md',48 'memories': self.hermes_home / 'memories',49 'skills': self.hermes_home / 'skills',50 'sessions': self.hermes_home / 'sessions',51 'state_db': self.hermes_home / 'state.db',52 'logs': self.hermes_home / 'logs',53 'cron': self.hermes_home / 'cron',54 'webui_token': Path('/data/.hermes-web-ui') / '.token',55 'image_cache': self.hermes_home / 'image_cache',56 'baoyu_skills': Path('/home/appuser/.baoyu-skills'),57 }58 59 def validate(self) -> bool:60 """验证配置是否正确"""61 if not self.dataset_repo:62 logger.error("HF_DATASET_REPO not set")63 return False64 65 if not self.token:66 logger.warning("HF_TOKEN not set, will try public dataset")67 68 return True69 70 def prepare_backup_data(self) -> Path:71 """准备备份数据到临时目录"""72 logger.info("Preparing backup data...")73 74 # 清理并创建临时目录75 if self.temp_dir.exists():76 shutil.rmtree(self.temp_dir)77 self.temp_dir.mkdir(parents=True)78 79 # 创建目录结构80 (self.temp_dir / 'config').mkdir()81 (self.temp_dir / 'personality').mkdir()82 (self.temp_dir / 'memories').mkdir()83 (self.temp_dir / 'skills').mkdir()84 (self.temp_dir / 'sessions').mkdir()85 (self.temp_dir / 'state').mkdir()86 (self.temp_dir / 'logs').mkdir()87 (self.temp_dir / 'cron').mkdir()88 (self.temp_dir / 'webui').mkdir()89 (self.temp_dir / 'image_cache').mkdir()90 (self.temp_dir / 'baoyu_skills').mkdir()91 92 # 复制文件93 try:94 # 配置文件95 if self.path_mapping['config'].exists():96 shutil.copy2(self.path_mapping['config'], self.temp_dir / 'config' / 'config.yaml')97 98 # 环境变量（敏感信息）- 上传前 masking 处理99 if self.path_mapping['env'].exists():100 env_src = self.path_mapping['env']101 env_dst = self.temp_dir / 'config' / '.env'102 with open(env_src, 'r', encoding='utf-8') as f:103 env_content = f.read()104 # Mask Hugging Face tokens to prevent upload rejection105 env_content = re.sub(r'(HF_TOKEN|HUGGING_FACE_HUB_TOKEN)=hf_[a-zA-Z0-9]+', r'\1=***masked***', env_content)106 env_content = re.sub(r'(HF_TOKEN|HUGGING_FACE_HUB_TOKEN)=.+', r'\1=***masked***', env_content)107 with open(env_dst, 'w', encoding='utf-8') as f:108 f.write(env_content)109 110 # OAuth 认证111 if self.path_mapping['auth'].exists():112 shutil.copy2(self.path_mapping['auth'], self.temp_dir / 'config' / 'auth.json')113 114 # 人格定义115 if self.path_mapping['soul'].exists():116 shutil.copy2(self.path_mapping['soul'], self.temp_dir / 'personality' / 'SOUL.md')117 118 # 记忆119 if self.path_mapping['memories'].exists():120 shutil.copytree(self.path_mapping['memories'], self.temp_dir / 'memories', dirs_exist_ok=True)121 122 # 技能123 if self.path_mapping['skills'].exists():124 shutil.copytree(self.path_mapping['skills'], self.temp_dir / 'skills', dirs_exist_ok=True)125 126 # 会话127 if self.path_mapping['sessions'].exists():128 shutil.copytree(self.path_mapping['sessions'], self.temp_dir / 'sessions', dirs_exist_ok=True)129 130 # 数据库131 if self.path_mapping['state_db'].exists():132 shutil.copy2(self.path_mapping['state_db'], self.temp_dir / 'state' / 'state.db')133 134 # 日志135 if self.path_mapping['logs'].exists():136 shutil.copytree(self.path_mapping['logs'], self.temp_dir / 'logs', dirs_exist_ok=True)137 138 # 定时任务139 if self.path_mapping['cron'].exists():140 shutil.copytree(self.path_mapping['cron'], self.temp_dir / 'cron', dirs_exist_ok=True)141 142 # Image Cache143 if self.path_mapping['image_cache'].exists():144 shutil.copytree(self.path_mapping['image_cache'], self.temp_dir / 'image_cache', dirs_exist_ok=True)145 146 # baoyu-skills 用户配置 (EXTEND.md 等)147 if self.path_mapping['baoyu_skills'].exists():148 shutil.copytree(self.path_mapping['baoyu_skills'], self.temp_dir / 'baoyu_skills', dirs_exist_ok=True)149 150 # WebUI 认证 Token151 if self.path_mapping['webui_token'].exists():152 (self.temp_dir / 'webui').mkdir(exist_ok=True)153 shutil.copy2(self.path_mapping['webui_token'], self.temp_dir / 'webui' / '.token')154 155 # 修复临时目录权限（源文件可能被设为只读，如 baoyu-imagine scripts 的 555）156 # upload_folder 需要能正常读取所有文件157 logger.info("Fixing permissions in temp backup dir...")158 import stat159 for root, dirs, files in os.walk(self.temp_dir):160 for d in dirs:161 dir_path = os.path.join(root, d)162 try:163 os.chmod(dir_path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)164 except Exception:165 pass166 for f in files:167 file_path = os.path.join(root, f)168 try:169 os.chmod(file_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)170 except Exception:171 pass172 173 # 添加元数据174 metadata = {175 'timestamp': datetime.now().isoformat(),176 'version': '0.10.0',177 'hermes_home': str(self.hermes_home)178 }179 with open(self.temp_dir / 'metadata.json', 'w') as f:180 json.dump(metadata, f, indent=2)181 182 logger.success(f"Backup prepared at {self.temp_dir}")183 return self.temp_dir184 185 except Exception as e:186 logger.error(f"Failed to prepare backup: {e}")187 raise188 189 def upload_to_dataset(self, force: bool = False) -> bool:190 """上传数据到 Hugging Face Dataset"""191 try:192 backup_dir = self.prepare_backup_data()193 194 logger.info(f"Uploading to dataset: {self.dataset_repo}")195 196 # 上传文件夹到 dataset197 self.api.upload_folder(198 folder_path=str(backup_dir),199 repo_id=self.dataset_repo,200 repo_type="dataset",201 commit_message=f"Hermes Agent backup - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"202 )203 204 logger.success("Backup uploaded successfully")205 return True206 207 except Exception as e:208 logger.error(f"Failed to upload to dataset: {e}")209 return False210 211 def download_from_dataset(self) -> bool:212 """从 Hugging Face Dataset 下载数据"""213 try:214 logger.info(f"Downloading from dataset: {self.dataset_repo}")215 216 # 创建临时下载目录217 download_dir = Path('/tmp/hermes_download')218 if download_dir.exists():219 shutil.rmtree(download_dir)220 download_dir.mkdir(parents=True)221 222 # 下载所有文件223 self.api.snapshot_download(224 repo_id=self.dataset_repo,225 repo_type="dataset",226 local_dir=str(download_dir)227 )228 229 logger.success("Download completed")230 231 # 恢复数据到 Hermes 目录232 self.restore_from_download(download_dir)233 return True234 235 except Exception as e:236 logger.error(f"Failed to download from dataset: {e}")237 return False238 239 def restore_from_download(self, download_dir: Path):240 """从下载的目录恢复数据241 242 注意: config.yaml 在恢复时被跳过，因为 entrypoint.sh 会根据环境变量243 重新生成正确的 config.yaml。如果恢复旧的 config.yaml，会导致模型244 配置被覆盖（例如 minimaxai/minimax-m2.7 被替换为旧模型）。245 """246 logger.info("Restoring data to Hermes home...")247 248 # 确保目标目录存在249 self.hermes_home.mkdir(parents=True, exist_ok=True)250 251 # 恢复策略控制：252 # SKIP_CONFIG_RESTORE=true（默认）：跳过由 entrypoint.sh 动态生成的配置，防止旧备份覆盖新配置253 # 跳过项：config.yaml、baoyu_skills/254 # 恢复项：skills/（Skills Hub 安装的技能，entrypoint.sh 不会重新安装 cover/illustrator 等）255 # 保留项：memories、sessions、state.db、.env、auth.json、logs、cron、webui/.token、SOUL.md（用户数据）256 # SKIP_CONFIG_RESTORE=false：恢复所有备份（配置稳定后使用）257 skip_restore = os.environ.get('SKIP_CONFIG_RESTORE', 'true').lower() in ('true', '1', 'yes')258 259 # 定义跳过恢复的路径（entrypoint.sh 会重新生成这些配置）260 # NOTE: skills 目录不跳过！它包含 Skills Hub 安装的技能（cover-image, article-illustrator 等），261 # entrypoint.sh 只重新安装 baoyu-imagine，不会安装其他技能。262 # 恢复后 entrypoint.sh 会覆盖 baoyu-imagine 确保其最新（第652-654行 cp -r）。263 if skip_restore:264 skip_paths = {265 'config/config.yaml', # 模型/供应商配置由 entrypoint.sh 根据环境变量生成266 'baoyu_skills', # baoyu-skills EXTEND.md 由 entrypoint.sh 重新生成267 }268 logger.info(f"SKIP_CONFIG_RESTORE=true, skipping: {', '.join(skip_paths)}")269 else:270 skip_paths = set()271 logger.info("SKIP_CONFIG_RESTORE=false, restoring all backed-up configurations")272 273 restore_mapping = {274 'config/.env': self.path_mapping['env'],275 'config/auth.json': self.path_mapping['auth'],276 'personality/SOUL.md': self.path_mapping['soul'],277 'memories': self.path_mapping['memories'],278 'skills': self.path_mapping['skills'],279 'sessions': self.path_mapping['sessions'],280 'state/state.db': self.path_mapping['state_db'],281 'logs': self.path_mapping['logs'],282 'cron': self.path_mapping['cron'],283 'webui/.token': self.path_mapping['webui_token'],284 'image_cache': self.path_mapping['image_cache'],285 'baoyu_skills': self.path_mapping['baoyu_skills'],286 }287 288 # config.yaml 特殊处理：289 # 即使不在 skip_paths 中，如果 skip_restore=true，也不直接覆盖，而是保存到 .restored 供合并290 if not skip_restore:291 restore_mapping['config/config.yaml'] = self.path_mapping['config']292 else:293 # 恢复到 .restored 文件，供 entrypoint.sh 合并用户修改的配置区块（如 channels、display 等）294 restored_path = self.hermes_home / 'config.yaml.restored'295 src = download_dir / 'config' / 'config.yaml'296 if src.exists():297 shutil.copy2(src, restored_path)298 logger.info("Restored config.yaml to config.yaml.restored for merge")299 300 for src_rel, dst in restore_mapping.items():301 # 检查是否在跳过列表中302 if src_rel in skip_paths:303 logger.info(f"Skipping restore of {src_rel} (will be regenerated by entrypoint.sh)")304 continue305 306 src = download_dir / src_rel307 if src.exists():308 try:309 if src.is_file():310 dst.parent.mkdir(parents=True, exist_ok=True)311 shutil.copy2(src, dst)312 logger.info(f"Restored: {src_rel}")313 elif src.is_dir():314 if dst.exists():315 shutil.rmtree(dst)316 shutil.copytree(src, dst)317 logger.info(f"Restored directory: {src_rel}")318 except Exception as e:319 logger.error(f"Failed to restore {src_rel}: {e}")320 else:321 logger.warning(f"Not found in backup: {src_rel}")322 323 logger.success("Data restoration completed")324 325 326class ConfigFileHandler(FileSystemEventHandler):327 """配置文件变化处理器 - 实时同步到 Dataset 并触发重载"""328 329 # 启动静默期（秒）：在此期间内的文件变更不予备份，避免启动阶段冗余上传330 STARTUP_GRACE_PERIOD = 30331 332 def __init__(self, manager: DatasetManager):333 self.manager = manager334 self.last_backup_time = 0335 self.backup_cooldown = 5 # 5秒内不重复备份336 self.start_time = time.time() # 记录处理器创建时间337 self._startup_logged = False338 339 def on_modified(self, event):340 """文件被修改时触发"""341 if event.is_directory:342 return343 344 # 启动静默期：跳过启动阶段的配置变更备份345 elapsed = time.time() - self.start_time346 if elapsed < self.STARTUP_GRACE_PERIOD:347 if not self._startup_logged:348 logger.info(f"In startup grace period ({int(self.STARTUP_GRACE_PERIOD - elapsed)}s remaining), skipping backup for: {event.src_path}")349 self._startup_logged = True350 return351 352 # 只关注关键配置文件353 watched_files = ['config.yaml', '.env', 'auth.json']354 if any(event.src_path.endswith(f) for f in watched_files):355 current_time = time.time()356 if current_time - self.last_backup_time > self.backup_cooldown:357 logger.info(f"Config file changed: {event.src_path}")358 logger.info("Triggering immediate backup...")359 try:360 self.manager.upload_to_dataset()361 self.last_backup_time = current_time362 logger.success("Immediate backup completed")363 364 # 尝试触发 Hermes 配置重载365 self._trigger_reload()366 367 except Exception as e:368 logger.error(f"Immediate backup failed: {e}")369 370 def _trigger_reload(self):371 """尝试触发 Hermes 配置重载"""372 # 注意：Hermes 目前没有 config reload 命令373 # 配置将在下次 Space 重启时自动生效374 logger.info("Configuration saved. Please restart Space to apply changes immediately.")375 376 377def run_daemon():378 """后台守护进程模式 - 定期同步 + 实时文件监听"""379 logger.info("Starting data sync daemon...")380 381 sync_interval = int(os.environ.get('SYNC_INTERVAL', '60')) # 默认60秒（实时模式）382 manager = DatasetManager()383 384 if not manager.validate():385 logger.error("Configuration invalid, exiting")386 sys.exit(1)387 388 logger.info(f"Sync interval: {sync_interval} seconds")389 390 # 如果 watchdog 可用，启动文件监听391 observer = None392 if WATCHDOG_AVAILABLE:393 try:394 logger.info("Starting file watcher for real-time sync...")395 event_handler = ConfigFileHandler(manager)396 observer = Observer()397 observer.schedule(event_handler, str(manager.hermes_home), recursive=False)398 observer.start()399 logger.success("File watcher started - config changes will trigger immediate backup")400 except Exception as e:401 logger.error(f"Failed to start file watcher: {e}")402 logger.warning("Falling back to scheduled sync only")403 observer = None404 else:405 logger.warning("Watchdog not available, using scheduled sync only")406 407 try:408 while True:409 try:410 time.sleep(sync_interval)411 logger.info("Performing scheduled backup...")412 manager.upload_to_dataset()413 except KeyboardInterrupt:414 logger.info("Daemon stopped")415 break416 except Exception as e:417 logger.error(f"Sync error: {e}")418 finally:419 # 清理文件监听器420 if observer:421 logger.info("Stopping file watcher...")422 observer.stop()423 observer.join()424 logger.info("File watcher stopped")425 426 427def main():428 parser = argparse.ArgumentParser(description='Hermes Agent Data Sync')429 parser.add_argument('action', choices=['backup', 'restore', 'daemon'],430 help='Action to perform')431 parser.add_argument('--force', '-f', action='store_true',432 help='Force backup even if no changes')433 434 args = parser.parse_args()435 436 manager = DatasetManager()437 438 if not manager.validate():439 logger.error("Configuration invalid")440 sys.exit(1)441 442 if args.action == 'backup':443 success = manager.upload_to_dataset(force=args.force)444 sys.exit(0 if success else 1)445 446 elif args.action == 'restore':447 success = manager.download_from_dataset()448 sys.exit(0 if success else 1)449 450 elif args.action == 'daemon':451 run_daemon()452 453 454if __name__ == '__main__':455 main()456