agenticverse/AgentNexa

1#!/usr/bin/env python32"""3HermesFace Full Directory Persistence for Hugging Face Spaces4=============================================================5 6Tar-gz snapshot of /opt/data with atomic upload, rotation, and integrity check.7 8Usage:9    python3 hermes_persist.py save     # backup10    python3 hermes_persist.py load     # restore latest11    python3 hermes_persist.py status   # show current backups12 13Env vars:14    HF_TOKEN              HF access token with write permission15    HERMES_DATASET_REPO   Target dataset repo (e.g. username/HermesFace-data)16    HERMES_HOME           Source directory (default: /opt/data)17    MAX_BACKUPS           Rotation size (default: 5)18"""19import hashlib20import json21import os22import shutil23import sys24import tarfile25import tempfile26from datetime import datetime27from pathlib import Path28from typing import Optional29 30from huggingface_hub import HfApi, hf_hub_download31from huggingface_hub.utils import RepositoryNotFoundError32 33 34class Config:35    HERMES_HOME = Path(os.environ.get("HERMES_HOME", "/opt/data"))36    DATASET_REPO = os.environ.get("HERMES_DATASET_REPO", "")37    HF_TOKEN = os.environ.get("HF_TOKEN", "")38    MAX_BACKUPS = int(os.environ.get("MAX_BACKUPS", "5"))39    BACKUP_PREFIX = "backup-"40 41    EXCLUDE_SUFFIXES = (".lock", ".tmp", ".pyc", ".socket", ".pid")42    EXCLUDE_NAMES = {"__pycache__", "node_modules", ".DS_Store", ".git"}43    SKIP_DIRS = {".cache", "logs/sys_logs"}44 45 46def _log(level: str, msg: str, **kv: object) -> None:47    payload = {"timestamp": datetime.now().isoformat(), "level": level,48               "module": "hermes-persist", "message": msg, **kv}49    print(json.dumps(payload))50 51 52def _tar_filter(info: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:53    name = info.name54    if any(name.endswith(suf) for suf in Config.EXCLUDE_SUFFIXES):55        return None56    parts = set(Path(name).parts)57    if parts & Config.EXCLUDE_NAMES:58        return None59    rel = name.lstrip("./")60    for skip in Config.SKIP_DIRS:61        if rel == skip or rel.startswith(skip + "/"):62            return None63    return info64 65 66def _api() -> HfApi:67    if not Config.HF_TOKEN:68        raise RuntimeError("HF_TOKEN not set")69    if not Config.DATASET_REPO:70        raise RuntimeError("HERMES_DATASET_REPO not set")71    return HfApi(token=Config.HF_TOKEN)72 73 74def _ensure_repo(api: HfApi) -> None:75    try:76        api.repo_info(repo_id=Config.DATASET_REPO, repo_type="dataset")77    except RepositoryNotFoundError:78        _log("INFO", "creating_dataset", repo=Config.DATASET_REPO)79        api.create_repo(repo_id=Config.DATASET_REPO, repo_type="dataset", private=True)80 81 82def save() -> int:83    if not Config.HERMES_HOME.exists():84        _log("ERROR", "no_source_dir", path=str(Config.HERMES_HOME))85        return 186 87    api = _api()88    _ensure_repo(api)89 90    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")91    archive_name = f"state/{Config.BACKUP_PREFIX}{timestamp}.tar.gz"92 93    with tempfile.TemporaryDirectory() as tmp:94        tar_path = Path(tmp) / "hermes.tar.gz"95        with tarfile.open(tar_path, "w:gz") as tf:96            tf.add(str(Config.HERMES_HOME), arcname=".", filter=_tar_filter)97 98        size = tar_path.stat().st_size99        sha256 = hashlib.sha256(tar_path.read_bytes()).hexdigest()100        _log("INFO", "uploading_backup", file=archive_name, bytes=size, sha256=sha256[:12])101 102        api.upload_file(103            path_or_fileobj=str(tar_path),104            path_in_repo=archive_name,105            repo_id=Config.DATASET_REPO,106            repo_type="dataset",107            commit_message=f"HermesFace backup {timestamp}",108        )109 110    try:111        files = api.list_repo_files(repo_id=Config.DATASET_REPO, repo_type="dataset")112        backups = sorted(113            f for f in files114            if f.startswith(f"state/{Config.BACKUP_PREFIX}") and f.endswith(".tar.gz")115        )116        if len(backups) > Config.MAX_BACKUPS:117            for old in backups[: -Config.MAX_BACKUPS]:118                _log("INFO", "rotating_backup", delete=old)119                api.delete_file(120                    path_in_repo=old,121                    repo_id=Config.DATASET_REPO,122                    repo_type="dataset",123                    token=Config.HF_TOKEN,124                )125    except Exception as e:126        _log("WARNING", "rotation_failed", error=str(e))127 128    _log("INFO", "save_completed", file=archive_name)129    return 0130 131 132def load() -> int:133    api = _api()134    try:135        files = api.list_repo_files(repo_id=Config.DATASET_REPO, repo_type="dataset")136    except RepositoryNotFoundError:137        _log("ERROR", "repo_not_found", repo=Config.DATASET_REPO)138        return 1139 140    backups = sorted(141        (f for f in files142         if f.startswith(f"state/{Config.BACKUP_PREFIX}") and f.endswith(".tar.gz")),143        reverse=True,144    )145    if not backups:146        _log("WARNING", "no_backups_found")147        return 1148 149    Config.HERMES_HOME.mkdir(parents=True, exist_ok=True)150 151    for backup in backups:152        _log("INFO", "attempting_restore", file=backup)153        try:154            local = hf_hub_download(155                repo_id=Config.DATASET_REPO,156                repo_type="dataset",157                filename=backup,158                token=Config.HF_TOKEN,159            )160            # Local safety backup161            snapshot = Config.HERMES_HOME.parent / f"hermes_backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}"162            if Config.HERMES_HOME.exists() and any(Config.HERMES_HOME.iterdir()):163                shutil.copytree(Config.HERMES_HOME, snapshot, dirs_exist_ok=True)164            with tarfile.open(local, "r:*") as tf:165                tf.extractall(str(Config.HERMES_HOME))166            _log("INFO", "restore_completed", file=backup)167            return 0168        except Exception as e:169            _log("ERROR", "restore_failed", file=backup, error=str(e))170            continue171 172    _log("ERROR", "all_restore_attempts_failed")173    return 1174 175 176def status() -> int:177    api = _api()178    try:179        files = api.list_repo_files(repo_id=Config.DATASET_REPO, repo_type="dataset")180    except RepositoryNotFoundError:181        _log("WARNING", "repo_not_found", repo=Config.DATASET_REPO)182        return 0183 184    backups = sorted(185        (f for f in files186         if f.startswith(f"state/{Config.BACKUP_PREFIX}") and f.endswith(".tar.gz")),187        reverse=True,188    )189    local_files = 0190    if Config.HERMES_HOME.exists():191        local_files = sum(1 for _ in Config.HERMES_HOME.rglob("*") if _.is_file())192 193    print(json.dumps({194        "repo": Config.DATASET_REPO,195        "local_dir": str(Config.HERMES_HOME),196        "local_files": local_files,197        "remote_backups": backups,198        "max_backups": Config.MAX_BACKUPS,199    }, indent=2))200    return 0201 202 203COMMANDS = {"save": save, "load": load, "status": status}204 205 206def main() -> None:207    if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS:208        print(f"Usage: {sys.argv[0]} {{{'|'.join(COMMANDS)}}}", file=sys.stderr)209        sys.exit(2)210    sys.exit(COMMANDS[sys.argv[1]]())211 212 213if __name__ == "__main__":214    main()215