agenticverse/AgentNexa

1#!/usr/bin/env python32"""3Simple tar-gz backup of /opt/data → HF Dataset repo.4Keeps the last 5 backups to prevent data loss from corruption.5 6Env vars:7  HF_TOKEN              HF access token with write permission8  HERMES_DATASET_REPO   Target dataset repo, e.g. username/HermesFace-data9  HERMES_HOME           Source directory (default: /opt/data)10"""11import os12import shutil13import sys14import tarfile15import tempfile16from datetime import datetime17 18from huggingface_hub import HfApi19 20 21def main() -> None:22    repo_id = os.environ.get("HERMES_DATASET_REPO")23    token = os.environ.get("HF_TOKEN")24    state_dir = os.environ.get("HERMES_HOME", "/opt/data")25 26    if not repo_id or not token:27        print("[save_to_dataset] Missing HF_TOKEN or HERMES_DATASET_REPO.", file=sys.stderr)28        return29    if not os.path.isdir(state_dir):30        print(f"[save_to_dataset] No state directory to save: {state_dir}", file=sys.stderr)31        return32 33    api = HfApi(token=token)34 35    # Sync container logs into state dir for persistence36    try:37        sys_log_path = "/opt/data/logs"38        backup_log_path = os.path.join(state_dir, "logs/sys_logs")39        if os.path.exists(sys_log_path) and sys_log_path != backup_log_path:40            if os.path.exists(backup_log_path):41                shutil.rmtree(backup_log_path)42            shutil.copytree(sys_log_path, backup_log_path, ignore_dangling_symlinks=True)43    except Exception as e:44        print(f"[save_to_dataset] Warning: Failed to sync logs: {e}")45 46    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")47    filename = f"state/backup-{timestamp}.tar.gz"48 49    with tempfile.TemporaryDirectory() as tmpdir:50        tar_path = os.path.join(tmpdir, "hermes.tar.gz")51        try:52            with tarfile.open(tar_path, "w:gz") as tf:53                def exclude(info: tarfile.TarInfo):54                    bad = (".lock", ".tmp", ".pid", ".socket")55                    if info.name.endswith(bad):56                        return None57                    if "__pycache__" in info.name:58                        return None59                    return info60                tf.add(state_dir, arcname=".", filter=exclude)61        except Exception as e:62            print(f"[save_to_dataset] Failed to compress: {e}", file=sys.stderr)63            return64 65        print(f"[save_to_dataset] Uploading backup: {filename}")66        try:67            api.upload_file(68                path_or_fileobj=tar_path,69                path_in_repo=filename,70                repo_id=repo_id,71                repo_type="dataset",72            )73        except Exception as e:74            print(f"[save_to_dataset] Upload failed: {e}", file=sys.stderr)75            return76 77    try:78        files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")79        backups = sorted(80            f for f in files81            if f.startswith("state/backup-") and (f.endswith(".tar") or f.endswith(".tar.gz"))82        )83        if len(backups) > 5:84            to_delete = backups[:-5]85            print(f"[save_to_dataset] Rotating backups, deleting: {to_delete}")86            for old in to_delete:87                api.delete_file(path_in_repo=old, repo_id=repo_id, repo_type="dataset", token=token)88    except Exception as e:89        print(f"[save_to_dataset] Rotation failed (non-fatal): {e}", file=sys.stderr)90 91 92if __name__ == "__main__":93    main()94