DGXAI/driftcall

1"""Cell 24 — Hugging Face Hub + Spaces deployment.2 3Implements ``docs/modules/deploy_env_space.md`` §8.2 and DESIGN.md §11.3, §11.44deliverables. Four push helpers, all using the **new** ``hf upload`` CLI per5deploy_env_space.md §8.2 (deprecated ``huggingface-cli`` is forbidden).6 7Public surface:8  * ``push_lora_to_hub(checkpoint_path, repo_id, token)`` — LoRA-only adapter9    push with ``safe_serialization=True``. Never the naive 4-bit → 16-bit merge10    path (DESIGN.md §10.5, CLAUDE.md §13).11  * ``push_env_space(repo_id, token)`` — Docker-based env Space (CPU basic,12    deploy_env_space.md §6.3).13  * ``push_demo_space(repo_id, token)`` — Demo Space targeting ZeroGPU with14    A10G fallback (deploy_demo_space.md §3.1, §3.7).15  * ``push_dataset(brief_path, repo_id, token)`` — ``driftcall-indic-briefs``16    dataset (DESIGN.md §11.4).17 18All four return a frozen :class:`DeploymentResult` so a caller can audit the19exact ``hf`` invocation. Heavy deps (``huggingface_hub``, ``subprocess`` for20``hf``) are loaded lazily; tests monkeypatch the loaders to assert the21command construction without making network calls.22"""23 24from __future__ import annotations25 26import logging27import subprocess28from dataclasses import dataclass29from pathlib import Path30from typing import TYPE_CHECKING, Any, Literal31 32if TYPE_CHECKING:33    from collections.abc import Callable, Mapping34 35logger = logging.getLogger(__name__)36 37 38# ---------------------------------------------------------------------------39# Constants — repo defaults (DESIGN.md §11.3, §11.4, deploy_*_space.md §3.7)40# ---------------------------------------------------------------------------41 42 43DEFAULT_LORA_REPO_ID: str = "DGXAI/gemma-3n-e2b-driftcall-lora"44DEFAULT_DATASET_REPO_ID: str = "driftcall/driftcall-indic-briefs"45DEFAULT_ENV_SPACE_ID: str = "driftcall/driftcall-env"46DEFAULT_DEMO_SPACE_ID: str = "driftcall/driftcall-demo"47 48RepoType = Literal["model", "dataset", "space"]49 50DEPRECATED_CLI_NAMES: tuple[str, ...] = ("huggingface-cli",)51 52 53# ---------------------------------------------------------------------------54# Errors55# ---------------------------------------------------------------------------56 57 58class DeploymentError(Exception):59    """Root for every typed deploy-cell error."""60 61 62class HFTokenMissingError(DeploymentError):63    """Raised when the ``token`` argument is None or empty."""64 65 66class CheckpointPathMissingError(DeploymentError):67    """Raised when the LoRA checkpoint path does not exist."""68 69 70class NaiveMergeForbiddenError(DeploymentError):71    """Raised when the caller requests a 4-bit → 16-bit merge path72    (CLAUDE.md §13, DESIGN.md §10.5)."""73 74 75class DeploymentCommandError(DeploymentError):76    """Raised when the ``hf upload`` invocation exits non-zero."""77 78 79class DeprecatedCliError(DeploymentError):80    """Raised when a caller would invoke ``huggingface-cli`` instead of ``hf``."""81 82 83# ---------------------------------------------------------------------------84# DeploymentResult85# ---------------------------------------------------------------------------86 87 88@dataclass(frozen=True)89class DeploymentResult:90    """Audit record for one deployment call."""91 92    repo_id: str93    repo_type: RepoType94    command: tuple[str, ...]95    return_code: int96    stdout: str97    stderr: str98    success: bool99 100 101# ---------------------------------------------------------------------------102# Lazy dep loaders — patched by tests103# ---------------------------------------------------------------------------104 105 106def _load_hf_api() -> Any:107    """Return the ``huggingface_hub.HfApi`` class. Patched in tests."""108 109    from huggingface_hub import HfApi110 111    return HfApi112 113 114def _load_subprocess_run() -> Callable[..., Any]:115    """Return ``subprocess.run``. Patched in tests."""116 117    return subprocess.run118 119 120# ---------------------------------------------------------------------------121# Argument validation helpers122# ---------------------------------------------------------------------------123 124 125def _validate_token(token: str | None) -> str:126    if token is None or token.strip() == "":127        raise HFTokenMissingError("token argument is required and must be non-empty")128    return token129 130 131def _validate_repo_id(repo_id: str) -> str:132    if not isinstance(repo_id, str) or "/" not in repo_id:133        raise DeploymentError(f"repo_id must be 'org/name'; got {repo_id!r}")134    org, name = repo_id.split("/", 1)135    if not org or not name:136        raise DeploymentError(f"repo_id must be 'org/name'; got {repo_id!r}")137    return repo_id138 139 140def _validate_path_exists(path: Path, *, label: str) -> Path:141    if not isinstance(path, Path):142        raise DeploymentError(f"{label} must be pathlib.Path; got {type(path).__name__}")143    if not path.exists():144        raise CheckpointPathMissingError(f"{label} not found: {path}")145    return path146 147 148def _ensure_not_deprecated(executable: str) -> str:149    if executable in DEPRECATED_CLI_NAMES:150        raise DeprecatedCliError(151            f"{executable!r} is deprecated; use 'hf upload' (deploy_env_space.md §8.2)",152        )153    return executable154 155 156# ---------------------------------------------------------------------------157# Command construction158# ---------------------------------------------------------------------------159 160 161def build_hf_upload_command(162    *,163    repo_id: str,164    local_path: Path,165    repo_type: RepoType,166    revision: str | None = None,167    extra_args: tuple[str, ...] = (),168) -> tuple[str, ...]:169    """Construct an argv tuple for ``hf upload``.170 171    Shape per the new ``hf`` CLI (deploy_env_space.md §8.2):172        ``hf upload <repo_id> <local_path> --repo-type=<type> [--revision=<r>]``173    """174 175    _validate_repo_id(repo_id)176    if repo_type not in ("model", "dataset", "space"):177        raise DeploymentError(f"repo_type must be model|dataset|space; got {repo_type!r}")178    executable = _ensure_not_deprecated("hf")179    cmd: list[str] = [180        executable,181        "upload",182        repo_id,183        str(local_path),184        f"--repo-type={repo_type}",185    ]186    if revision is not None:187        cmd.append(f"--revision={revision}")188    cmd.extend(extra_args)189    return tuple(cmd)190 191 192def _run_command(193    cmd: tuple[str, ...],194    *,195    token: str,196    env_extra: Mapping[str, str] | None = None,197) -> tuple[int, str, str]:198    """Invoke ``cmd`` via subprocess; return ``(rc, stdout, stderr)``.199 200    The token is passed via environment, never via argv (avoids shell201    history leak). ``env_extra`` lets callers add per-deploy env vars.202    """203 204    import os205 206    run = _load_subprocess_run()207    env = dict(os.environ)208    env["HF_TOKEN"] = token209    env["HUGGINGFACE_HUB_TOKEN"] = token210    if env_extra is not None:211        env.update(env_extra)212    try:213        completed = run(214            list(cmd),215            check=False,216            capture_output=True,217            text=True,218            env=env,219        )220    except FileNotFoundError as exc:221        raise DeploymentCommandError(f"hf CLI not found on PATH: {exc}") from exc222    rc = int(getattr(completed, "returncode", 1))223    stdout = str(getattr(completed, "stdout", "") or "")224    stderr = str(getattr(completed, "stderr", "") or "")225    return rc, stdout, stderr226 227 228# ---------------------------------------------------------------------------229# push_lora_to_hub (DESIGN.md §11.3)230# ---------------------------------------------------------------------------231 232 233def push_lora_to_hub(234    checkpoint_path: Path,235    repo_id: str = DEFAULT_LORA_REPO_ID,236    token: str | None = None,237    *,238    merge_4bit_to_16bit: bool = False,239    revision: str | None = None,240) -> DeploymentResult:241    """Push the LoRA adapter directory to the HF Hub.242 243    Pushes adapter-only artifacts (``adapter_config.json``,244    ``adapter_model.safetensors``, ``tokenizer.json``, ``README.md``).245    Never the merged-fp16 weights — see DESIGN.md §10.5 + CLAUDE.md §13:246    naive 4-bit → 16-bit merging is the catastrophic-quality path.247    """248 249    if merge_4bit_to_16bit:250        raise NaiveMergeForbiddenError(251            "merge_4bit_to_16bit=True is forbidden: 4-bit → 16-bit merge "252            "produces silently broken weights (DESIGN.md §10.5, CLAUDE.md §13). "253            "Push the LoRA adapter only.",254        )255    resolved_token = _validate_token(token)256    _validate_path_exists(checkpoint_path, label="checkpoint_path")257    cmd = build_hf_upload_command(258        repo_id=repo_id,259        local_path=checkpoint_path,260        repo_type="model",261        revision=revision,262    )263    rc, stdout, stderr = _run_command(cmd, token=resolved_token)264    success = rc == 0265    if not success:266        logger.warning("push_lora_to_hub failed (rc=%d): %s", rc, stderr)267    return DeploymentResult(268        repo_id=repo_id,269        repo_type="model",270        command=cmd,271        return_code=rc,272        stdout=stdout,273        stderr=stderr,274        success=success,275    )276 277 278# ---------------------------------------------------------------------------279# push_env_space (deploy_env_space.md §4.4, §6.3)280# ---------------------------------------------------------------------------281 282 283def push_env_space(284    repo_id: str = DEFAULT_ENV_SPACE_ID,285    token: str | None = None,286    *,287    space_dir: Path | None = None,288    revision: str | None = None,289) -> DeploymentResult:290    """Push the env Space (Docker SDK, CPU basic). deploy_env_space.md §4.4."""291 292    resolved_token = _validate_token(token)293    if space_dir is None:294        space_dir = Path(".")295    _validate_path_exists(space_dir, label="space_dir")296    cmd = build_hf_upload_command(297        repo_id=repo_id,298        local_path=space_dir,299        repo_type="space",300        revision=revision,301    )302    rc, stdout, stderr = _run_command(cmd, token=resolved_token)303    success = rc == 0304    return DeploymentResult(305        repo_id=repo_id,306        repo_type="space",307        command=cmd,308        return_code=rc,309        stdout=stdout,310        stderr=stderr,311        success=success,312    )313 314 315# ---------------------------------------------------------------------------316# push_demo_space (deploy_demo_space.md §3.1, §3.7)317# ---------------------------------------------------------------------------318 319 320def push_demo_space(321    repo_id: str = DEFAULT_DEMO_SPACE_ID,322    token: str | None = None,323    *,324    space_dir: Path | None = None,325    hardware: Literal["zero-gpu", "a10g-small"] = "zero-gpu",326    revision: str | None = None,327) -> DeploymentResult:328    """Push the demo Space. Default hardware ``zero-gpu`` per329    deploy_demo_space.md §3.1; pass ``a10g-small`` to redeploy on the330    fallback hardware (§3.1 step 2)."""331 332    resolved_token = _validate_token(token)333    if hardware not in ("zero-gpu", "a10g-small"):334        raise DeploymentError(335            f"hardware must be zero-gpu|a10g-small; got {hardware!r}",336        )337    if space_dir is None:338        space_dir = Path(".")339    _validate_path_exists(space_dir, label="space_dir")340    cmd = build_hf_upload_command(341        repo_id=repo_id,342        local_path=space_dir,343        repo_type="space",344        revision=revision,345    )346    env_extra = {"DRIFTCALL_HARDWARE": hardware}347    rc, stdout, stderr = _run_command(cmd, token=resolved_token, env_extra=env_extra)348    success = rc == 0349    return DeploymentResult(350        repo_id=repo_id,351        repo_type="space",352        command=cmd,353        return_code=rc,354        stdout=stdout,355        stderr=stderr,356        success=success,357    )358 359 360# ---------------------------------------------------------------------------361# push_dataset (DESIGN.md §11.4)362# ---------------------------------------------------------------------------363 364 365def push_dataset(366    brief_path: Path,367    repo_id: str = DEFAULT_DATASET_REPO_ID,368    token: str | None = None,369    *,370    revision: str | None = None,371) -> DeploymentResult:372    """Push the ``driftcall-indic-briefs`` dataset (DESIGN.md §11.4)."""373 374    resolved_token = _validate_token(token)375    _validate_path_exists(brief_path, label="brief_path")376    cmd = build_hf_upload_command(377        repo_id=repo_id,378        local_path=brief_path,379        repo_type="dataset",380        revision=revision,381    )382    rc, stdout, stderr = _run_command(cmd, token=resolved_token)383    success = rc == 0384    return DeploymentResult(385        repo_id=repo_id,386        repo_type="dataset",387        command=cmd,388        return_code=rc,389        stdout=stdout,390        stderr=stderr,391        success=success,392    )393 394 395__all__ = [396    "DEFAULT_DATASET_REPO_ID",397    "DEFAULT_DEMO_SPACE_ID",398    "DEFAULT_ENV_SPACE_ID",399    "DEFAULT_LORA_REPO_ID",400    "DEPRECATED_CLI_NAMES",401    "CheckpointPathMissingError",402    "DeploymentCommandError",403    "DeploymentError",404    "DeploymentResult",405    "DeprecatedCliError",406    "HFTokenMissingError",407    "NaiveMergeForbiddenError",408    "RepoType",409    "build_hf_upload_command",410    "push_dataset",411    "push_demo_space",412    "push_env_space",413    "push_lora_to_hub",414]415