TReqs/hf-aibom-scorer

1"""HF Parser — converts Hugging Face model metadata into a partial CycloneDX BOM.2 3Conservative by design: unknown values stay absent rather than guessed.4Mirrors external-bom-builder.ts from the design doc. In particular,5`formulation`, `dependencies`, and the per-task lineage fields are never6populated here — no pipeline was observed, only a description of a result.7"""8 9from __future__ import annotations10 11import copy12import uuid13from datetime import datetime, timezone14from typing import Any15 16from src.hf_client import HfModelMetadata17 18TOOL_NAME = "GLaaS External Scorer (demo)"19TOOL_VERSION = "0.1.0"20MANUFACTURER_NAME = "GLaaS"21 22 23def _extract_description(readme_text: str | None) -> str | None:24    if not readme_text:25        return None26    text = readme_text27    if text.startswith("---"):28        end = text.find("---", 3)29        if end != -1:30            text = text[end + 3:]31    for paragraph in text.split("\n\n"):32        stripped = paragraph.strip().lstrip("#").strip()33        if stripped and not stripped.startswith("!["):34            return stripped[:300]35    return None36 37 38def _primary_weight_file(siblings: list[dict[str, Any]]) -> dict[str, Any] | None:39    return siblings[0] if siblings else None40 41 42def _license_ids(card_data: dict[str, Any]) -> list[str]:43    license_value = card_data.get("license")44    if not license_value:45        return []46    if isinstance(license_value, list):47        return [str(v) for v in license_value]48    return [str(license_value)]49 50 51def _package_url(model_id: str, sha: str) -> str:52    # No official purl-spec type exists for Hugging Face; "huggingface" here53    # mirrors the same ad-hoc convention the OWASP AIBOM Generator uses54    # (pkg:huggingface/{org}/{name}@{commit}), not a real IANA/purl-spec type.55    ref = f"@{sha[:12]}" if sha else ""56    return f"pkg:huggingface/{model_id}{ref}"57 58 59def build_partial_bom(metadata: HfModelMetadata) -> dict[str, Any]:60    now_iso = datetime.now(timezone.utc).isoformat()61    model_id = metadata.model_id62 63    hashes = []64    for sibling in metadata.siblings:65        lfs = sibling.get("lfs") or {}66        sha256 = lfs.get("sha256")67        if sha256:68            hashes.append({"alg": "SHA-256", "content": sha256})69 70    properties: list[dict[str, str]] = [71        {"name": "glaas:git:commit", "value": metadata.sha},72        {"name": "glaas:git:repo", "value": f"https://huggingface.co/{model_id}"},73        {"name": "glaas:git:branch", "value": "main"},74    ]75    if metadata.pipeline_tag:76        properties.append({"name": "glaas:label:pipeline_tag", "value": metadata.pipeline_tag})77    if metadata.library_name:78        properties.append({"name": "glaas:label:library_name", "value": metadata.library_name})79    for tag in metadata.tags[:20]:80        properties.append({"name": "glaas:label:tag", "value": tag})81 82    external_references = [83        {"type": "vcs", "url": f"https://huggingface.co/{model_id}"},84        {"type": "documentation", "url": f"https://huggingface.co/{model_id}"},85    ]86    primary_weight = _primary_weight_file(metadata.siblings)87    if primary_weight:88        filename = primary_weight.get("rfilename")89        if filename:90            external_references.append({91                "type": "distribution",92                "url": f"https://huggingface.co/{model_id}/resolve/main/{filename}",93            })94 95    component: dict[str, Any] = {96        "type": "machine-learning-model",97        "name": model_id,98        "version": metadata.sha,99        "purl": _package_url(model_id, metadata.sha),100        "properties": properties,101        "externalReferences": external_references,102    }103    if hashes:104        component["hashes"] = hashes105    description = _extract_description(metadata.readme_text)106    if description:107        component["description"] = description108    license_ids = _license_ids(metadata.card_data)109    if license_ids:110        component["licenses"] = [{"license": {"id": lic}} for lic in license_ids]111 112    metadata_block: dict[str, Any] = {113        "timestamp": now_iso,114        "tools": {"components": [{"type": "application", "name": TOOL_NAME, "version": TOOL_VERSION}]},115        "manufacturer": {"name": MANUFACTURER_NAME},116        "component": {"type": "machine-learning-model", "name": model_id},117    }118    if metadata.namespace:119        metadata_block["supplier"] = {"name": metadata.namespace}120 121    bom: dict[str, Any] = {122        "bomFormat": "CycloneDX",123        "specVersion": "1.6",124        "serialNumber": f"urn:uuid:{uuid.uuid4()}",125        "version": 1,126        "metadata": metadata_block,127        "components": [component],128        # Deliberately absent: "dependencies", "formulation" — not observable129        # from Hugging Face metadata alone (see design doc's field table).130    }131    return bom132 133 134def build_illustrative_roar_bom(bom: dict[str, Any]) -> dict[str, Any]:135    """Layer a synthetic roar-instrumented pipeline onto an already-scored BOM.136 137    Illustrates what GLaaS + roar would have captured automatically during138    training — a typical two-task pipeline (prepare_data -> train_model),139    NOT a measurement of the entered model's actual training process (no140    pipeline was observed for it). Mirrors the "illustrative projection"141    approach in design-docs/20260709 HF-ai-bom-space.md: same scorer, same142    component, only a synthetic formulation/dependencies layered on top —143    every caller must show this alongside the real score, clearly labeled144    as illustrative, never as a claim about the entered model.145    """146    illustrative = copy.deepcopy(bom)147    illustrative["formulation"] = [{148        "bom-ref": "formulation-1",149        "workflows": [{150            "bom-ref": "workflow-1",151            "tasks": [152                {153                    "bom-ref": "task-prepare-data",154                    "name": "prepare_data",155                    "outputs": [{"source": {"ref": "artifact-training-data"}}],156                },157                {158                    "bom-ref": "task-train-model",159                    "name": "train_model",160                    "inputs": [{"source": {"ref": "artifact-training-data"}}],161                    "outputs": [{"source": {"ref": "component-model"}}],162                },163            ],164        }],165    }]166    illustrative["dependencies"] = [167        {"ref": "task-train-model", "dependsOn": ["task-prepare-data"]},168    ]169    return illustrative170