meshllm/layer-split-output

1#!/bin/bash2set -euo pipefail3 4# This script runs inside an HF Job container.5# It clones mesh-llm, builds the splitter, splits the model, validates, and publishes.6#7# Environment variables (set by mesh-llm model-package job spec):8#   SOURCE_REPO, SOURCE_FILE, SOURCE_QUANT, TARGET_REPO, MODEL_ID, SOURCE_REVISION9#   MESH_LLM_REF — git ref to build from (default: main)10#   CATALOG_CREATE_PR — "true" to open a PR for catalog updates (non-org members)11#   HF_TOKEN — injected as a secret by HF Jobs12#13# Volumes:14#   /bucket  — writable storage bucket for script and fallback source cache15 16MESH_LLM_REF="${MESH_LLM_REF:-main}"17SOURCE_REVISION="${SOURCE_REVISION:-main}"18SOURCE_QUANT="${SOURCE_QUANT:-}"19if [ -z "$SOURCE_QUANT" ] && [[ "${MODEL_ID:-}" == *:* ]]; then20    SOURCE_QUANT="${MODEL_ID##*:}"21fi22if [ -z "$SOURCE_QUANT" ]; then23    echo "ERROR: SOURCE_QUANT is required to resolve the source GGUF without a model volume" >&224    exit 125fi26 27echo "╔══════════════════════════════════════════════════════════╗"28echo "║  Layer Package Split Job                                 ║"29echo "╠══════════════════════════════════════════════════════════╣"30echo "║  Source: ${SOURCE_REPO}/${SOURCE_FILE}"31echo "║  Quant:  ${SOURCE_QUANT}"32echo "║  Target: ${TARGET_REPO}"33echo "║  Model:  ${MODEL_ID}"34echo "║  Build:  mesh-llm @ ${MESH_LLM_REF}"35echo "╚══════════════════════════════════════════════════════════╝"36echo ""37 38# Keep executable toolchains/build products on local ephemeral storage:39# HF bucket mounts can be unsuitable for dynamic loader/toolchain execution.40# Package artifacts are also written locally, uploaded one at a time, and41# removed immediately so the job never accumulates a full 400GB+ package.42JOB_WORK_ROOT="${JOB_WORK_ROOT:-/bucket/job-work}"43SAFE_TARGET_REPO="$(printf '%s' "$TARGET_REPO" | tr -c '[:alnum:]._-' '_')"44LOCAL_WORK_DIR="${LOCAL_WORK_DIR:-/tmp/meshllm-layer-job-${SAFE_TARGET_REPO}-$$}"45if [ -z "${JOB_WORK_DIR:-}" ]; then46    JOB_WORK_DIR="${JOB_WORK_ROOT}/${SAFE_TARGET_REPO}-$(date +%Y%m%d%H%M%S)-$$"47    CLEANUP_JOB_WORK_DIR="${CLEANUP_JOB_WORK_DIR:-true}"48else49    CLEANUP_JOB_WORK_DIR="${CLEANUP_JOB_WORK_DIR:-false}"50fi51PACKAGE_DIR="${PACKAGE_DIR:-${LOCAL_WORK_DIR}/package}"52HF_HOME="${HF_HOME:-${JOB_WORK_DIR}/hf-home}"53HF_HUB_CACHE="${HF_HUB_CACHE:-${HF_HOME}/hub}"54HF_XET_CACHE="${HF_XET_CACHE:-${HF_HOME}/xet}"55JOB_TMP_DIR="${JOB_TMP_DIR:-${LOCAL_WORK_DIR}/tmp}"56BUILD_DIR="${BUILD_DIR:-${LOCAL_WORK_DIR}/build}"57TOOL_DIR="${TOOL_DIR:-${LOCAL_WORK_DIR}/tools}"58VENV_DIR="${VENV_DIR:-${LOCAL_WORK_DIR}/venv}"59ARTIFACT_UPLOAD_SCRIPT="${ARTIFACT_UPLOAD_SCRIPT:-${LOCAL_WORK_DIR}/upload-package-artifact.py}"60ARTIFACT_UPLOAD_HOOK="${ARTIFACT_UPLOAD_HOOK:-${LOCAL_WORK_DIR}/upload-package-artifact.sh}"61CARGO_HOME="${CARGO_HOME:-${LOCAL_WORK_DIR}/cargo-home}"62RUSTUP_HOME="${RUSTUP_HOME:-${LOCAL_WORK_DIR}/rustup-home}"63CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-${LOCAL_WORK_DIR}/cargo-target}"64XDG_CACHE_HOME="${XDG_CACHE_HOME:-${LOCAL_WORK_DIR}/xdg-cache}"65PIP_CACHE_DIR="${PIP_CACHE_DIR:-${LOCAL_WORK_DIR}/pip-cache}"66BUILD_TMP_DIR="${BUILD_TMP_DIR:-${LOCAL_WORK_DIR}/tmp}"67TMPDIR="$BUILD_TMP_DIR"68TEMP="$BUILD_TMP_DIR"69TMP="$BUILD_TMP_DIR"70export JOB_WORK_DIR PACKAGE_DIR HF_HOME HF_HUB_CACHE HF_XET_CACHE VENV_DIR ARTIFACT_UPLOAD_SCRIPT71export TMPDIR TEMP TMP CARGO_HOME RUSTUP_HOME CARGO_TARGET_DIR XDG_CACHE_HOME PIP_CACHE_DIR72 73cleanup_job_work_dir() {74    if [ -n "${LOCAL_WORK_DIR:-}" ]; then75        echo "Cleaning local work dir: ${LOCAL_WORK_DIR}"76        rm -rf "$LOCAL_WORK_DIR" || true77    fi78    if [ "${CLEANUP_JOB_WORK_DIR}" = "true" ] && [ -n "${JOB_WORK_DIR:-}" ]; then79        echo "Cleaning job work dir: ${JOB_WORK_DIR}"80        rm -rf "$JOB_WORK_DIR" || true81    fi82}83trap cleanup_job_work_dir EXIT84 85log_storage_snapshot() {86    local label="$1"87    echo "  Storage snapshot (${label}):"88    df -h / /bucket "$PACKAGE_DIR" "$TMPDIR" 2>/dev/null || true89    echo "  Mounts (${label}):"90    mount | grep -E ' on / | on /bucket ' || true91}92 93on_error() {94    local status=$?95    local line=${BASH_LINENO[0]:-unknown}96    local command=${BASH_COMMAND:-unknown}97    echo "ERROR: split job command failed at line ${line} with status ${status}: ${command}" >&298    log_storage_snapshot "error" >&2 || true99    exit "$status"100}101trap on_error ERR102 103start_heartbeat() {104    local label="$1"105    (106        while true; do107            sleep "${JOB_HEARTBEAT_SECONDS:-60}"108            echo "  Heartbeat (${label}) $(date -u +%Y-%m-%dT%H:%M:%SZ)"109            df -h / /bucket "$PACKAGE_DIR" "$TMPDIR" 2>/dev/null || true110            if [ -d "$PACKAGE_DIR" ]; then111                du -sh "$PACKAGE_DIR" 2>/dev/null || true112            fi113            if [ -d "$HF_HUB_CACHE" ]; then114                du -sh "$HF_HUB_CACHE" 2>/dev/null || true115            fi116        done117    ) &118    HEARTBEAT_PID=$!119}120 121stop_heartbeat() {122    if [ -n "${HEARTBEAT_PID:-}" ]; then123        kill "$HEARTBEAT_PID" 2>/dev/null || true124        wait "$HEARTBEAT_PID" 2>/dev/null || true125        HEARTBEAT_PID=""126    fi127}128 129mkdir -p "$PACKAGE_DIR" "$HF_HUB_CACHE" "$HF_XET_CACHE" "$JOB_TMP_DIR" "$TOOL_DIR" \130    "$CARGO_HOME" "$RUSTUP_HOME" "$CARGO_TARGET_DIR" "$XDG_CACHE_HOME" "$PIP_CACHE_DIR" \131    "$BUILD_TMP_DIR"132 133format_bytes() {134    python3 - "$1" <<'PYTHON'135import sys136value = float(int(sys.argv[1]))137for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]:138    if value < 1024 or unit == "PiB":139        if unit == "B":140            print(f"{int(value)} {unit}")141        else:142            print(f"{value:.1f} {unit}")143        break144    value /= 1024145PYTHON146}147 148estimate_bucket_workspace_bytes() {149    python3 - "$1" <<'PYTHON'150import sys151source = int(sys.argv[1])152# Source and package artifacts are not meant to accumulate in the bucket. This153# estimate is retained only as a fallback-source-cache warning when /source is154# unavailable.155headroom = 32 * 1024 ** 3156print(source + headroom)157PYTHON158}159 160# ─── Build tools ──────────────────────────────────────────────────────────161echo "=== [1/9] Installing build dependencies ==="162apt-get update -qq && apt-get install -y -qq \163    cmake git curl build-essential pkg-config libssl-dev \164    python3-pip python3-venv > /dev/null 2>&1165apt-get clean166rm -rf /var/lib/apt/lists/*167 168echo "=== [2/9] Installing Rust ==="169curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y > /dev/null 2>&1170source "${CARGO_HOME}/env"171 172echo "=== [3/9] Cloning mesh-llm and building skippy-model-package ==="173git clone --filter=blob:none https://github.com/Mesh-LLM/mesh-llm.git "$BUILD_DIR"174cd "$BUILD_DIR"175if git ls-remote --exit-code --heads origin "$MESH_LLM_REF" >/dev/null 2>&1 || \176   git ls-remote --exit-code --tags origin "$MESH_LLM_REF" >/dev/null 2>&1; then177    git fetch --depth 1 origin "$MESH_LLM_REF"178    git checkout --detach FETCH_HEAD179elif git cat-file -e "$MESH_LLM_REF^{commit}" 2>/dev/null; then180    git checkout --detach "$MESH_LLM_REF"181else182    git fetch --depth 1 origin "$MESH_LLM_REF"183    git checkout --detach FETCH_HEAD184fi185 186# Full clone needed for git-am patches in prepare-llama187sed -i 's/--filter=blob:none //' scripts/prepare-llama.sh188echo "  Running prepare-llama.sh..."189scripts/prepare-llama.sh pinned 2>&1 | tail -5190echo "  Running build-llama.sh..."191scripts/build-llama.sh 2>&1 | tail -5192 193# Locate the llama.cpp build directory (build-llama.sh puts it here)194LLAMA_BUILD_DIR=".deps/llama-build/build-stage-abi-cpu"195echo "  Verifying llama.cpp build at $LLAMA_BUILD_DIR..."196find "$LLAMA_BUILD_DIR" -name "*.a" 2>/dev/null | head -10 || echo "  WARNING: no .a files found"197 198# Build the splitter binary199echo "  Building skippy-model-package..."200SKIPPY_LLAMA_BUILD_DIR="$LLAMA_BUILD_DIR" \201    cargo build --release -p skippy-model-package 2>&1 | tail -20202SLICER="${CARGO_TARGET_DIR}/release/skippy-model-package"203if [ ! -f "$SLICER" ]; then204    echo "ERROR: Build failed — binary not found at $SLICER"205    echo "Retrying with full output..."206    SKIPPY_LLAMA_BUILD_DIR=.deps/llama.cpp/build-stage-abi-static \207        cargo build --release -p skippy-model-package 2>&1208    exit 1209fi210cp "$SLICER" "${TOOL_DIR}/skippy-model-package"211SLICER="${TOOL_DIR}/skippy-model-package"212chmod +x "$SLICER"213cd /214rm -rf "$BUILD_DIR" "$CARGO_TARGET_DIR" "$CARGO_HOME" "$RUSTUP_HOME"215TMPDIR="$JOB_TMP_DIR"216TEMP="$JOB_TMP_DIR"217TMP="$JOB_TMP_DIR"218export TMPDIR TEMP TMP219echo "  ✓ Built: $SLICER"220echo "  Root filesystem after build cleanup:"221df -h / || true222 223echo "  Preparing Hugging Face uploader..."224python3 -m venv "$VENV_DIR" > /dev/null225"$VENV_DIR/bin/pip" install -q huggingface_hub226"$VENV_DIR/bin/python3" << 'PYTHON'227from huggingface_hub import HfApi228import os229 230api = HfApi(token=os.environ["HF_TOKEN"])231api.create_repo(os.environ["TARGET_REPO"], exist_ok=True)232PYTHON233cat > "$ARTIFACT_UPLOAD_SCRIPT" <<'PYTHON'234from huggingface_hub import HfApi235from pathlib import Path236import os237 238path = Path(os.environ["SKIPPY_PACKAGE_ARTIFACT_PATH"])239relative = os.environ["SKIPPY_PACKAGE_ARTIFACT_RELATIVE_PATH"]240target_repo = os.environ["TARGET_REPO"]241 242api = HfApi(token=os.environ["HF_TOKEN"])243api.upload_file(244    repo_id=target_repo,245    path_or_fileobj=str(path),246    path_in_repo=relative,247    repo_type="model",248    commit_message=f"Add package artifact {relative}",249)250size = path.stat().st_size251path.unlink()252print(f"  Uploaded and removed {relative} ({size} bytes)")253PYTHON254cat > "$ARTIFACT_UPLOAD_HOOK" <<'BASH'255#!/bin/bash256set -euo pipefail257"${VENV_DIR}/bin/python3" "${ARTIFACT_UPLOAD_SCRIPT}"258BASH259chmod +x "$ARTIFACT_UPLOAD_HOOK"260 261# ─── Split ────────────────────────────────────────────────────────────────262echo ""263echo "=== [4/9] Splitting model ==="264if [ "$SOURCE_REVISION" = "main" ]; then265    SOURCE_REF="${SOURCE_REPO}:${SOURCE_QUANT}"266else267    SOURCE_REF="${SOURCE_REPO}@${SOURCE_REVISION}:${SOURCE_QUANT}"268fi269echo "  Source ref: $SOURCE_REF"270if [ -n "${SOURCE_TOTAL_BYTES:-}" ]; then271    echo "  Source bytes: $SOURCE_TOTAL_BYTES"272    ESTIMATED_BUCKET_BYTES="$(estimate_bucket_workspace_bytes "$SOURCE_TOTAL_BYTES")"273    echo "  Estimated fallback /bucket cache needed: $(format_bytes "$ESTIMATED_BUCKET_BYTES")"274fi275MOUNTED_SOURCE_PATH="/source/${SOURCE_FILE}"276if [ -f "$MOUNTED_SOURCE_PATH" ]; then277    WRITE_PACKAGE_INPUT="$MOUNTED_SOURCE_PATH"278    WRITE_PACKAGE_IDENTITY_ARGS=(279        --model-id "$MODEL_ID"280        --source-repo "$SOURCE_REPO"281        --source-revision "$SOURCE_REVISION"282        --source-file "$SOURCE_FILE"283    )284    echo "  Source mount: $MOUNTED_SOURCE_PATH"285else286    WRITE_PACKAGE_INPUT="$SOURCE_REF"287    WRITE_PACKAGE_IDENTITY_ARGS=()288    echo "  Source mount: not available; falling back to Hugging Face cache download"289fi290echo "  Hugging Face cache: $HF_HUB_CACHE"291echo "  Package workspace: $PACKAGE_DIR"292echo "  Temporary workspace: $TMPDIR"293log_storage_snapshot "before write-package"294ROOT_FS="$(df -P / | awk 'NR==2 {print $1}')"295PACKAGE_FS="$(df -P "$PACKAGE_DIR" | awk 'NR==2 {print $1}')"296if [ -n "$ROOT_FS" ] && [ "$ROOT_FS" = "$PACKAGE_FS" ]; then297    echo "WARNING: package workspace is on the container root filesystem; very large splits may hit the HF Jobs 50G ephemeral storage limit." >&2298fi299if [ -n "${ESTIMATED_BUCKET_BYTES:-}" ]; then300    PACKAGE_AVAILABLE_BYTES="$(df -Pk "$PACKAGE_DIR" | awk 'NR==2 {printf "%.0f", $4 * 1024}')"301    if [ -n "$PACKAGE_AVAILABLE_BYTES" ] && [ "$PACKAGE_AVAILABLE_BYTES" -gt 0 ] && \302        [ "$PACKAGE_AVAILABLE_BYTES" -lt "$ESTIMATED_BUCKET_BYTES" ]; then303        echo "WARNING: package workspace has $(format_bytes "$PACKAGE_AVAILABLE_BYTES") available, below estimated need $(format_bytes "$ESTIMATED_BUCKET_BYTES")." >&2304    fi305fi306echo "  Starting write-package at $(date -u +%Y-%m-%dT%H:%M:%SZ)"307start_heartbeat "write-package"308set +e309time "$SLICER" write-package "$WRITE_PACKAGE_INPUT" \310    --out-dir "$PACKAGE_DIR" \311    --after-artifact-command "$ARTIFACT_UPLOAD_HOOK" \312    "${WRITE_PACKAGE_IDENTITY_ARGS[@]}"313WRITE_PACKAGE_STATUS=$?314set -e315stop_heartbeat316if [ "$WRITE_PACKAGE_STATUS" -ne 0 ]; then317    echo "ERROR: write-package failed with status $WRITE_PACKAGE_STATUS" >&2318    log_storage_snapshot "write-package failed" >&2 || true319    exit "$WRITE_PACKAGE_STATUS"320fi321echo "  Finished write-package at $(date -u +%Y-%m-%dT%H:%M:%SZ)"322log_storage_snapshot "after write-package"323 324SOURCE_PATH="$(python3 -c "import json, os; m=json.load(open(os.path.join(os.environ['PACKAGE_DIR'], 'model-package.json'))); print(m['source_model']['path'])")"325echo "  Cached source: $SOURCE_PATH ($(du -h "$SOURCE_PATH" | cut -f1))"326 327LAYER_COUNT="$(python3 -c "import json, os; m=json.load(open(os.path.join(os.environ['PACKAGE_DIR'], 'model-package.json'))); print(m['layer_count'])")"328TOTAL_SIZE="$(python3 -c "import json, os; m=json.load(open(os.path.join(os.environ['PACKAGE_DIR'], 'model-package.json'))); print(sum(int(a.get('artifact_bytes') or 0) for a in list(m['shared'].values()) + m.get('layers', []) + m.get('projectors', [])))")"329TOTAL_SIZE_LABEL="$(format_bytes "$TOTAL_SIZE")"330echo "  ✓ Split into $LAYER_COUNT layers; artifacts uploaded incrementally (${TOTAL_SIZE_LABEL} total)"331 332# ─── Verify manifest ──────────────────────────────────────────────────────333echo ""334echo "=== [5/9] Verifying package manifest ==="335"$VENV_DIR/bin/python3" << 'PYTHON'336import json337import os338from pathlib import Path339 340manifest_path = Path(os.environ["PACKAGE_DIR"]) / "model-package.json"341manifest = json.loads(manifest_path.read_text())342required = [343    manifest["shared"]["metadata"],344    manifest["shared"]["embeddings"],345    manifest["shared"]["output"],346    *manifest.get("layers", []),347    *manifest.get("projectors", []),348]349missing = [artifact for artifact in required if not artifact.get("path") or not artifact.get("sha256")]350if missing:351    raise SystemExit(f"manifest contains {len(missing)} artifacts without path/checksum")352print(f"  ✓ Manifest records {len(required)} uploaded artifacts")353PYTHON354 355# ─── Publish ──────────────────────────────────────────────────────────────356echo ""357echo "=== [6/9] Publishing to HuggingFace ==="358"$VENV_DIR/bin/python3" << PYTHON359from huggingface_hub import HfApi360import os, json361from pathlib import Path362 363api = HfApi(token=os.environ['HF_TOKEN'])364target_repo = os.environ['TARGET_REPO']365source_repo = os.environ['SOURCE_REPO']366model_id = os.environ.get('MODEL_ID', '')367manifest_path = Path(os.environ['PACKAGE_DIR']) / 'model-package.json'368 369api.upload_file(370    repo_id=target_repo,371    path_or_fileobj=str(manifest_path),372    path_in_repo='model-package.json',373    repo_type='model',374    commit_message=f'Add layer package manifest from {source_repo} ({model_id})',375)376 377# Print summary378manifest = json.load(open(manifest_path))379print(f'  ✓ Published: https://huggingface.co/{target_repo}')380print(f'    Model:  {manifest["model_id"]}')381print(f'    Layers: {manifest["layer_count"]}')382print(f'    Schema: {manifest["schema_version"]}')383PYTHON384 385# ─── Update catalog ───────────────────────────────────────────────────────386echo ""387echo "=== [7/9] Updating meshllm/catalog ==="388"$VENV_DIR/bin/python3" << 'PYTHON'389from huggingface_hub import HfApi390import os, json, tempfile391 392api = HfApi(token=os.environ['HF_TOKEN'])393source_repo = os.environ['SOURCE_REPO']394target_repo = os.environ['TARGET_REPO']395source_file = os.environ['SOURCE_FILE']396source_revision = os.environ.get('SOURCE_REVISION', 'main')397model_id = os.environ.get('MODEL_ID', '')398package_dir = os.environ['PACKAGE_DIR']399 400# Read manifest for metadata401manifest = json.load(open(os.path.join(package_dir, 'model-package.json')))402layer_count = manifest['layer_count']403 404# Determine catalog entry path: entries/<owner>/<repo-name>.json405owner, repo_name = source_repo.split('/', 1)406entry_path = f"entries/{owner}/{repo_name}.json"407 408# Try to fetch existing entry409catalog_repo = "meshllm/catalog"410try:411    existing_path = api.hf_hub_download(412        repo_id=catalog_repo,413        filename=entry_path,414        repo_type="dataset",415    )416    entry = json.load(open(existing_path))417except Exception:418    # Create new entry419    entry = {"schema_version": 1, "source_repo": source_repo, "variants": {}}420 421# Build variant name from source file stem (not MODEL_ID).422# For "UD-Q4_K_XL/Qwen3-32B-UD-Q4_K_XL-00001-of-00002.gguf" → "Qwen3-32B-UD-Q4_K_XL"423import re424file_stem = source_file.split('/')[-1].replace('.gguf', '')425# Strip shard suffix like "-00001-of-00002"426variant_name = re.sub(r'-\d{5}-of-\d{5}$', '', file_stem)427 428package_entry = {429    "type": "layer-package",430    "repo": target_repo,431    "layer_count": layer_count,432}433 434# Handle both dict-style and list-style variants435variants = entry.get("variants", {})436if isinstance(variants, dict):437    # Dict-keyed by variant name (existing catalog format)438    if variant_name in variants:439        packages = variants[variant_name].get("packages", [])440        packages = [p for p in packages if p.get("repo") != target_repo]441        packages.append(package_entry)442        variants[variant_name]["packages"] = packages443    else:444        variants[variant_name] = {445            "source": {446                "repo": source_repo,447                "file": source_file,448                "revision": source_revision,449            },450            "curated": {451                "name": variant_name,452                "size": f"{layer_count} layers",453                "description": f"Layer package for {model_id}",454            },455            "packages": [package_entry],456        }457    entry["variants"] = variants458else:459    # List-style (fallback)460    existing_variant = None461    for v in variants:462        if v.get("curated", {}).get("name") == variant_name:463            existing_variant = v464            break465    if existing_variant:466        packages = existing_variant.get("packages", [])467        packages = [p for p in packages if p.get("repo") != target_repo]468        packages.append(package_entry)469        existing_variant["packages"] = packages470    else:471        variants.append({472            "source": {473                "repo": source_repo,474                "file": source_file,475                "revision": source_revision,476            },477            "curated": {478                "name": variant_name,479                "size": f"{layer_count} layers",480                "description": f"Layer package for {model_id}",481            },482            "packages": [package_entry],483        })484 485# Write and upload486with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:487    json.dump(entry, f, indent=2)488    tmp_path = f.name489 490create_pr = os.environ.get('CATALOG_CREATE_PR', 'false').lower() == 'true'491 492api.upload_file(493    repo_id=catalog_repo,494    path_or_fileobj=tmp_path,495    path_in_repo=entry_path,496    repo_type="dataset",497    commit_message=f"Add layer package for {model_id} ({target_repo})",498    create_pr=create_pr,499)500print(f"  ✓ Catalog updated: {catalog_repo}/{entry_path}")501print(f"    Variant: {variant_name}")502print(f"    Package: {target_repo} ({layer_count} layers)")503PYTHON504 505# ─── Model Card ────────────────────────────────────────────────────────────506echo ""507echo "=== [8/9] Uploading model card ==="508"$VENV_DIR/bin/python3" << 'PYTHON'509from huggingface_hub import HfApi510from pathlib import Path511import hashlib512import json513import os514 515package_dir = Path(os.environ["PACKAGE_DIR"])516manifest_path = package_dir / "model-package.json"517manifest = json.loads(manifest_path.read_text())518 519source_repo = os.environ["SOURCE_REPO"]520source_file = os.environ["SOURCE_FILE"]521source_revision = os.environ.get("SOURCE_REVISION", "main")522target_repo = os.environ["TARGET_REPO"]523model_id = os.environ.get("MODEL_ID", manifest.get("model_id", target_repo))524mesh_llm_ref = os.environ.get("MESH_LLM_REF", "main")525 526def sha256(path: Path) -> str:527    digest = hashlib.sha256()528    with path.open("rb") as file:529        for chunk in iter(lambda: file.read(1024 * 1024), b""):530            digest.update(chunk)531    return digest.hexdigest()532 533def fmt_bytes(size: int) -> str:534    value = float(size)535    for unit in ["B", "KB", "MB", "GB", "TB"]:536        if value < 1024 or unit == "TB":537            if unit == "B":538                return f"{int(value)} {unit}"539            return f"{value:.1f} {unit}"540        value /= 1024541 542def artifact_bytes(artifact: dict) -> int:543    return int(artifact.get("artifact_bytes") or 0)544 545def md_cell(value) -> str:546    text = "" if value is None else str(value)547    return text.replace("|", "\\|").replace("\n", "<br>")548 549def link(label: str, url: str) -> str:550    return f"[{md_cell(label)}]({url})"551 552def code(value) -> str:553    return f"`{md_cell(value)}`"554 555def yaml_quote(value: str) -> str:556    return json.dumps(value)557 558def infer_model_family(name: str) -> str:559    lowered = name.lower()560    for family in ["Qwen3", "Qwen2.5", "DeepSeek", "Kimi", "Gemma", "GLM", "Llama"]:561        if family.lower() in lowered:562            return family563    return name.split("-")[0] if name else "Unknown"564 565def infer_parameter_scale(name: str) -> str:566    import re567    match = re.search(r"(?i)(\d+(?:\.\d+)?[BM](?:-A\d+(?:\.\d+)?B)?)", name)568    return match.group(1) if match else "not recorded"569 570def infer_quantization(name: str, source_path: str) -> str:571    import re572    combined = f"{name}/{source_path}"573    patterns = [574        r"UD-Q\d+_[A-Z]+(?:_[A-Z]+)?",575        r"Q\d+_[A-Z]+(?:_[A-Z]+)?",576        r"IQ\d+_[A-Z]+(?:_[A-Z]+)?",577        r"BF16",578        r"F16",579    ]580    for pattern in patterns:581        match = re.search(pattern, combined, re.IGNORECASE)582        if match:583            return match.group(0)584    return "not recorded"585 586shared = manifest.get("shared", {})587layers = manifest.get("layers", [])588projectors = manifest.get("projectors", [])589manifest_hash = sha256(manifest_path)590total_bytes = sum(artifact_bytes(artifact) for artifact in shared.values())591total_bytes += sum(artifact_bytes(layer) for layer in layers)592total_bytes += sum(artifact_bytes(projector) for projector in projectors)593 594source_model = manifest.get("source_model", {})595display_name = source_model.get("distribution_id") or model_id596model_family = infer_model_family(display_name)597parameter_scale = infer_parameter_scale(display_name)598quantization = infer_quantization(display_name, source_file)599source_path = source_model.get("path") or f"/hf-cache/{source_file}"600activation_width = manifest.get("activation_width") or "not recorded"601skippy_abi = manifest.get("skippy_abi_version") or "not recorded"602source_sha = source_model.get("sha256") or "not recorded"603canonical_ref = source_model.get("canonical_ref") or f"{source_repo}@{source_revision}/{source_file}"604 605file_rows = [606    ("Manifest", "model-package.json", "Package schema, source identity, checksums", manifest_hash),607]608for label, key in [609    ("Metadata", "metadata"),610    ("Embeddings", "embeddings"),611    ("Output head", "output"),612]:613    artifact = shared.get(key)614    if artifact:615        file_rows.append((616            label,617            artifact.get("path", f"shared/{key}.gguf"),618            f"{artifact.get('tensor_count', 'unknown')} tensors, {fmt_bytes(artifact_bytes(artifact))}",619            artifact.get("sha256", "not recorded"),620        ))621if layers:622    layer_bytes = sum(artifact_bytes(layer) for layer in layers)623    layer_tensors = sum(int(layer.get("tensor_count") or 0) for layer in layers)624    file_rows.append((625        "Transformer layers",626        "layers/layer-*.gguf",627        f"{len(layers)} layer artifacts, {layer_tensors} tensors, {fmt_bytes(layer_bytes)}",628        "see model-package.json",629    ))630for projector in projectors:631    file_rows.append((632        "Projector",633        projector.get("path", "projectors/projector.gguf"),634        f"{projector.get('kind', 'multimodal')} projector, {fmt_bytes(artifact_bytes(projector))}",635        projector.get("sha256", "not recorded"),636    ))637 638rows = [639    ("Source model", link(source_repo, f"https://huggingface.co/{source_repo}")),640    ("Model id", code(model_id)),641    ("Family", model_family),642    ("Parameter scale", parameter_scale),643    ("Quantization", code(quantization)),644    ("Layer count", manifest.get("layer_count", len(layers))),645    ("Activation width", activation_width),646    ("Package size", fmt_bytes(total_bytes)),647    ("Source file", code(source_file)),648    ("Package repo", link(target_repo, f"https://huggingface.co/{target_repo}")),649]650 651readme = f"""---652library_name: mesh-llm653base_model:654- {yaml_quote(source_repo)}655pipeline_tag: text-generation656tags:657- gguf658- mesh-llm659- layer-package660- skippy661- distributed-inference662- local-inference663- openai-compatible664---665 666<div align="center">667  <a href="https://www.meshllm.cloud">668    <img src="https://github.com/Mesh-LLM/mesh-llm/raw/main/docs/mesh-llm-logo.svg" alt="Mesh LLM" width="220">669  </a>670 671  <h1>{display_name}</h1>672 673  <p>674    <strong>Distributed GGUF inference package for Mesh LLM</strong>675  </p>676 677  <p>678    <a href="https://www.meshllm.cloud"><img alt="Website" src="https://img.shields.io/badge/Website-meshllm.cloud-111111?style=for-the-badge"></a>679    <a href="https://github.com/Mesh-LLM/mesh-llm"><img alt="GitHub" src="https://img.shields.io/badge/GitHub-Mesh--LLM-24292f?style=for-the-badge&logo=github"></a>680    <a href="https://discord.gg/rs6fmc63eN"><img alt="Discord" src="https://img.shields.io/badge/Discord-Join-5865F2?style=for-the-badge&logo=discord&logoColor=white"></a>681  </p>682</div>683 684GGUF layer package for running **{display_name}** across a local Mesh LLM cluster.685 686This package is derived from [{source_repo}](https://huggingface.co/{source_repo}) and keeps the original GGUF distribution split into per-layer artifacts for distributed inference.687 688## Highlights689 690| Run locally | Pool multiple machines | OpenAI-compatible | Package variant |691|---|---|---|---|692| Private inference on your hardware | Split layers across peers | Serve `/v1/chat/completions` locally | `{quantization}` layer package |693 694## Model Overview695 696| Property | Value |697|---|---|698"""699 700for key, value in rows:701    readme += f"| **{md_cell(key)}** | {md_cell(value)} |\n"702 703readme += f"""704## Recommended Use705 706- Local and private inference with Mesh LLM.707- Multi-machine serving when the full GGUF is too large for one host.708- OpenAI-compatible chat/completions workflows through Mesh LLM's local API.709 710For upstream architecture details, chat template guidance, sampling recommendations, license terms, and benchmark notes, see the source model card: [{source_repo}](https://huggingface.co/{source_repo}).711 712## Quickstart713 714```bash715# Run this on each machine that should contribute memory/compute.716mesh-llm serve --model "{target_repo}" --split717```718 719```bash720# Check the mesh and discover the OpenAI-compatible model name.721curl -s http://localhost:3131/api/status722curl -s http://localhost:3131/v1/models723```724 725```bash726# Send an OpenAI-compatible chat request.727curl -s http://localhost:3131/v1/chat/completions \\728  -H "Content-Type: application/json" \\729  -d '{{730    "model": "{model_id}",731    "messages": [{{"role": "user", "content": "Write a tiny hello-world function in Rust."}}],732    "max_tokens": 128733  }}'734```735 736## Package Variant737 738| Property | Value |739|---|---|740"""741 742for key, value in [743    ("Format", code(manifest.get("format", "layer-package"))),744    ("Canonical source ref", code(canonical_ref)),745    ("Source revision", code(source_revision)),746    ("Source SHA-256", code(source_sha)),747    ("Skippy ABI", code(skippy_abi)),748    ("Package manifest SHA-256", code(manifest_hash)),749]:750    readme += f"| **{md_cell(key)}** | {md_cell(value)} |\n"751 752readme += f"""753## What Is Included754 755| Artifact | Path | Contents | SHA-256 |756|---|---|---|---|757"""758 759for label, path, contents, checksum in file_rows:760    readme += f"| {md_cell(label)} | {code(path)} | {md_cell(contents)} | {code(checksum)} |\n"761 762readme += f"""763## Validation764 765Generated by the Mesh LLM HF Jobs splitter from `mesh-llm` ref `{mesh_llm_ref}`.766Each artifact is checksummed as it is written, uploaded to this repository, and removed from the job workspace before the next artifact is produced.767 768```bash769skippy-model-package write-package "{source_path}" --out-dir "{package_dir}"770```771 772## Links773 774- Source model: [{source_repo}](https://huggingface.co/{source_repo})775- Mesh LLM website: [meshllm.cloud](https://www.meshllm.cloud)776- Mesh LLM: [github.com/Mesh-LLM/mesh-llm](https://github.com/Mesh-LLM/mesh-llm)777- Discord: [discord.gg/rs6fmc63eN](https://discord.gg/rs6fmc63eN)778- Package catalog: [meshllm/catalog](https://huggingface.co/datasets/meshllm/catalog)779- Package format: [layer-package-repos.md](https://github.com/Mesh-LLM/mesh-llm/blob/main/docs/specs/layer-package-repos.md)780"""781 782Path("/tmp/README.md").write_text(readme)783 784api = HfApi(token=os.environ["HF_TOKEN"])785api.upload_file(786    path_or_fileobj="/tmp/README.md",787    path_in_repo="README.md",788    repo_id=target_repo,789    repo_type="model",790)791print("  ✓ Model card uploaded")792PYTHON793 794# ─── Summary ──────────────────────────────────────────────────────────────795echo ""796echo "=== [9/9] Done ==="797echo ""798echo "  Published:  https://huggingface.co/${TARGET_REPO}"799echo "  Layers:     ${LAYER_COUNT}"800echo "  Total size: ${TOTAL_SIZE_LABEL}"801echo ""802echo "  Use with mesh-llm:"803echo "    mesh-llm serve --model ${TARGET_REPO} --split"804