build-small-hackathon/hackathon-advisor

1"""Shared taxonomy and prompt format for quest classification.2 3The dashboard refresh asks MiniCPM5-1B to classify each hackathon project against4the Build Small Hackathon judging dimensions. Beyond the six merit-badge side5quests the advisor already tracks, the contest also runs two main tracks and a set6of sponsor / special awards that are equally detectable from a project's README and7app file (which model it loads, whether it runs on Modal, whether it is agentic).8This module is the single source of truth for that label space and for the strict9two-segment prompt, so the LoRA training data and the live analyzer stay aligned.10 11Model output schema (one JSON object, nothing else):12    {"matches": [{"quest": str, "confidence": 0.0-1.0, "evidence": str,13                  "source": "readme" | "app_file"}]}14 15The live analyzer also accepts deterministic ``metadata`` matches produced from16official Space README tags. Those are not model outputs; they are structured17submission intent recorded by the hackathon submitter.18"""19from __future__ import annotations20 21from collections.abc import Mapping, Sequence22import json23import re24from typing import Any25 26 27SOURCE_README = "readme"28SOURCE_APP_FILE = "app_file"29SOURCE_METADATA = "metadata"30QUEST_SOURCES = (SOURCE_README, SOURCE_APP_FILE, SOURCE_METADATA)31 32# Canonical system prompt shared by the SFT dataset and the live analyzer so the33# model is trained and served under the exact same instruction.34QUEST_SYSTEM_PROMPT = (35    "You classify hackathon projects against fixed quest dimensions. "36    "Return exactly one strict JSON object and nothing else. "37    "The first character must be { and the last character must be }. "38    "Each match needs quest, confidence, evidence, and source (readme or app_file). "39    "Never emit markdown, prose, a top-level array, extra keys, or an unknown or rephrased quest name."40)41 42# README / app-file budgets used when rendering a project into the prompt. Kept43# small enough that prompt + completion fit the LoRA max_seq_length with headroom.44README_PROMPT_CHAR_LIMIT = 150045APP_PROMPT_CHAR_LIMIT = 190046 47 48# Ordered label space. The first six ids match the merit-badge GOALS the advisor49# already uses elsewhere; the rest are the tracks and sponsor / special awards.50QUEST_PROFILES: tuple[dict[str, str], ...] = (51    {52        "id": "Off the Grid",53        "label": "Local-first",54        "description": "Runs the model on-device with no remote inference call: weights load locally and "55        "inference happens in-process, not over a hosted API.",56        "signals": "AWARD on a local in-process load: from_pretrained / pipeline / llama_cpp / diffusers / "57        "vLLM / ONNX, GGUF weights, @spaces.GPU. DISQUALIFY (do NOT award) on ANY remote inference call, even "58        "via huggingface_hub: InferenceClient, HF Inference API/Endpoints, gradio_client to a remote Space, "59        "replicate/together/openrouter/fal/groq, a *.modal.run or other HTTP inference endpoint, or "60        "openai/anthropic/gemini/cohere clients. A remote call disqualifies regardless of which model it names.",61    },62    {63        "id": "Well-Tuned",64        "label": "Fine-tuned",65        "description": "Uses or publishes a fine-tuned or LoRA-adapted model rather than only stock checkpoints.",66        "signals": "LoRA/PEFT adapter, fine-tuned model repo, training script, words like fine-tune, adapter, SFT, distilled.",67    },68    {69        "id": "Off-Brand",70        "label": "Custom frontend",71        "description": "Ships a custom interface beyond default Gradio styling, with a memorable look or voice.",72        "signals": "custom CSS/HTML/JS, gr.HTML, gr.Blocks theme/css=, gr.Server, custom components, bespoke theming.",73    },74    {75        "id": "Llama Champion",76        "label": "llama.cpp path",77        "description": "Runs a model through the llama.cpp runtime.",78        "signals": "llama-cpp-python, from llama_cpp import Llama, GGUF file, llama.cpp, Llama( constructor.",79    },80    {81        "id": "Sharing is Caring",82        "label": "Shareable artifact",83        "description": "Produces an output people can save, post, or compare, or publishes an agent trace to the Hub.",84        "signals": "download/export button, gr.File/gr.DownloadButton, save PNG/PDF/JSON, push_to_hub of a trace or dataset.",85    },86    {87        "id": "Field Notes",88        "label": "Build notes",89        "description": "Documents the build itself with notes, a write-up, or a blog/report link.",90        "signals": "README has a substantial build write-up, devlog, lessons learned, or a blog/report/Notion link.",91    },92    {93        "id": "Backyard AI",94        "label": "Real problem for one person",95        "description": "Solves a concrete real-world problem for a specific, named person or persona.",96        "signals": "README frames a real user and task (caregiving, a relative, a job, a household chore), practical utility.",97    },98    {99        "id": "Thousand Token Wood",100        "label": "Delightful & creative",101        "description": "A delightful, playful, or artistic experience that would not exist without AI.",102        "signals": "story/game/art/whimsy framing, generative characters or worlds, playful tone, creative novelty.",103    },104    {105        "id": "OpenBMB",106        "label": "OpenBMB model",107        "description": "Uses a model published by OpenBMB (the openbmb org), such as the MiniCPM family.",108        "signals": "The model id org prefix must be exactly openbmb/ (openbmb/MiniCPM*, OpenCPM). A model from "109        "any other org is NOT OpenBMB: openai/gpt-oss, Qwen/..., meta-llama/..., google/..., nvidia/..., "110        "microsoft/..., mistralai/... do NOT count just because a model id is present.",111    },112    {113        "id": "Codex",114        "label": "OpenAI Codex",115        "description": "Uses Codex during development with Codex-attributed commits in the Space history or "116        "a linked GitHub repository.",117        "signals": "Codex co-author trailers, Codex-attributed commits, README or article describes Codex as "118        "part of implementation, debugging, documentation, deployment, or demo preparation.",119    },120    {121        "id": "Nemotron",122        "label": "NVIDIA Nemotron",123        "description": "Uses an NVIDIA Nemotron model (Nemotron LLM, Parakeet, Nemotron-Speech, Canary).",124        "signals": "model repo nvidia/...nemotron..., Parakeet, nemotron-speech, Canary ASR.",125    },126    {127        "id": "Modal",128        "label": "Modal-powered",129        "description": "Uses Modal for training, inference, or background compute.",130        "signals": "import modal, modal.App, @app.function, Modal endpoint/volume, README cites Modal compute.",131    },132    {133        "id": "Tiny Titan",134        "label": "Small model (<=4B)",135        "description": "Runs on a genuinely small model of about four billion parameters or fewer.",136        "signals": "AWARD when the model name says <=4B: 0.5B/1B/1.5B/2B/3B/4B or tiny/small/nano/mini "137        "(Qwen2.5-1.5B, MiniCPM5-1B, gemma-2b). Do NOT award for 7B/8B/12B/13B/20B/27B/35B+ models "138        "(e.g. gpt-oss-20b, Qwen2.5-7B); a version number like V-4.6 is not a parameter count.",139    },140    {141        "id": "Best Agent",142        "label": "Agentic",143        "description": "An agentic build: tool use, function calling, planning, or an autonomous multi-step loop.",144        "signals": "tool/function calling, an agent/planner loop, multiple orchestrated tools, ReAct, multi-step reasoning over tools.",145    },146)147 148QUESTS: tuple[str, ...] = tuple(profile["id"] for profile in QUEST_PROFILES)149QUEST_PROFILE_BY_ID: dict[str, dict[str, str]] = {profile["id"]: profile for profile in QUEST_PROFILES}150 151 152def _quest_key(raw: Any) -> str:153    text = " ".join(str(raw or "").replace("&", " and ").casefold().split())154    return re.sub(r"[^a-z0-9]+", " ", text).strip()155 156 157_QUEST_ALIASES: dict[str, str] = {}158for _profile in QUEST_PROFILES:159    _QUEST_ALIASES[_quest_key(_profile["id"])] = _profile["id"]160    _QUEST_ALIASES[_quest_key(_profile["label"])] = _profile["id"]161    _QUEST_ALIASES[_quest_key(f"Best {_profile['id']}")] = _profile["id"]162    _QUEST_ALIASES[_quest_key(f"Best {_profile['label']}")] = _profile["id"]163    _QUEST_ALIASES[_quest_key(f"Best Use of {_profile['id']}")] = _profile["id"]164    _QUEST_ALIASES[_quest_key(f"Best Use of {_profile['label']}")] = _profile["id"]165_QUEST_ALIASES.update(166    {167        _quest_key("Best MiniCPM Build"): "OpenBMB",168        _quest_key("MiniCPM Build"): "OpenBMB",169        _quest_key("MiniCPM"): "OpenBMB",170        _quest_key("OpenBMB / MiniCPM"): "OpenBMB",171        _quest_key("Best Use of Codex"): "Codex",172        _quest_key("OpenAI Codex"): "Codex",173        _quest_key("OpenAI"): "Codex",174        _quest_key("Codex"): "Codex",175        _quest_key("Offbrand"): "Off-Brand",176        _quest_key("Offgrid"): "Off the Grid",177        _quest_key("Llama Champion badge"): "Llama Champion",178        _quest_key("Modal first"): "Modal",179        _quest_key("Nemotron 3 Nano 4B"): "Nemotron",180        _quest_key("Nemotron Mini 4B"): "Nemotron",181        _quest_key("Small model <=4B"): "Tiny Titan",182        _quest_key("Small model under 4B"): "Tiny Titan",183        _quest_key("Shareable output"): "Sharing is Caring",184        _quest_key("Custom UI"): "Off-Brand",185        _quest_key("Custom interface"): "Off-Brand",186        _quest_key("Local first"): "Off the Grid",187        _quest_key("Fine tuned"): "Well-Tuned",188        _quest_key("Fine tune"): "Well-Tuned",189    }190)191 192 193TRACK_METADATA_TAG_TO_QUEST: dict[str, str] = {194    "track:backyard": "Backyard AI",195    "track:wood": "Thousand Token Wood",196}197TRACK_QUESTS: tuple[str, ...] = tuple(TRACK_METADATA_TAG_TO_QUEST.values())198 199OFFICIAL_METADATA_TAG_TO_QUEST: dict[str, str] = {200    **TRACK_METADATA_TAG_TO_QUEST,201    "sponsor:openbmb": "OpenBMB",202    "sponsor:openai": "Codex",203    "sponsor:nvidia": "Nemotron",204    "sponsor:modal": "Modal",205    "achievement:offgrid": "Off the Grid",206    "achievement:welltuned": "Well-Tuned",207    "achievement:offbrand": "Off-Brand",208    "achievement:llama": "Llama Champion",209    "achievement:sharing": "Sharing is Caring",210    "achievement:fieldnotes": "Field Notes",211    # These tags are not emitted by the current submitter, but they appear in212    # hand-authored Space metadata and map cleanly to existing official prizes.213    "tiny-titan": "Tiny Titan",214    "best-agent": "Best Agent",215}216 217 218def quest_profiles(quest_ids: Sequence[str] | None = None) -> list[dict[str, str]]:219    return [220        {"id": profile["id"], "label": profile["label"], "description": profile["description"]}221        for profile in _selected_quest_profiles(quest_ids)222    ]223 224 225def quest_label(quest: str) -> str:226    return QUEST_PROFILE_BY_ID.get(quest, {}).get("label", quest)227 228 229def canonical_quest_id(raw_quest: Any) -> str:230    quest = " ".join(str(raw_quest or "").split())231    if quest in QUEST_PROFILE_BY_ID:232        return quest233    alias = _QUEST_ALIASES.get(_quest_key(quest))234    if alias:235        return alias236    folded = quest.casefold()237    for known in QUESTS:238        known_folded = known.casefold()239        if folded == known_folded:240            return known241        if folded.startswith(f"{known_folded} (") or folded.startswith(f"{known_folded} - "):242            return known243    raise ValueError(f"unknown quest: {quest!r}")244 245 246def canonical_quest_ids(raw_quest: Any) -> tuple[str, ...]:247    quest = " ".join(str(raw_quest or "").split())248    try:249        return (canonical_quest_id(quest),)250    except ValueError as original_error:251        parts = [part.strip() for part in re.split(r"\s*/\s*", quest) if part.strip()]252        if len(parts) <= 1:253            raise original_error254    canonical: list[str] = []255    for part in parts:256        try:257            quest_id = canonical_quest_id(part)258        except ValueError as error:259            raise ValueError(f"unknown quest in composite {quest!r}: {part!r}") from error260        if quest_id not in canonical:261            canonical.append(quest_id)262    return tuple(canonical)263 264 265def declared_quest_matches_from_tags(tags: Sequence[Any]) -> list[dict[str, Any]]:266    """Return deterministic quest matches declared by official Space metadata tags."""267    matches: list[dict[str, Any]] = []268    seen: set[str] = set()269    for raw_tag in tags or ():270        tag = " ".join(str(raw_tag or "").split())271        quest = OFFICIAL_METADATA_TAG_TO_QUEST.get(tag.casefold())272        if not quest or quest in seen:273            continue274        seen.add(quest)275        matches.append(276            normalize_match(277                {278                    "quest": quest,279                    "confidence": 1.0,280                    "evidence": f"official Space tag {tag.casefold()}",281                    "source": SOURCE_METADATA,282                }283            )284        )285    return matches286 287 288def metadata_suppressed_quests_from_tags(tags: Sequence[Any]) -> set[str]:289    """Return quest ids that metadata should remove from model inference."""290    suppressed: set[str] = set()291    declared_tags = {" ".join(str(raw_tag or "").split()).casefold() for raw_tag in tags or ()}292    if declared_tags & set(TRACK_METADATA_TAG_TO_QUEST):293        suppressed.update(TRACK_QUESTS)294    return suppressed295 296 297def _selected_quest_profiles(quest_ids: Sequence[str] | None) -> tuple[dict[str, str], ...]:298    if quest_ids is None:299        return QUEST_PROFILES300    selected: list[dict[str, str]] = []301    seen: set[str] = set()302    for raw_quest in quest_ids:303        quest = canonical_quest_id(raw_quest)304        if quest in seen:305            continue306        seen.add(quest)307        selected.append(QUEST_PROFILE_BY_ID[quest])308    return tuple(selected)309 310 311def _clip(text: str, limit: int) -> str:312    cleaned = (text or "").strip()313    if len(cleaned) <= limit:314        return cleaned315    return cleaned[:limit].rstrip() + " ..."316 317 318_IMPORT_RE = re.compile(r"^\s*(?:import\s+\w|from\s+\w[\w.]*\s+import)\b")319_REPO_ID_RE = re.compile(r"\b[\w-]+/[\w.\-]+\b")320 321 322def build_readme_segment(readme_body: str) -> str:323    return " ".join(str(readme_body or "").split())[: README_PROMPT_CHAR_LIMIT * 2]324 325 326def build_app_segment(app_source: str, app_signals: str = "") -> str:327    """Compose an app-file view that keeps imports and asset ids inside budget.328 329    Gradio apps front-load the decisive quest signals (which library is imported,330    which model repo is loaded) but a deep model id can fall outside a head slice,331    so imports are hoisted and any repo-id-looking tokens from the AST signals that332    are still missing are appended as a compact ASSETS line. The SFT dataset and the333    live analyzer both call this so the model sees the same app view either way.334    """335    source = str(app_source or "")336    if not source.strip() and not str(app_signals or "").strip():337        return ""338    imports = [line.strip() for line in source.splitlines() if _IMPORT_RE.match(line)]339    seen: set[str] = set()340    ordered_imports = [imp for imp in imports if not (imp in seen or seen.add(imp))][:40]341    head_budget = APP_PROMPT_CHAR_LIMIT * 2342    parts: list[str] = []343    if ordered_imports:344        parts.append("\n".join(ordered_imports))345    parts.append(source)346    composed = "\n\n".join(parts)[:head_budget]347    repo_ids = {token for token in _REPO_ID_RE.findall(app_signals or "") if "/" in token}348    missing = sorted(rid for rid in repo_ids if rid not in composed)349    if missing:350        composed = f"{composed}\n\nASSETS: {', '.join(missing[:12])}"351    return composed352 353 354def render_quest_prompt(355    *,356    title: str,357    sdk: str,358    declared_models: Sequence[str],359    tags: Sequence[str],360    readme_segment: str,361    app_file_name: str,362    app_file_segment: str,363    include_signals: bool = True,364    quest_ids: Sequence[str] | None = None,365) -> str:366    """Render the canonical two-segment classification prompt.367 368    The same renderer feeds both the SFT dataset and the live analyzer so the model369    never sees a different shape at training and inference time.370    """371    profiles = _selected_quest_profiles(quest_ids)372    quest_lines = [f"- {profile['id']}: {profile['description']}" for profile in profiles]373    if include_signals:374        quest_lines = [375            f"- {profile['id']}: {profile['description']} Signals: {profile['signals']}"376            for profile in profiles377        ]378    readme_text = _clip(readme_segment, README_PROMPT_CHAR_LIMIT) or "(no README description provided)"379    app_label = app_file_name.strip() or "(unknown)"380    app_text = _clip(app_file_segment, APP_PROMPT_CHAR_LIMIT) or "(no app file available)"381    metadata = {382        "title": (title or "").strip(),383        "sdk": (sdk or "").strip(),384        "declared_models": [str(model) for model in declared_models or []],385        "tags": [str(tag) for tag in tags or []],386    }387    return "\n".join(388        [389            "Classify this hackathon project against the quest dimensions below.",390            "Read the two evidence segments (README and APP_FILE) and judge each quest only from them.",391            "",392            "Quests (copy the id on the left verbatim):",393            *quest_lines,394            "",395            "Rules:",396            "- Include a quest only when a segment gives clear, specific evidence.",397            "- quest must be one id from the list above, copied exactly. Never invent or rephrase a quest name.",398            "- confidence is a number between 0 and 1.",399            "- evidence is a 3-to-12 word quote or tight paraphrase taken from the segment you cite.",400            '- source is "readme" when the evidence is in the README segment, "app_file" when it is in the APP_FILE segment.',401            "- At most one match per quest. Sort matches by confidence, highest first.",402            "- If no quest has clear evidence, return an empty matches list.",403            '- Output exactly one JSON object: {"matches":[{"quest":"...","confidence":0.0,"evidence":"...","source":"readme"}]}.',404            "- No markdown, no code fences, no commentary, no extra keys.",405            "",406            f"METADATA: {json.dumps(metadata, ensure_ascii=False)}",407            "",408            "[README]",409            readme_text,410            "",411            f"[APP_FILE] {app_label}",412            app_text,413        ]414    )415 416 417def normalize_match(match: Mapping[str, Any], *, evidence_limit: int = 360) -> dict[str, Any]:418    """Validate and canonicalize one match dict. Raises ValueError on schema drift."""419    quest = canonical_quest_id(match.get("quest"))420    try:421        confidence = float(match.get("confidence"))422    except (TypeError, ValueError) as error:423        raise ValueError("confidence must be numeric") from error424    if not 0.0 < confidence <= 1.0:425        raise ValueError("confidence must be greater than 0 and no more than 1")426    evidence = " ".join(str(match.get("evidence") or "").split())427    if not evidence:428        raise ValueError("evidence must not be empty")429    if _looks_like_prompt_taxonomy(evidence):430        raise ValueError("evidence must come from README or APP_FILE, not quest instructions")431    source = str(match.get("source") or "")432    if source not in QUEST_SOURCES:433        raise ValueError(f"source must be one of {QUEST_SOURCES}, got {source!r}")434    return {435        "quest": quest,436        "confidence": round(confidence, 3),437        "evidence": evidence[:evidence_limit],438        "source": source,439    }440 441 442def _looks_like_prompt_taxonomy(evidence: str) -> bool:443    normalized = " ".join(evidence.casefold().split())444    if "signals:" in normalized:445        return True446    return any(447        normalized.startswith(" ".join(profile[field].casefold().split())[:80])448        for profile in QUEST_PROFILES449        for field in ("description",)450    )451