build-small-hackathon/hackathon-advisor

1from __future__ import annotations2 3from collections.abc import Mapping, Sequence4from contextlib import nullcontext5from dataclasses import dataclass6import json7import os8from typing import Any, Protocol9 10from hackathon_advisor.config import first_nonempty_env11from hackathon_advisor.data import Project, normalize_project_tags12from hackathon_advisor.model_runtime import (13    DEFAULT_MODEL_ID,14    _minicpm_generation_kwargs,15    _load_minicpm_causal_lm,16    _minicpm_chat_inputs,17    _resolve_torch_device,18)19from hackathon_advisor.quest_taxonomy import (20    QUEST_SYSTEM_PROMPT,21    QUESTS,22    build_app_segment,23    build_readme_segment,24    canonical_quest_ids,25    declared_quest_matches_from_tags,26    metadata_suppressed_quests_from_tags,27    normalize_match,28    render_quest_prompt,29)30 31 32MAX_QUEST_TOKENS = 102433DEFAULT_QUEST_ADAPTER_ID = "build-small-hackathon/hackathon-advisor-quest-minicpm5-lora"34DEFAULT_QUEST_ADAPTER_REVISION = ""35METADATA_FIRST_QUEST_ANALYZER_SOURCE = "metadata-first-minicpm-json-quest-analyzer"36 37 38class QuestAnalysisError(RuntimeError):39    pass40 41 42class QuestAnalyzer(Protocol):43    source: str44 45    def analyze(self, projects: Sequence[Project]) -> dict[str, list[dict[str, Any]]]:46        ...47 48 49@dataclass(frozen=True)50class ValidatedQuestAnalysis:51    matches_by_project: dict[str, list[dict[str, Any]]]52    source: str53 54 55class MiniCPMQuestAnalyzer:56    source = METADATA_FIRST_QUEST_ANALYZER_SOURCE57 58    def __init__(59        self,60        model_id: str = DEFAULT_MODEL_ID,61        *,62        device: str = "auto",63        adapter_id: str = DEFAULT_QUEST_ADAPTER_ID,64        adapter_revision: str = DEFAULT_QUEST_ADAPTER_REVISION,65    ) -> None:66        self.model_id = model_id.strip() or DEFAULT_MODEL_ID67        self.device = (device or "auto").strip().lower() or "auto"68        self.adapter_id = adapter_id.strip()69        self.adapter_revision = adapter_revision.strip()70        self.resolved_device = ""71        self._tokenizer = None72        self._model = None73 74    def analyze(self, projects: Sequence[Project]) -> dict[str, list[dict[str, Any]]]:75        matches: dict[str, list[dict[str, Any]]] = {}76        for project in projects:77            declared_matches = declared_project_quest_matches(project)78            remaining_quests = remaining_project_quest_ids(project)79            if not remaining_quests:80                matches[project.id] = declared_matches81                continue82            self._ensure_loaded()83            try:84                raw = self._generate_json(85                    render_project_quest_prompt(project, quest_ids=remaining_quests)86                )87                validated = self._validate_or_repair_project(project, raw).matches_by_project88                matches[project.id] = merge_declared_and_inferred_matches(89                    declared_matches,90                    validated.get(project.id, []),91                )92            except QuestAnalysisError as error:93                # Tolerate a single unparseable project: record empty matches and continue, so one94                # malformed model output never aborts a whole-org refresh.95                print(f"[quest-analysis] skipped {project.id}: {error}", flush=True)96                matches[project.id] = declared_matches97        return matches98 99    def _validate_or_repair_project(self, project: Project, raw: Mapping[str, Any]) -> ValidatedQuestAnalysis:100        try:101            return _validate_single_project_payload(project, raw)102        except QuestAnalysisError as error:103            repaired = self._repair_schema_json(raw, str(error))104            try:105                return _validate_single_project_payload(project, repaired)106            except QuestAnalysisError as repair_error:107                raise QuestAnalysisError(f"{error}; MiniCPM schema repair failed: {repair_error}") from repair_error108 109    def _ensure_loaded(self) -> None:110        if self._model is not None and self._tokenizer is not None:111            return112        try:113            import torch114            from transformers import AutoModelForCausalLM, AutoTokenizer115            if self.adapter_id:116                from peft import PeftConfig, PeftModel117        except ImportError as error:118            raise QuestAnalysisError(119                "MiniCPM quest analysis requires torch and transformers (and peft when "120                "ADVISOR_QUEST_ADAPTER_ID is set). Install runtime requirements before enabling dashboard refresh."121            ) from error122 123        base_model_id = self.model_id124        tokenizer_id = self.adapter_id or base_model_id125        adapter_kwargs = {"revision": self.adapter_revision} if self.adapter_revision else {}126        if self.adapter_id:127            adapter_config = PeftConfig.from_pretrained(self.adapter_id, **adapter_kwargs)128            base_model_id = str(adapter_config.base_model_name_or_path or base_model_id)129 130        target = _resolve_torch_device(self.device, torch)131        self.resolved_device = target132        self._tokenizer = AutoTokenizer.from_pretrained(133            tokenizer_id,134            trust_remote_code=True,135            **(adapter_kwargs if self.adapter_id else {}),136        )137        model = _load_minicpm_causal_lm(AutoModelForCausalLM, base_model_id, target, torch)138        if self.adapter_id:139            model = PeftModel.from_pretrained(model, self.adapter_id, **adapter_kwargs)140            if target not in ("auto", "cpu"):141                model = model.to(target)142        model.eval()143        self._model = model144 145    def _generate_json(self, prompt: str) -> dict[str, Any]:146        text = self._generate_text(QUEST_SYSTEM_PROMPT, prompt)147        try:148            parsed = _extract_json_object(text)149        except QuestAnalysisError as error:150            try:151                # Deterministic repair first: escape unescaped double quotes inside string values152                # (the model copies snippets like class="x" verbatim). Avoids an LLM round-trip and153                # preserves the evidence text exactly.154                parsed = _extract_json_object(_escape_unescaped_quotes(text))155            except QuestAnalysisError:156                repaired = self._repair_invalid_json(text)157                try:158                    parsed = _extract_json_object(repaired)159                except QuestAnalysisError as repair_error:160                    preview = " ".join(text.split())[:280]161                    repair_preview = " ".join(repaired.split())[:280]162                    raise QuestAnalysisError(163                        f"{error}: {preview}; MiniCPM JSON repair failed: {repair_error}: {repair_preview}"164                    ) from repair_error165        if not isinstance(parsed, dict):166            raise QuestAnalysisError("quest analyzer did not return a JSON object")167        return parsed168 169    def _generate_text(self, system_prompt: str, user_prompt: str, *, disable_adapter: bool = False) -> str:170        import torch171 172        assert self._tokenizer is not None173        assert self._model is not None174        messages = [175            {"role": "system", "content": system_prompt},176            {"role": "user", "content": user_prompt},177        ]178        inputs = _minicpm_chat_inputs(179            self._tokenizer,180            messages,181            enable_thinking=False,182            device=next(self._model.parameters()).device,183        )184        generation_kwargs = _minicpm_generation_kwargs(185            inputs,186            max_new_tokens=MAX_QUEST_TOKENS,187            temperature=0.0,  # strict JSON wants deterministic greedy decoding188        )189        generation_kwargs["eos_token_id"] = self._chat_eos_token_id()190        adapter_context = _disabled_adapter(self._model) if disable_adapter else nullcontext()191        with adapter_context, torch.inference_mode():192            output = self._model.generate(**generation_kwargs)193        generated = output[:, inputs["input_ids"].shape[-1] :]194        return self._tokenizer.decode(generated[0], skip_special_tokens=True).strip()195 196    def _repair_invalid_json(self, invalid_output: str) -> str:197        repair_system = "You repair JSON. Return exactly one valid JSON object and nothing else."198        repair_prompt = "\n".join(199            [200                "Rewrite this invalid JSON as valid compact JSON.",201                "Every match object must contain exactly these keys: quest, confidence, evidence, source.",202                "Keep the same matches, quest names, confidence values, and source values.",203                "If an evidence value contains unescaped double quote characters, escape them or paraphrase the evidence.",204                "Never omit source. If a source key was damaged by malformed JSON, infer readme or app_file from the damaged object.",205                "Drop any match whose quest is not valid, or whose source cannot be inferred as readme or app_file.",206                f"Valid quests: {', '.join(QUESTS)}.",207                'Valid sources: readme, app_file.',208                "Do not copy any text from these repair instructions into evidence.",209                "",210                "Invalid JSON:",211                invalid_output,212            ]213        )214        return self._generate_text(repair_system, repair_prompt, disable_adapter=True)215 216    def _repair_schema_json(self, parsed_output: Mapping[str, Any], validation_error: str) -> dict[str, Any]:217        repair_system = "You repair JSON schemas. Return exactly one valid JSON object and nothing else."218        repair_prompt = "\n".join(219            [220                "The following quest-classification JSON parsed, but failed schema validation.",221                f"Validation error: {validation_error}",222                "",223                "Rewrite it to satisfy this schema exactly:",224                '{"matches":[{"quest":"...","confidence":0.1,"evidence":"...","source":"readme"}]}',225                "",226                "Rules:",227                f"- quest must be one of: {', '.join(QUESTS)}.",228                "- source must be readme or app_file.",229                "- Every match object must include source; never omit it.",230                "- If source is missing but evidence clearly came from code, use app_file; if it clearly came from prose, use readme.",231                "- confidence must be greater than 0 and no more than 1.",232                "- Keep at most one match per quest; keep the strongest and clearest evidence.",233                "- Remove matches with empty evidence or evidence copied from the quest instructions.",234                "- Do not copy any text from these repair instructions into evidence.",235                "- Do not add new quests that are not already intended by the input.",236                "",237                "Input JSON:",238                json.dumps(parsed_output, ensure_ascii=False, separators=(",", ":")),239            ]240        )241        repaired = self._generate_text(repair_system, repair_prompt, disable_adapter=True)242        parsed = _extract_json_object(repaired)243        if not isinstance(parsed, dict):244            raise QuestAnalysisError("MiniCPM schema repair did not return a JSON object")245        return parsed246 247    def _chat_eos_token_id(self) -> int:248        assert self._tokenizer is not None249        token_id = self._tokenizer.convert_tokens_to_ids("<|im_end|>")250        if not isinstance(token_id, int) or token_id < 0:251            raise QuestAnalysisError("MiniCPM tokenizer is missing the <|im_end|> chat terminator")252        return token_id253 254 255def resolve_quest_identity(env: Mapping[str, str] | None = None) -> tuple[str, str, str]:256    """Resolve ``(model_id, adapter_id, adapter_revision)`` for the quest analyzer.257 258    Shared by ``create_quest_analyzer`` (the live load) and the quest-cache fingerprint so259    the serving runtime and the cache key resolve identically (e.g. on whitespace-padded env).260    """261    model_id = first_nonempty_env(262        "ADVISOR_QUEST_MODEL_ID", "ADVISOR_MODEL_ID", default=DEFAULT_MODEL_ID, env=env263    )264    adapter_id = first_nonempty_env("ADVISOR_QUEST_ADAPTER_ID", default=DEFAULT_QUEST_ADAPTER_ID, env=env)265    adapter_revision = first_nonempty_env(266        "ADVISOR_QUEST_ADAPTER_REVISION", default=DEFAULT_QUEST_ADAPTER_REVISION, env=env267    )268    return model_id, adapter_id, adapter_revision269 270 271def create_quest_analyzer(device: str = "auto") -> QuestAnalyzer:272    backend = os.environ.get("ADVISOR_QUEST_ANALYZER_BACKEND", "").strip().lower()273    if not backend:274        backend = os.environ.get("ADVISOR_MODEL_BACKEND", "").strip().lower()275    if backend in {"minicpm", "minicpm-transformers"}:276        model_id, adapter_id, adapter_revision = resolve_quest_identity()277        return MiniCPMQuestAnalyzer(278            model_id,279            device=device,280            adapter_id=adapter_id,281            adapter_revision=adapter_revision,282        )283    raise QuestAnalysisError(284        "Dashboard refresh requires ADVISOR_QUEST_ANALYZER_BACKEND=minicpm-transformers. "285        f"Got {backend or 'unset'}."286    )287 288 289def validate_quest_analysis_payload(290    payload: Mapping[str, Any],291    projects: Sequence[Project],292    *,293    source: str = "validated-json",294) -> ValidatedQuestAnalysis:295    rows = payload.get("projects")296    if not isinstance(rows, list):297        raise QuestAnalysisError("quest analysis JSON must contain a projects list")298    expected_ids = [project.id for project in projects]299    expected = set(expected_ids)300    seen: set[str] = set()301    matches_by_project: dict[str, list[dict[str, Any]]] = {}302    for row in rows:303        if not isinstance(row, dict):304            raise QuestAnalysisError("quest project rows must be objects")305        project_id = str(row.get("project_id") or "")306        if project_id not in expected:307            raise QuestAnalysisError(f"quest analysis returned an unknown project id: {project_id}")308        if project_id in seen:309            raise QuestAnalysisError(f"quest analysis returned a duplicate project id: {project_id}")310        seen.add(project_id)311        matches_by_project[project_id] = _validate_project_matches(row.get("matches"), project_id)312    missing = [project_id for project_id in expected_ids if project_id not in seen]313    if missing:314        raise QuestAnalysisError(f"quest analysis missed {len(missing)} projects")315    return ValidatedQuestAnalysis(matches_by_project=matches_by_project, source=source)316 317 318def validate_matches_by_project(319    matches_by_project: Mapping[str, Sequence[Mapping[str, Any]]],320    projects: Sequence[Project],321    *,322    source: str,323) -> ValidatedQuestAnalysis:324    payload = {325        "projects": [326            {327                "project_id": project.id,328                "matches": list(matches_by_project.get(project.id, [])),329            }330            for project in projects331        ]332    }333    return validate_quest_analysis_payload(payload, projects, source=source)334 335 336def _validate_single_project_payload(project: Project, raw: Mapping[str, Any]) -> ValidatedQuestAnalysis:337    return validate_quest_analysis_payload(338        {339            "projects": [340                {341                    "project_id": project.id,342                    "matches": raw.get("matches"),343                }344            ]345        },346        [project],347    )348 349 350def render_project_quest_prompt(project: Project, quest_ids: Sequence[str] | None = None) -> str:351    """Render the strict two-segment quest prompt from a snapshot Project.352 353    Refresh snapshots carry the same raw README body and main app source segments354    used for quest LoRA training.355    """356    return render_quest_prompt(357        title=project.title,358        sdk=project.sdk,359        declared_models=project.models,360        tags=normalize_project_tags(project.tags),361        readme_segment=build_readme_segment(project.readme_body),362        app_file_name=project.app_file,363        app_file_segment=build_app_segment(project.app_file_source, project.app_file_embedding_text),364        quest_ids=quest_ids,365    )366 367 368def render_project_inference_prompt(project: Project) -> str:369    """Render the exact prompt used for model inference after metadata claims."""370    return render_project_quest_prompt(project, quest_ids=remaining_project_quest_ids(project))371 372 373def declared_project_quest_matches(project: Project) -> list[dict[str, Any]]:374    """Return quest matches declared by official Build Small Space metadata."""375    return declared_quest_matches_from_tags(normalize_project_tags(project.tags))376 377 378def remaining_project_quest_ids(project: Project) -> tuple[str, ...]:379    tags = normalize_project_tags(project.tags)380    declared = {match["quest"] for match in declared_quest_matches_from_tags(tags)}381    suppressed = metadata_suppressed_quests_from_tags(tags)382    return tuple(quest for quest in QUESTS if quest not in declared and quest not in suppressed)383 384 385def merge_declared_and_inferred_matches(386    declared_matches: Sequence[Mapping[str, Any]],387    inferred_matches: Sequence[Mapping[str, Any]],388) -> list[dict[str, Any]]:389    """Merge metadata and model matches, keeping metadata as the authority."""390    merged: list[dict[str, Any]] = []391    seen: set[str] = set()392    for raw_match in (*declared_matches, *inferred_matches):393        match = normalize_match(raw_match)394        if match["quest"] in seen:395            continue396        seen.add(match["quest"])397        merged.append(match)398    return merged399 400 401def _validate_project_matches(raw_matches: Any, project_id: str) -> list[dict[str, Any]]:402    if not isinstance(raw_matches, list):403        raise QuestAnalysisError(f"quest matches for {project_id} must be a list")404    matches: list[dict[str, Any]] = []405    seen: set[str] = set()406    for raw_match in raw_matches:407        if not isinstance(raw_match, dict):408            raise QuestAnalysisError(f"quest matches for {project_id} must be objects")409        try:410            quest_ids = canonical_quest_ids(raw_match.get("quest"))411        except ValueError as error:412            raise QuestAnalysisError(f"quest match for {project_id}: {error}") from error413        for quest_id in quest_ids:414            try:415                match = normalize_match({**raw_match, "quest": quest_id})416            except ValueError as error:417                raise QuestAnalysisError(f"quest match for {project_id}: {error}") from error418            if match["quest"] in seen:419                raise QuestAnalysisError(f"duplicate quest for {project_id}: {match['quest']}")420            seen.add(match["quest"])421            matches.append(match)422    return matches423 424 425def _escape_unescaped_quotes(text: str) -> str:426    """Escape double quotes inside JSON string values that are not string terminators.427 428    The quest model sometimes copies code verbatim into a free-text field, e.g.429    ``"evidence":"class="x" ..."``. A quote closes a string only when the next430    non-whitespace character is a JSON structural token (``: , } ]``) or end of input;431    any other in-string quote is escaped so ``json.loads`` can parse the value.432    """433    out: list[str] = []434    in_string = False435    i = 0436    length = len(text)437    while i < length:438        char = text[i]439        if not in_string:440            out.append(char)441            if char == '"':442                in_string = True443            i += 1444            continue445        if char == "\\":446            out.append(char)447            if i + 1 < length:448                out.append(text[i + 1])449                i += 2450            else:451                i += 1452            continue453        if char == '"':454            nxt = i + 1455            while nxt < length and text[nxt] in " \t\r\n":456                nxt += 1457            if nxt >= length or text[nxt] in ":,}]":458                out.append(char)459                in_string = False460            else:461                out.append('\\"')462            i += 1463            continue464        out.append(char)465        i += 1466    return "".join(out)467 468 469def _extract_json_object(text: str) -> Any:470    text = _strip_json_fence(text.strip())471    decoder = json.JSONDecoder()472    for index, char in enumerate(text):473        if char != "{":474            continue475        try:476            value, offset = decoder.raw_decode(text[index:])477        except json.JSONDecodeError:478            continue479        if text[index + offset :].strip():480            continue481        return value482    raise QuestAnalysisError("quest analyzer returned invalid JSON")483 484 485def _disabled_adapter(model: Any) -> Any:486    disable_adapter = getattr(model, "disable_adapter", None)487    if callable(disable_adapter):488        return disable_adapter()489    return nullcontext()490 491 492def _strip_json_fence(text: str) -> str:493    if not text.startswith("```"):494        return text495    lines = text.splitlines()496    if len(lines) < 3 or not lines[-1].strip().startswith("```"):497        return text498    opener = lines[0].strip().lower()499    if opener not in {"```", "```json"}:500        return text501    return "\n".join(lines[1:-1]).strip()502