build-small-hackathon/hackathon-advisor
16
1from __future__ import annotations2 3from hackathon_advisor.dashboard import (4 CLUSTER_LABEL_ALGORITHM,5 build_dashboard_payload,6 validate_dashboard_payload,7)8from hackathon_advisor.data import Project, ProjectIndex, build_index_payload9from hackathon_advisor.quest_analysis import (10 MiniCPMQuestAnalyzer,11 QuestAnalysisError,12 _extract_json_object,13 create_quest_analyzer,14 render_project_quest_prompt,15 remaining_project_quest_ids,16 validate_quest_analysis_payload,17)18from hackathon_advisor.quest_taxonomy import declared_quest_matches_from_tags19from hackathon_advisor.tools import GOALS20 21 22def test_dashboard_builder_projects_embeddings_with_tsne_and_clusters() -> None:23 index = fake_index()24 quest_matches = {25 project.id: [26 {27 "quest": GOALS[project_index % len(GOALS)],28 "confidence": 0.75,29 "evidence": "project evidence matches the quest",30 "source": "readme" if project_index % 2 == 0 else "app_file",31 }32 ]33 for project_index, project in enumerate(index.projects)34 }35 36 payload = build_dashboard_payload(37 index,38 quest_matches=quest_matches,39 quest_source="fake-strict-analyzer",40 generated_at="2026-06-08T00:00:00+00:00",41 )42 43 validate_dashboard_payload(payload)44 assert payload["layout"]["algorithm"] == "tsne"45 assert payload["layout"]["metric"] == "cosine"46 assert len(payload["points"]) == len(index.projects)47 assert len(payload["clusters"]) == 648 assert payload["links"]49 assert payload["quest_report"]["status"] == "analyzed"50 assert all(0 <= point["x"] <= 100 and 0 <= point["y"] <= 100 for point in payload["points"])51 assert all(point["quest_ids"] for point in payload["points"])52 assert payload["cluster_label_algorithm"] == CLUSTER_LABEL_ALGORITHM53 54 55def test_dashboard_builder_is_deterministic_for_fixed_vectors() -> None:56 index = fake_index()57 58 left = build_dashboard_payload(index, generated_at="2026-06-08T00:00:00+00:00")59 right = build_dashboard_payload(index, generated_at="2026-06-08T00:00:00+00:00")60 61 assert [(point["id"], point["x"], point["y"]) for point in left["points"]] == [62 (point["id"], point["x"], point["y"]) for point in right["points"]63 ]64 assert left["clusters"] == right["clusters"]65 66 67def test_dashboard_cluster_labels_ignore_hackathon_wide_noise() -> None:68 index = noisy_cluster_label_index()69 70 payload = build_dashboard_payload(index, generated_at="2026-06-08T00:00:00+00:00")71 72 banned = {"ai", "build-small-hackathon", "gradio", "hackathon", "project", "region", "us"}73 keywords = {keyword for cluster in payload["clusters"] for keyword in cluster["keywords"]}74 assert keywords.isdisjoint(banned)75 assert {"dream", "family", "garden", "notice", "order", "repair"} & keywords76 assert all("region:us" not in point["tags"] for point in payload["points"])77 78 79def test_quest_analysis_validation_accepts_strict_project_coverage() -> None:80 projects = fake_projects(4)81 raw = {82 "projects": [83 {84 "project_id": project.id,85 "matches": [86 {87 "quest": "Off the Grid",88 "confidence": 0.9,89 "evidence": "runs without proprietary inference APIs",90 "source": "app_file",91 }92 ],93 }94 for project in projects95 ]96 }97 98 validated = validate_quest_analysis_payload(raw, projects, source="fake")99 100 assert validated.source == "fake"101 assert set(validated.matches_by_project) == {project.id for project in projects}102 assert validated.matches_by_project[projects[0].id][0]["quest"] == "Off the Grid"103 104 105def test_quest_analysis_validation_rejects_malformed_output() -> None:106 projects = fake_projects(2)107 raw = {108 "projects": [109 {110 "project_id": projects[0].id,111 "matches": [{"quest": "Off the Grid", "confidence": 1.2, "evidence": ""}],112 }113 ]114 }115 116 try:117 validate_quest_analysis_payload(raw, projects)118 except QuestAnalysisError as error:119 assert "confidence" in str(error) or "missed" in str(error)120 else:121 raise AssertionError("malformed quest analysis should fail")122 123 124def test_quest_analysis_validation_requires_source_field() -> None:125 projects = fake_projects(1)126 raw = {127 "projects": [128 {129 "project_id": projects[0].id,130 "matches": [{"quest": "Off the Grid", "confidence": 0.9, "evidence": "local model"}],131 }132 ]133 }134 135 try:136 validate_quest_analysis_payload(raw, projects)137 except QuestAnalysisError as error:138 assert "source" in str(error)139 else:140 raise AssertionError("a match without a source must be rejected")141 142 143def test_quest_analysis_validation_rejects_prompt_taxonomy_as_evidence() -> None:144 projects = fake_projects(1)145 raw = {146 "projects": [147 {148 "project_id": projects[0].id,149 "matches": [150 {151 "quest": "Off the Grid",152 "confidence": 0.0,153 "evidence": (154 "Runs entirely on local or open-weight models with no proprietary cloud inference APIs. "155 "Signals: local transformers model load"156 ),157 "source": "readme",158 }159 ],160 }161 ]162 }163 164 try:165 validate_quest_analysis_payload(raw, projects)166 except QuestAnalysisError as error:167 assert "confidence" in str(error) or "quest instructions" in str(error)168 else:169 raise AssertionError("prompt taxonomy evidence must be rejected")170 171 172def test_quest_analysis_validation_accepts_expanded_track_quests() -> None:173 projects = fake_projects(1)174 raw = {175 "projects": [176 {177 "project_id": projects[0].id,178 "matches": [179 {"quest": "Nemotron", "confidence": 0.8, "evidence": "nvidia parakeet asr", "source": "app_file"},180 {"quest": "Tiny Titan", "confidence": 0.7, "evidence": "MiniCPM5-1B", "source": "readme"},181 ],182 }183 ]184 }185 186 validated = validate_quest_analysis_payload(raw, projects, source="fake")187 188 quests = {match["quest"] for match in validated.matches_by_project[projects[0].id]}189 assert quests == {"Nemotron", "Tiny Titan"}190 assert validated.matches_by_project[projects[0].id][0]["source"] in {"readme", "app_file"}191 192 193def test_declared_quest_matches_from_official_space_tags() -> None:194 matches = declared_quest_matches_from_tags(195 [196 "track:wood",197 "sponsor:openai",198 "sponsor:nvidia",199 "achievement:llama",200 "tiny-titan",201 "region:us",202 "unknown:tag",203 ]204 )205 206 quests = [match["quest"] for match in matches]207 assert quests == ["Thousand Token Wood", "Codex", "Nemotron", "Llama Champion", "Tiny Titan"]208 assert {match["source"] for match in matches} == {"metadata"}209 assert all(match["confidence"] == 1.0 for match in matches)210 211 212def test_quest_analysis_validation_canonicalizes_known_label_suffixes() -> None:213 projects = fake_projects(1)214 raw = {215 "projects": [216 {217 "project_id": projects[0].id,218 "matches": [219 {220 "quest": "Off the Grid (LOCAL-FIRST)",221 "confidence": 0.9,222 "evidence": "local gguf model",223 "source": "app_file",224 }225 ],226 }227 ]228 }229 230 validated = validate_quest_analysis_payload(raw, projects, source="fake")231 232 assert validated.matches_by_project[projects[0].id][0]["quest"] == "Off the Grid"233 234 235def test_quest_analysis_validation_expands_known_composite_quest_labels() -> None:236 projects = fake_projects(1)237 raw = {238 "projects": [239 {240 "project_id": projects[0].id,241 "matches": [242 {243 "quest": "Best MiniCPM Build / Tiny Titan",244 "confidence": 0.84,245 "evidence": "MiniCPM5-1B model",246 "source": "app_file",247 },248 {249 "quest": "Off-Brand / Sharing is Caring",250 "confidence": 0.72,251 "evidence": "custom UI exports a card",252 "source": "readme",253 },254 ],255 }256 ]257 }258 259 validated = validate_quest_analysis_payload(raw, projects, source="fake")260 261 quests = [match["quest"] for match in validated.matches_by_project[projects[0].id]]262 assert quests == ["OpenBMB", "Tiny Titan", "Off-Brand", "Sharing is Caring"]263 264 265def test_quest_analysis_validation_accepts_best_prefixed_known_labels() -> None:266 projects = fake_projects(1)267 raw = {268 "projects": [269 {270 "project_id": projects[0].id,271 "matches": [272 {273 "quest": "Best Well-Tuned",274 "confidence": 0.84,275 "evidence": "PEFT adapter",276 "source": "app_file",277 }278 ],279 }280 ]281 }282 283 validated = validate_quest_analysis_payload(raw, projects, source="fake")284 285 assert validated.matches_by_project[projects[0].id][0]["quest"] == "Well-Tuned"286 287 288def test_quest_analysis_validation_accepts_best_use_of_known_labels() -> None:289 projects = fake_projects(1)290 raw = {291 "projects": [292 {293 "project_id": projects[0].id,294 "matches": [295 {296 "quest": "Best Use of Modal",297 "confidence": 0.84,298 "evidence": "modal.App background worker",299 "source": "app_file",300 }301 ],302 }303 ]304 }305 306 validated = validate_quest_analysis_payload(raw, projects, source="fake")307 308 assert validated.matches_by_project[projects[0].id][0]["quest"] == "Modal"309 310 311def test_quest_analysis_validation_accepts_common_compact_aliases() -> None:312 projects = fake_projects(1)313 raw = {314 "projects": [315 {316 "project_id": projects[0].id,317 "matches": [318 {319 "quest": "Offbrand",320 "confidence": 0.84,321 "evidence": "custom frontend",322 "source": "readme",323 },324 {325 "quest": "Offgrid",326 "confidence": 0.82,327 "evidence": "loads weights locally",328 "source": "app_file",329 },330 {331 "quest": "Llama Champion badge",332 "confidence": 0.8,333 "evidence": "llama-cpp-python",334 "source": "app_file",335 },336 {337 "quest": "Modal-first",338 "confidence": 0.79,339 "evidence": "Modal serverless GPUs",340 "source": "readme",341 },342 {343 "quest": "Nemotron-3 Nano 4B",344 "confidence": 0.78,345 "evidence": "Nemotron 3 Nano 4B",346 "source": "readme",347 },348 ],349 }350 ]351 }352 353 validated = validate_quest_analysis_payload(raw, projects, source="fake")354 355 quests = [match["quest"] for match in validated.matches_by_project[projects[0].id]]356 assert quests == ["Off-Brand", "Off the Grid", "Llama Champion", "Modal", "Nemotron"]357 358 359def test_quest_analysis_validation_rejects_unknown_composite_quest_labels() -> None:360 projects = fake_projects(1)361 raw = {362 "projects": [363 {364 "project_id": projects[0].id,365 "matches": [366 {367 "quest": "Mystery Award / Tiny Titan",368 "confidence": 0.84,369 "evidence": "tiny model",370 "source": "app_file",371 }372 ],373 }374 ]375 }376 377 try:378 validate_quest_analysis_payload(raw, projects, source="fake")379 except QuestAnalysisError as error:380 assert "unknown quest in composite" in str(error)381 else:382 raise AssertionError("unknown composite quest labels must be rejected")383 384 385def test_quest_json_extractor_accepts_fenced_object() -> None:386 payload = _extract_json_object('```json\n{"projects":[]}\n```')387 388 assert payload == {"projects": []}389 390 391def test_minicpm_quest_analyzer_attaches_project_id_to_match_payload(monkeypatch) -> None:392 project = fake_projects(1)[0]393 analyzer = MiniCPMQuestAnalyzer()394 monkeypatch.setattr(analyzer, "_ensure_loaded", lambda: None)395 monkeypatch.setattr(396 analyzer,397 "_generate_json",398 lambda _prompt: {399 "matches": [400 {401 "quest": "Off the Grid",402 "confidence": 0.86,403 "evidence": "local model artifact",404 "source": "app_file",405 }406 ]407 },408 )409 410 result = analyzer.analyze([project])411 412 assert set(result) == {project.id}413 assert result[project.id][0]["quest"] == "Off the Grid"414 415 416def test_minicpm_quest_analyzer_uses_metadata_before_model_matches(monkeypatch) -> None:417 project = fake_projects(1)[0]418 project = Project(419 **{420 **project.to_refresh_snapshot_dict(),421 "tags": ["track:wood", "sponsor:openai"],422 "readme_body": "Uses MiniCPM locally.",423 "app_file_source": "model = 'openbmb/MiniCPM5-1B'",424 }425 )426 prompts: list[str] = []427 analyzer = MiniCPMQuestAnalyzer()428 monkeypatch.setattr(analyzer, "_ensure_loaded", lambda: None)429 430 def fake_generate(prompt: str) -> dict:431 prompts.append(prompt)432 return {433 "matches": [434 {435 "quest": "Best Use of Codex",436 "confidence": 0.88,437 "evidence": "Codex helped development",438 "source": "readme",439 },440 {441 "quest": "OpenBMB",442 "confidence": 0.9,443 "evidence": "openbmb/MiniCPM5-1B",444 "source": "app_file",445 },446 ]447 }448 449 monkeypatch.setattr(analyzer, "_generate_json", fake_generate)450 451 result = analyzer.analyze([project])452 453 assert prompts454 assert "- Thousand Token Wood:" not in prompts[0]455 assert "- Backyard AI:" not in prompts[0]456 assert "- Codex:" not in prompts[0]457 assert [match["quest"] for match in result[project.id]] == [458 "Thousand Token Wood",459 "Codex",460 "OpenBMB",461 ]462 assert result[project.id][1]["source"] == "metadata"463 464 465def test_minicpm_quest_analyzer_skips_model_when_metadata_covers_every_profile(monkeypatch) -> None:466 project = fake_projects(1)[0]467 project = Project(468 **{469 **project.to_refresh_snapshot_dict(),470 "tags": [471 "achievement:offgrid",472 "achievement:welltuned",473 "achievement:offbrand",474 "achievement:llama",475 "achievement:sharing",476 "achievement:fieldnotes",477 "track:backyard",478 "track:wood",479 "sponsor:openbmb",480 "sponsor:openai",481 "sponsor:nvidia",482 "sponsor:modal",483 "tiny-titan",484 "best-agent",485 ],486 "readme_body": "All declared in metadata.",487 "app_file_source": "",488 }489 )490 analyzer = MiniCPMQuestAnalyzer()491 492 def fail_load() -> None:493 raise AssertionError("fully declared metadata should not load MiniCPM")494 495 monkeypatch.setattr(analyzer, "_ensure_loaded", fail_load)496 497 result = analyzer.analyze([project])498 499 assert not remaining_project_quest_ids(project)500 assert len(result[project.id]) == 14501 assert {match["source"] for match in result[project.id]} == {"metadata"}502 503 504def test_minicpm_quest_analyzer_repairs_invalid_json_with_base_model(monkeypatch) -> None:505 analyzer = MiniCPMQuestAnalyzer()506 monkeypatch.setattr(analyzer, "_ensure_loaded", lambda: None)507 outputs = [508 # truncated output the deterministic quote-escaper cannot fix -> falls through to base-model repair509 '{"matches":[{"quest":"Off-Brand","confidence":0.8,"evidence":"truncated',510 '{"matches":[{"quest":"Off-Brand","confidence":0.8,"evidence":"custom Server title","source":"app_file"}]}',511 ]512 calls: list[bool] = []513 514 def fake_generate(_system: str, _prompt: str, *, disable_adapter: bool = False) -> str:515 calls.append(disable_adapter)516 return outputs.pop(0)517 518 monkeypatch.setattr(analyzer, "_generate_text", fake_generate)519 520 result = analyzer.analyze([fake_projects(1)[0]])521 522 assert calls == [False, True]523 assert result["build-small-hackathon/project-0"][0]["evidence"] == "custom Server title"524 525 526def test_minicpm_quest_analyzer_escapes_inner_quotes_without_repair(monkeypatch) -> None:527 analyzer = MiniCPMQuestAnalyzer()528 monkeypatch.setattr(analyzer, "_ensure_loaded", lambda: None)529 calls: list[bool] = []530 531 def fake_generate(_system: str, _prompt: str, *, disable_adapter: bool = False) -> str:532 calls.append(disable_adapter)533 return (534 '{"matches":[{"quest":"Off-Brand","confidence":0.8,'535 '"evidence":"app = Server(title="Broken")","source":"app_file"}]}'536 )537 538 monkeypatch.setattr(analyzer, "_generate_text", fake_generate)539 540 result = analyzer.analyze([fake_projects(1)[0]])541 542 assert calls == [False] # deterministic escape; no base-model repair round-trip543 assert result["build-small-hackathon/project-0"][0]["evidence"] == 'app = Server(title="Broken")'544 545 546def test_minicpm_quest_analyzer_tolerates_unparseable_project(monkeypatch) -> None:547 analyzer = MiniCPMQuestAnalyzer()548 monkeypatch.setattr(analyzer, "_ensure_loaded", lambda: None)549 550 def fail(_prompt: str) -> dict:551 raise QuestAnalysisError("quest analyzer returned invalid JSON")552 553 monkeypatch.setattr(analyzer, "_generate_json", fail)554 555 result = analyzer.analyze([fake_projects(1)[0]])556 557 assert result == {"build-small-hackathon/project-0": []}558 559 560def test_minicpm_quest_analyzer_repairs_schema_errors_with_base_model(monkeypatch) -> None:561 project = fake_projects(1)[0]562 analyzer = MiniCPMQuestAnalyzer()563 monkeypatch.setattr(analyzer, "_ensure_loaded", lambda: None)564 monkeypatch.setattr(565 analyzer,566 "_generate_json",567 lambda _prompt: {568 "matches": [569 {"quest": "Off the Grid", "confidence": 0.4, "evidence": "local model", "source": "readme"},570 {"quest": "Off the Grid", "confidence": 0.8, "evidence": "no cloud API", "source": "app_file"},571 ]572 },573 )574 repair_errors: list[str] = []575 576 def fake_repair(_raw, error: str):577 repair_errors.append(error)578 return {579 "matches": [580 {"quest": "Off the Grid", "confidence": 0.8, "evidence": "no cloud API", "source": "app_file"}581 ]582 }583 584 monkeypatch.setattr(analyzer, "_repair_schema_json", fake_repair)585 586 result = analyzer.analyze([project])587 588 assert "duplicate quest" in repair_errors[0]589 assert result[project.id] == [590 {"quest": "Off the Grid", "confidence": 0.8, "evidence": "no cloud API", "source": "app_file"}591 ]592 593 594def test_quest_prompt_uses_raw_readme_and_app_source_segments() -> None:595 project = Project(596 id="build-small-hackathon/two-segment",597 title="Two Segment",598 summary="card summary should not drive quest analysis",599 tags=("gradio", "region:us"),600 models=("openbmb/MiniCPM5-1B",),601 datasets=(),602 likes=1,603 sdk="gradio",604 license="mit",605 created_at="2026-06-01T00:00:00+00:00",606 last_modified="2026-06-08T00:00:00+00:00",607 host="https://two-segment.hf.space",608 url="https://huggingface.co/spaces/build-small-hackathon/two-segment",609 app_file="app.py",610 app_file_embedding_text="compact app signals should not drive quest analysis",611 readme_body="README evidence: field notes and a tiny OpenBMB model.",612 app_file_source="from llama_cpp import Llama\nmodel = 'openbmb/MiniCPM5-1B'",613 )614 615 prompt = render_project_quest_prompt(project)616 617 assert "[README]" in prompt618 assert "README evidence: field notes" in prompt619 assert "[APP_FILE] app.py" in prompt620 assert "from llama_cpp import Llama" in prompt621 assert "card summary should not drive quest analysis" not in prompt622 assert "compact app signals should not drive quest analysis" not in prompt623 assert "region:us" not in prompt624 625 626def test_quest_analyzer_rejects_non_minicpm_backend(monkeypatch) -> None:627 monkeypatch.setenv("ADVISOR_QUEST_ANALYZER_BACKEND", "rules")628 629 try:630 create_quest_analyzer(device="local")631 except QuestAnalysisError as error:632 assert "minicpm-transformers" in str(error)633 else:634 raise AssertionError("dashboard refresh should only accept MiniCPM quest analysis")635 636 637def fake_index() -> ProjectIndex:638 projects = fake_projects(12)639 embeddings = []640 for index, _project in enumerate(projects):641 vector = [0.0] * 16642 vector[index % 3] = 3.0643 vector[3 + index % 5] = 1.5644 vector[8 + index % 8] = 0.7645 embeddings.append(vector)646 snapshot_generated_at = "2026-06-08T00:00:00+00:00"647 source = "https://example.test/spaces"648 payload = build_index_payload(projects, snapshot_generated_at, source, embeddings)649 return ProjectIndex(650 projects=projects,651 generated_at=snapshot_generated_at,652 source=source,653 index_payload=payload,654 )655 656 657def noisy_cluster_label_index() -> ProjectIndex:658 themes = [659 ("dream", ("Dream Lantern", "Dream Atlas"), "dream journal symbolic oracle"),660 ("family", ("Family Ledger", "Care Kinship"), "family care bill coordination"),661 ("garden", ("Garden Notebook", "Seed Exchange"), "garden seed neighborhood plants"),662 ("notice", ("Notice Helper", "Scam Screen"), "notice scam safety verification"),663 ("order", ("Order Desk", "Inventory Voice"), "order inventory audio assistant"),664 ("repair", ("Repair Coach", "Tool Shed"), "repair maintenance workshop"),665 ]666 projects: list[Project] = []667 embeddings = []668 for theme_index, (theme, titles, summary) in enumerate(themes):669 for title in titles:670 projects.append(671 Project(672 id=f"build-small-hackathon/{title.lower().replace(' ', '-')}",673 title=title,674 summary=(675 f"{summary} for a build-small-hackathon AI project in the US region "676 "with a Gradio demo."677 ),678 tags=("build-small-hackathon", "ai", "gradio", "region:us", theme),679 models=("tiny-model",),680 datasets=(),681 likes=theme_index,682 sdk="gradio",683 license="mit",684 created_at="2026-06-01T00:00:00+00:00",685 last_modified=f"2026-06-{theme_index + 1:02d}T00:00:00+00:00",686 host=f"https://{title.lower().replace(' ', '-')}.hf.space",687 url=f"https://huggingface.co/spaces/build-small-hackathon/{title.lower().replace(' ', '-')}",688 app_file="app.py",689 app_file_embedding_text="shared local small model app",690 )691 )692 vector = [0.0] * len(themes)693 vector[theme_index] = 1.0694 embeddings.append(vector)695 696 snapshot_generated_at = "2026-06-08T00:00:00+00:00"697 source = "https://example.test/spaces"698 payload = build_index_payload(projects, snapshot_generated_at, source, embeddings)699 return ProjectIndex(700 projects=projects,701 generated_at=snapshot_generated_at,702 source=source,703 index_payload=payload,704 )705 706 707def fake_projects(count: int) -> list[Project]:708 return [709 Project(710 id=f"build-small-hackathon/project-{index}",711 title=f"Project {index}",712 summary=f"Offline project planner {index}",713 tags=("gradio", "local-first"),714 models=("tiny-model",),715 datasets=(),716 likes=index % 5,717 sdk="gradio",718 license="mit",719 created_at="2026-06-01T00:00:00+00:00",720 last_modified=f"2026-06-{index + 1:02d}T00:00:00+00:00",721 host=f"https://project-{index}.hf.space",722 url=f"https://huggingface.co/spaces/build-small-hackathon/project-{index}",723 app_file="app.py",724 app_file_embedding_text="local inference gradio small model artifact",725 )726 for index in range(count)727 ]728