build-small-hackathon/hackathon-advisor
16
1from hackathon_advisor.dashboard_chat import DashboardChatEngine2from tests.test_dashboard_repository import analyzed_repository, not_analyzed_repository3 4 5class ScriptedRunner:6 """ChatRunner double that replays prepared model outputs and records calls."""7 8 backend = "scripted"9 model_id = "scripted-test-model"10 supports_thinking = False11 12 def __init__(self, outputs: list[object]) -> None:13 self.outputs = list(outputs)14 self.calls: list[dict] = []15 16 def stream(self, messages, *, tools=None, max_new_tokens, enable_thinking=False):17 self.calls.append(18 {19 "messages": messages,20 "tools": tools,21 "max_new_tokens": max_new_tokens,22 "enable_thinking": enable_thinking,23 }24 )25 output = self.outputs.pop(0)26 pieces = output if isinstance(output, list) else [output]27 for count, piece in enumerate(pieces, start=1):28 yield count, piece29 30 31class ThinkingScriptedRunner(ScriptedRunner):32 """ScriptedRunner whose outputs start inside a <think> block (native thinking)."""33 34 supports_thinking = True35 36 37def run_turn(runner, repository, message, history=None):38 engine = DashboardChatEngine(runner, lambda: repository)39 return list(engine.turn_stream(message, history))40 41 42def events_of(events, event_type):43 return [event for event in events if event["type"] == event_type]44 45 46def test_tool_turn_streams_verified_data_before_prose() -> None:47 runner = ScriptedRunner(48 [49 '<function name="top_projects_by_quests"></function>',50 "Project 9 leads with two quests.",51 ]52 )53 54 events = run_turn(runner, analyzed_repository(), "who completed the most quests?")55 56 tool_call = events_of(events, "tool_call")[0]57 assert tool_call["status"] == "valid"58 assert tool_call["name"] == "top_projects_by_quests"59 60 tool_result = events_of(events, "tool_result")[0]61 assert tool_result["data"]["rows"][0]["id"] == "build-small-hackathon/project-9"62 assert tool_result["map_action"]["type"] == "highlight_projects"63 64 types = [event["type"] for event in events]65 assert types.index("tool_result") < types.index("token")66 67 done = events_of(events, "done")[0]68 assert done["response"] == "Project 9 leads with two quests."69 assert done["tool"] == "top_projects_by_quests"70 assert done["history"][-2:] == [71 {"role": "user", "content": "who completed the most quests?"},72 {"role": "assistant", "content": "Project 9 leads with two quests."},73 ]74 75 76def test_model_digest_strips_urls_and_ids() -> None:77 runner = ScriptedRunner(78 [79 '<function name="top_projects_by_quests"></function>',80 "Project 9 leads.",81 ]82 )83 84 run_turn(runner, analyzed_repository(), "quest leaderboard")85 86 answer_call = runner.calls[1]87 assert answer_call["tools"] is None88 digest = answer_call["messages"][-1]["content"]89 assert "url" not in digest90 assert "huggingface.co" not in digest91 assert 'title: "Project 9"' in digest92 assert "quest_count: 2" in digest93 94 95def test_model_digest_trims_long_listings_and_bm25_total() -> None:96 repository = analyzed_repository()97 runner = ScriptedRunner(98 ['<function name="search_projects"><param name="query">project</param></function>', "Found a few."]99 )100 run_turn(runner, repository, "find project planners")101 search_digest = runner.calls[1]["messages"][-1]["content"]102 assert "total:" not in search_digest103 104 runner = ScriptedRunner(['<function name="list_clusters"></function>', "A few clusters."])105 run_turn(runner, repository, "what clusters exist")106 cluster_digest = runner.calls[1]["messages"][-1]["content"]107 # Only the largest cluster is restatable; the full list lives on the UI cards.108 assert cluster_digest.count("label:") == 1109 assert f"cluster_count: {repository.list_clusters()['cluster_count']}" in cluster_digest110 111 112def test_not_analyzed_quests_skip_the_answer_pass() -> None:113 runner = ScriptedRunner(['<function name="top_projects_by_quests"></function>'])114 115 events = run_turn(runner, not_analyzed_repository(), "who completed the most quests?")116 117 skipped = events_of(events, "answer_skipped")[0]118 assert skipped["reason"] == "quests_not_analyzed"119 assert events_of(events, "token") == []120 assert len(runner.calls) == 1 # pass 2 never ran121 assert events_of(events, "done")[0]["response"] == skipped["text"]122 123 124def test_empty_search_skips_the_answer_pass() -> None:125 runner = ScriptedRunner(126 ['<function name="search_projects"><param name="query">zzzz qqqq</param></function>']127 )128 129 events = run_turn(runner, analyzed_repository(), "find zzzz qqqq")130 131 skipped = events_of(events, "answer_skipped")[0]132 assert skipped["reason"] == "no_search_results"133 assert "zzzz qqqq" in skipped["text"]134 assert len(runner.calls) == 1135 136 137def test_unknown_cluster_uses_templated_sentence() -> None:138 runner = ScriptedRunner(139 ['<function name="show_cluster"><param name="label">flying castles</param></function>']140 )141 142 events = run_turn(runner, analyzed_repository(), "show me the flying castles cluster")143 144 skipped = events_of(events, "answer_skipped")[0]145 assert skipped["reason"] == "unknown_cluster"146 assert "flying castles" in skipped["text"]147 assert events_of(events, "tool_result")[0]["map_action"] is None148 149 150def test_show_cluster_emits_filter_map_action() -> None:151 repository = analyzed_repository()152 label = repository.list_clusters()["clusters"][0]["label"]153 runner = ScriptedRunner(154 [155 f'<function name="show_cluster"><param name="label">{label}</param></function>',156 "That cluster groups the local-first planners.",157 ]158 )159 160 events = run_turn(runner, repository, f"what is in {label}?")161 162 map_action = events_of(events, "tool_result")[0]["map_action"]163 assert map_action == {"type": "filter_cluster", "label": label}164 165 166def test_show_cluster_map_action_carries_canonical_label_for_fuzzy_input() -> None:167 """The UI resolves filter_cluster by exact label match, so the engine must168 forward the repository's canonical label, never the user's fuzzy input."""169 repository = analyzed_repository()170 canonical = repository.list_clusters()["clusters"][0]["label"]171 fuzzy = canonical.split()[0].lower()172 assert fuzzy != canonical173 runner = ScriptedRunner(174 [175 f'<function name="show_cluster"><param name="label">{fuzzy}</param></function>',176 "Here is that cluster.",177 ]178 )179 180 events = run_turn(runner, repository, f"what is in the {fuzzy} cluster?")181 182 map_action = events_of(events, "tool_result")[0]["map_action"]183 assert map_action == {"type": "filter_cluster", "label": canonical}184 185 186def test_plain_prose_routes_to_dedicated_smalltalk_pass() -> None:187 runner = ScriptedRunner(188 [189 "Hello! Happy to help.",190 "Hi! Ask me what everyone is building.",191 ]192 )193 194 events = run_turn(runner, analyzed_repository(), "hello there")195 196 assert events_of(events, "tool_call")[0]["status"] == "none"197 assert events_of(events, "tool_result") == []198 done = events_of(events, "done")[0]199 assert done["tool"] == ""200 assert done["response"] == "Hi! Ask me what everyone is building."201 assert runner.calls[1]["tools"] is None202 203 204def test_unmatched_data_question_defaults_to_search_not_smalltalk() -> None:205 """Regression: 'how many voice apps' once slipped into ungrounded small talk."""206 runner = ScriptedRunner(207 [208 "I'm not sure, but I can provide more information if you'd like.",209 "The closest matches are shown on the cards.",210 ]211 )212 213 events = run_turn(runner, analyzed_repository(), "how many voice apps")214 215 tool_call = events_of(events, "tool_call")[0]216 assert tool_call["status"] == "defaulted"217 assert tool_call["name"] == "search_projects"218 assert tool_call["arguments"]["query"] == "how many voice apps"219 220 221def test_short_followup_stays_on_smalltalk_path() -> None:222 runner = ScriptedRunner(223 [224 "Because it scored well.",225 "I should look that up rather than guess - ask me about the quest leaderboard!",226 ]227 )228 229 events = run_turn(runner, analyzed_repository(), "are you sure?")230 231 assert events_of(events, "tool_call")[0]["status"] == "none"232 assert events_of(events, "tool_result") == []233 234 235def test_show_project_streams_readme_and_highlights_the_dot() -> None:236 runner = ScriptedRunner(237 [238 '<function name="show_project"><param name="project">Project 4</param></function>',239 "Project 4 is an offline planner; its app file loads gradio.",240 ]241 )242 243 events = run_turn(runner, analyzed_repository(), "how does Project 4 work?")244 245 tool_result = events_of(events, "tool_result")[0]246 assert tool_result["tool"] == "show_project"247 assert "README evidence for project 4" in tool_result["data"]["readme_excerpt"]248 assert tool_result["map_action"] == {249 "type": "highlight_projects",250 "ids": ["build-small-hackathon/project-4"],251 }252 digest = runner.calls[1]["messages"][-1]["content"]253 assert "README evidence for project 4" in digest254 assert "import gradio" in digest255 assert events_of(events, "done")[0]["tool"] == "show_project"256 257 258def test_show_project_falls_back_to_search_for_unknown_names() -> None:259 runner = ScriptedRunner(260 [261 '<function name="show_project"><param name="project">offline planner thing</param></function>',262 "The closest matches are on the cards.",263 ]264 )265 266 events = run_turn(runner, analyzed_repository(), "tell me about the offline planner thing")267 268 tool_result = events_of(events, "tool_result")[0]269 assert tool_result["tool"] == "search_projects"270 assert tool_result["data"]["results"]271 assert events_of(events, "done")[0]["tool"] == "search_projects"272 273 274def test_prose_answer_to_data_question_is_routed_by_intent() -> None:275 runner = ScriptedRunner(276 [277 "I do not have access to quest completion data.",278 "Project 9 leads with two quests.",279 ]280 )281 282 events = run_turn(runner, analyzed_repository(), "who completed the most quests?")283 284 tool_call = events_of(events, "tool_call")[0]285 assert tool_call["status"] == "defaulted"286 assert tool_call["name"] == "top_projects_by_quests"287 assert events_of(events, "tool_result")[0]["data"]["rows"]288 assert events_of(events, "done")[0]["response"] == "Project 9 leads with two quests."289 290 291def test_malformed_call_degrades_through_intent_router() -> None:292 runner = ScriptedRunner(293 [294 '<function name="nonexistent_tool"></function>',295 "Project 9 leads the quest board.",296 ]297 )298 299 events = run_turn(runner, analyzed_repository(), "who completed the most quests")300 301 tool_call = events_of(events, "tool_call")[0]302 assert tool_call["status"] == "defaulted"303 assert tool_call["name"] == "top_projects_by_quests"304 assert tool_call["errors"]305 306 307def test_stray_function_block_is_cut_from_the_answer() -> None:308 runner = ScriptedRunner(309 [310 '<function name="atlas_overview"></function>',311 ["The field has ten projects. ", '<function name="x">', "</function> extra"],312 ]313 )314 315 events = run_turn(runner, analyzed_repository(), "overview please")316 317 done = events_of(events, "done")[0]318 assert done["response"] == "The field has ten projects."319 for token in events_of(events, "token"):320 assert "<function" not in token["text"]321 322 323def test_thinking_trace_streams_separately_from_the_tool_call() -> None:324 runner = ThinkingScriptedRunner(325 [326 [327 "The user wants the leaderboard. I could emit ",328 '<function name="x"> here but the right tool is top_projects_by_quests.',329 "</th",330 'ink>\n\n<function name="top_projects_by_quests"></function>',331 ],332 ["Let me restate the digest.</think>\n\nProject 9 leads with two quests."],333 ]334 )335 336 events = run_turn(runner, analyzed_repository(), "who completed the most quests?")337 338 thinking_pass1 = [e for e in events if e["type"] == "thinking" and e["pass"] == 1]339 assert "".join(e["text"] for e in thinking_pass1).startswith("The user wants the leaderboard")340 # <function inside the REASONING must not be mistaken for the call itself.341 tool_call = events_of(events, "tool_call")[0]342 assert tool_call["status"] == "valid"343 assert tool_call["name"] == "top_projects_by_quests"344 345 thinking_pass2 = [e for e in events if e["type"] == "thinking" and e["pass"] == 2]346 assert "".join(e["text"] for e in thinking_pass2) == "Let me restate the digest."347 done = events_of(events, "done")[0]348 assert done["response"] == "Project 9 leads with two quests."349 for token in events_of(events, "token"):350 assert "</think>" not in token["text"]351 # Thinking never leaks into the durable history.352 assert all("Let me restate" not in entry["content"] for entry in done["history"])353 354 355def test_thinking_marker_split_across_pieces_is_handled() -> None:356 runner = ThinkingScriptedRunner(357 [358 ["step one ", "step two</t", "hink>", '\n\n<function name="list_quests"></function>'],359 ["ok</think>\n\nThe quests are on the cards."],360 ]361 )362 363 events = run_turn(runner, analyzed_repository(), "list the quests")364 365 thinking = "".join(e["text"] for e in events if e["type"] == "thinking" and e["pass"] == 1)366 assert thinking == "step one step two"367 assert events_of(events, "tool_call")[0]["name"] == "list_quests"368 369 370def test_truncated_thinking_degrades_gracefully() -> None:371 """A 4096-token cut inside <think> leaves no answer text; the turn must still372 resolve (intent backstop on pass 1, templated sentence on pass 2)."""373 runner = ThinkingScriptedRunner(374 [375 ["I am still reasoning about which tool to"], # no </think>, no call376 ["and the digest says</think>\n\nProject 9 leads."],377 ]378 )379 380 events = run_turn(runner, analyzed_repository(), "who completed the most quests?")381 382 tool_call = events_of(events, "tool_call")[0]383 assert tool_call["status"] == "defaulted"384 assert tool_call["name"] == "top_projects_by_quests"385 assert events_of(events, "done")[0]["response"] == "Project 9 leads."386 387 388def test_chat_generations_use_thinking_and_4096_budget() -> None:389 runner = ScriptedRunner(390 [391 '<function name="atlas_overview"></function>',392 "Ten projects in view.",393 ]394 )395 396 events = run_turn(runner, analyzed_repository(), "overview")397 398 for call in runner.calls:399 assert call["enable_thinking"] is True400 assert call["max_new_tokens"] >= 4096401 for progress in events_of(events, "model_progress"):402 assert progress["max_tokens"] >= 4096403 404 405def test_history_drops_repeated_assistant_answers() -> None:406 """Regression: a greedy 1B echoes any sentence that appears twice in history."""407 runner = ScriptedRunner(408 [409 '<function name="atlas_overview"></function>',410 "Ten projects in view.",411 ]412 )413 looping_history = [414 {"role": "user", "content": "why"},415 {"role": "assistant", "content": "I should look that up."},416 {"role": "user", "content": "are you sure?"},417 {"role": "assistant", "content": "I should look that up."},418 {"role": "user", "content": "really?"},419 {"role": "assistant", "content": "I should look that up."},420 ]421 422 run_turn(runner, analyzed_repository(), "overview", looping_history)423 424 pass1_messages = runner.calls[0]["messages"]425 repeated = [m for m in pass1_messages if m.get("content") == "I should look that up."]426 assert len(repeated) == 1427 428 429def test_grounded_answer_pass_sees_no_history() -> None:430 """Echoes regression: with history in the prompt, a greedy 1B repeats prior431 answers instead of reading the digest. Facts come from the digest alone."""432 runner = ScriptedRunner(433 [434 '<function name="atlas_overview"></function>',435 "Ten projects in view.",436 ]437 )438 long_history = []439 for index in range(6):440 long_history.append({"role": "user", "content": f"question {index}"})441 long_history.append({"role": "assistant", "content": f"answer {index}"})442 443 run_turn(runner, analyzed_repository(), "overview", long_history)444 445 answer_messages = runner.calls[1]["messages"]446 roles = [m["role"] for m in answer_messages]447 assert roles == ["system", "user", "assistant", "tool"]448 449 450def test_overview_digest_leads_with_most_liked_projects() -> None:451 runner = ScriptedRunner(452 [453 '<function name="atlas_overview"></function>',454 "Project 9 is the most liked.",455 ]456 )457 458 run_turn(runner, analyzed_repository(), "what is the coolest project?")459 460 digest = runner.calls[1]["messages"][-1]["content"]461 assert digest.startswith("most_liked_projects:")462 assert "most_completed_quests:" in digest463 464 465def test_history_is_normalized_and_capped() -> None:466 runner = ScriptedRunner(467 [468 '<function name="atlas_overview"></function>',469 "Ten projects in view.",470 ]471 )472 junk_history = [473 {"role": "user", "content": "old question"},474 {"role": "assistant", "content": "old answer"},475 {"role": "tool", "content": "should be dropped"},476 "not even a dict",477 {"role": "user", "content": ""},478 ]479 480 events = run_turn(runner, analyzed_repository(), "overview", junk_history)481 482 pass1_messages = runner.calls[0]["messages"]483 roles = [message["role"] for message in pass1_messages]484 assert roles == ["system", "user", "assistant", "user"]485 done = events_of(events, "done")[0]486 assert all(entry["role"] in ("user", "assistant") for entry in done["history"])487 assert len(done["history"]) <= 12488