pbhappliedsystems/quant-eval-agent-arena
1
1# eval_data.py2# PBH Applied Systems — quant_eval v7.21 scores and model metadata.3# Every value in this file is sourced directly from the published HF model cards.4# No values are assumed, estimated, or back-calculated.5#6# Aggregate dimension scores (Task Completion, Reasoning, Coherence, Instruction Following)7# are only available for models evaluated with both F16 and Q4_K_M runners.8# Qwen2.5-32B and Qwen3.6-27B were evaluated Q4_K_M only (F16 exceeds RTX 4090 VRAM).9# Those two models have per-family pass rates only — aggregate scores are None by design.10 11# ---------------------------------------------------------------------------12# Score dimension descriptions13# ---------------------------------------------------------------------------14 15DIMENSION_DESCRIPTIONS = {16 "task_completion": (17 "Measures whether the model completed the assigned task end-to-end. "18 "Evaluated across structured output, tool dispatch, and multi-step "19 "planning families. Score reflects pass rate weighted by task difficulty."20 ),21 "reasoning": (22 "Measures coherent, multi-step logical inference. Derived from "23 "json_multistep, stateful_followup, and fuzz family outcomes. "24 "High scores indicate reliable chain-of-thought under production conditions."25 ),26 "coherence": (27 "Measures output structural integrity and internal consistency across "28 "turns and task types. A low coherence score signals format instability "29 "or EOS/token contamination issues."30 ),31 "instruction_following": (32 "Measures schema compliance, constraint adherence, and output format "33 "fidelity. Evaluated across all 8 fixture families. Critical for "34 "agentic pipelines that depend on structured model output."35 ),36}37 38# ---------------------------------------------------------------------------39# Per-family fixture descriptions40# ---------------------------------------------------------------------------41 42FAMILY_DESCRIPTIONS = {43 "json_multistep": (44 "Multi-step planning with self-check and oracle verification. "45 "Hardest family — all four signals must pass: schema_ok, "46 "checks_consistent_ok, stop_semantics_ok, oracle_equiv_ok."47 ),48 "stateful_followup": (49 "Two-turn state tracking. Turn 2 only evaluated given correct Turn 1. "50 "Tests multi-turn memory under production conditions."51 ),52 "toolcall_only": (53 "Bare schema-only tool call: strict tool name + args check. "54 "No prose, no explanation — just schema-valid JSON. "55 "Where quantization most commonly degrades structured dispatch."56 ),57 "mixed_brief_json": (58 "Hybrid output: natural language answer + valid JSON block in same response. "59 "Both parts must be present and correct simultaneously."60 ),61 "toolcall": (62 "Tool call embedded in a broader response. More forgiving than toolcall_only. "63 "Tests inline tool dispatch with surrounding context."64 ),65 "json": (66 "Single-step structured JSON with constraint rules. "67 "Bucket-scored — max bucket = 10.0."68 ),69 "fuzz": (70 "Property-based regression across structured placement correctness. "71 "20 cases per model. Bucket-scored. Detects inconsistencies under input variation."72 ),73 "mcq": (74 "Multiple-choice extraction with exact answer signal. "75 "Bucket-scored. A-bias is a known characteristic in some models."76 ),77}78 79# ---------------------------------------------------------------------------80# Model registry — Q4_K_M variants only.81# All scores normalized [0.0 – 1.0]. Higher is better.82# Scores are None for single-runner models (no F16 baseline available).83# vram_gb: from model card Key Characteristics.84# ---------------------------------------------------------------------------85 86MODELS = {87 "qwen2.5-3b": {88 "display_name": "Qwen2.5-3B-Instruct Q4_K_M",89 "short_name": "Qwen2.5-3B",90 "family": "Qwen2.5",91 "params": "3B",92 "context_window": 32768,93 "file_size_gb": 1.93,94 "vram_gb": 4.0,95 "avg_inference_sec": 0.390,96 "hf_repo": "pbhappliedsystems/qwen-2.5-3B-instruct-gguf-Q4-K-M",97 "hf_filename": "qwen-2.5-3B-instruct-gguf-Q4-K-M.gguf",98 "sha256": "9ab3bc9beaddaec3700d5cc754b52e1501a3fd172bc7fc3ee3eb8e1d388ee043",99 "run_id": "20260221_041137",100 "license": "Qwen Research License (non-commercial)",101 "solo_only": False,102 "thinking_mode": False,103 "known_issues": [104 "A-bias on MCQ: mcq_02 and mcq_05 both produce 'A' (wrong). "105 "Add CoT prompting for MCQ pipelines.",106 "json_multistep: 0.200 pass rate — checks_consistent_ok fails on all "107 "cases except ms_easy_01.",108 ],109 "scores": {110 "task_completion": 0.4905,111 "reasoning": 0.3704,112 "coherence": 0.9074,113 "instruction_following": 0.6599,114 },115 "series_notes": (116 "Smallest and fastest model in series (0.390 sec/case, 1.93 GB). "117 "Runs on 4 GB VRAM or CPU. Strong coherence relative to size. "118 "Reasoning is weakest in the evaluated series."119 ),120 },121 122 "qwen2.5-7b": {123 "display_name": "Qwen2.5-7B-Instruct Q4_K_M",124 "short_name": "Qwen2.5-7B",125 "family": "Qwen2.5",126 "params": "7B",127 "context_window": 32768,128 "file_size_gb": 4.68,129 "vram_gb": 6.0,130 "avg_inference_sec": 0.554,131 "hf_repo": "pbhappliedsystems/qwen-2.5-7B-instruct-gguf-Q4-K-M",132 "hf_filename": "qwen-2.5-7B-instruct-gguf-Q4-K-M.gguf",133 "sha256": "863656d217841f5d3fb180d9dca4e4bbdaa071bde25885fa0d27fe7188a2cc85",134 "run_id": "20260221_024911",135 "license": "Qwen Research License (non-commercial)",136 "solo_only": False,137 "thinking_mode": False,138 "known_issues": [139 "toolcall_only: 0/2 pass — wrong schema key names "140 "('numbers' array instead of 'args' object).",141 "EOS token contamination on toolcall final answers — "142 "strip <|im_end|> before downstream processing.",143 ],144 "scores": {145 "task_completion": 0.6214,146 "reasoning": 0.9444,147 "coherence": 0.9021,148 "instruction_following": 0.8775,149 },150 "series_notes": (151 "Major capability step over 3B: reasoning +0.574. "152 "checks_consistent_ok goes 0.200 → 1.000. "153 "Fastest non-3B model at 0.554 sec/case."154 ),155 },156 157 "qwen2.5-14b-1m": {158 "display_name": "Qwen2.5-14B-Instruct-1M Q4_K_M",159 "short_name": "Qwen2.5-14B-1M",160 "family": "Qwen2.5",161 "params": "14B",162 "context_window": 1_000_000,163 "file_size_gb": 8.99,164 "vram_gb": 12.0,165 "avg_inference_sec": 2.683,166 "hf_repo": "pbhappliedsystems/qwen-2.5-14B-instruct-1m-gguf-Q4-K-M",167 "hf_filename": "qwen-2.5-14B-instruct-1m-gguf-Q4-K-M.gguf",168 "sha256": "5ad529ff2b1b192f31c8a638fe8756a0c628904e2ded797c11f9194216976973",169 "run_id": "20260210_235131",170 "license": "Apache 2.0",171 "solo_only": False,172 "thinking_mode": False,173 "known_issues": [174 "toolcall_only: args_ok=0.000 — 'input'/{x,y} wrapper instead of 'args'/{a,b}. "175 "Specify exact key names in system prompt.",176 "EOS token contamination on toolcall final answers — "177 "strip <|im_end|> before downstream processing.",178 ],179 "scores": {180 "task_completion": 0.6857,181 "reasoning": 0.9907, # #1 in series182 "coherence": 0.9259,183 "instruction_following": 0.9902, # #1 in series184 },185 "series_notes": (186 "#1 reasoning and #1 instruction-following in the evaluated series. "187 "Zero quantization degradation across all behavioral families — "188 "F16 and Q4_K_M produce identical pass rates on every fixture. "189 "1M context window. For deployment: set n_ctx to actual context needed; "190 "full 1M context requires ~80 GB VRAM."191 ),192 },193 194 "qwen2.5-32b": {195 "display_name": "Qwen2.5-32B-Instruct Q4_K_M",196 "short_name": "Qwen2.5-32B",197 "family": "Qwen2.5",198 "params": "32B",199 "context_window": 32768,200 "file_size_gb": 19.9,201 "vram_gb": 24.0,202 "avg_inference_sec": 9.282,203 "hf_repo": "pbhappliedsystems/qwen-2.5-32B-instruct-gguf-Q4-K-M",204 "hf_filename": "qwen-2.5-32B-instruct-gguf-Q4-K-M.gguf",205 "sha256": "6f810a332a884410aa65cc1b5a128a8603f083b36465acfbbf67a08f50a4d3e3",206 "run_id": "20260221_144732",207 "license": "Apache 2.0",208 "solo_only": False, # H200 141GB VRAM — all pairs feasible209 "thinking_mode": False,210 "known_issues": [211 "json_multistep: 0.600 pass rate — counterintuitively underperforms 7B and 14B-1M. "212 "ms_hard_01 fails with checks_consistent_ok=0 and oracle_equiv_ok=0.",213 "toolcall_only: args_ok=0.000 — uses 'params'/{a,b} instead of 'args'/{a,b}. "214 "Arg value names are correct; only outer wrapper key fails. "215 "Fixable with explicit key-name system prompt.",216 "EOS token contamination on toolcall final answers — "217 "strip <|im_end|> before downstream processing.",218 ],219 "scores": {220 # Single-runner evaluation: F16 GGUF (65.5 GB) exceeds RTX 4090 VRAM.221 # Aggregate dimension scores are not computed without an F16 baseline.222 # Per-family pass rates are published on the model card.223 "task_completion": None,224 "reasoning": None,225 "coherence": None,226 "instruction_following": None,227 },228 "series_notes": (229 "Largest evaluated model (19.9 GB, ~24 GB VRAM). "230 "Single-runner evaluation — no F16 baseline possible at this file size. "231 "Counterintuitively underperforms 7B and 14B-1M on json_multistep. "232 "MCQ: 5/5 perfect. stateful_followup: 1.000. mixed_brief_json: 1.000."233 ),234 },235 236 "ministral-14b-instruct": {237 "display_name": "Ministral-3-14B-Instruct-2512 Q4_K_M",238 "short_name": "Ministral-14B",239 "family": "Ministral",240 "params": "14B",241 "context_window": 32768,242 "file_size_gb": 8.24,243 "vram_gb": 11.0,244 "avg_inference_sec": 3.77,245 "hf_repo": "pbhappliedsystems/ministral-3-14b-instruct-2512-gguf-Q4-K-M",246 "hf_filename": "ministral-3-14b-instruct-2512-gguf-Q4-K-M.gguf",247 "sha256": "a23910514ee512aa28db8dddd390c26a73b9c318dcdec374ae02d722d9658749",248 "run_id": "20260209_170235",249 "license": "Apache 2.0",250 "solo_only": False,251 "thinking_mode": False,252 "known_issues": [253 "toolcall_only: F16=1.000 → Q4_K_M=0.000. Complete degradation on bare "254 "tool-call schema under quantization. Do not deploy in bare tool-call "255 "pipelines without schema enforcement.",256 ],257 "scores": {258 "task_completion": 0.6809,259 "reasoning": 0.9148,260 "coherence": 0.9259,261 "instruction_following": 0.9689,262 },263 "series_notes": (264 "Strong all-around scores. Critical finding: toolcall_only drops from "265 "1.000 (F16) to 0.000 (Q4_K_M) — the most severe quantization degradation "266 "event in the evaluated series on that family."267 ),268 },269 270 "ministral-14b-reasoning": {271 "display_name": "Ministral-3-14B-Reasoning-2512 Q4_K_M",272 "short_name": "Ministral-14B-R",273 "family": "Ministral",274 "params": "14B",275 "context_window": 32768,276 "file_size_gb": 8.24,277 "vram_gb": 11.0,278 "avg_inference_sec": 1.18,279 "hf_repo": "pbhappliedsystems/ministral-3-14b-reasoning-2512-gguf-Q4-K-M",280 "hf_filename": "ministral-3-14b-reasoning-2512-gguf-Q4-K-M.gguf",281 "sha256": "e7171d96748ddc948fd6d9edb3d1c6e3f9ba6b855ff964aee98519788da330c2",282 "run_id": "20260209_233252",283 "license": "Apache 2.0",284 "solo_only": False,285 "thinking_mode": False,286 "known_issues": [287 "Q4_K_M compresses chain-of-thought from F16's 65.67 sec/case to 1.18 sec/case "288 "(55.7x faster). Whether this is a feature or regression depends on use case.",289 "mcq_02: F16 fails due to markdown fence wrapping; Q4_K_M suppresses fencing "290 "but selects wrong answer.",291 ],292 "scores": {293 "task_completion": 0.6786,294 "reasoning": 0.9389,295 "coherence": 0.9259,296 "instruction_following": 0.9649,297 },298 "series_notes": (299 "Fastest non-3B model in the series at 1.18 sec/case Q4_K_M. "300 "Quantization dramatically compresses the reasoning chain vs F16 (65.67 sec). "301 "Use when speed matters and abbreviated reasoning is acceptable."302 ),303 },304 305 "phi4-reasoning-plus": {306 "display_name": "Phi-4-reasoning-plus Q4_K_M",307 "short_name": "Phi-4-R+",308 "family": "Phi-4",309 "params": "14B",310 "context_window": 16384,311 "file_size_gb": 9.05,312 "vram_gb": 12.0,313 "avg_inference_sec": 25.84,314 "hf_repo": "pbhappliedsystems/phi-4-reasoning-plus-gguf-Q4-K-M",315 "hf_filename": "phi-4-reasoning-plus-gguf-Q4-K-M.gguf",316 "sha256": "2fe74424b03433d11ccf3f2ce8da404810fa7eb9a269135b1f14bf0d88566e4d",317 "run_id": "20260222_170914",318 "license": "MIT",319 "solo_only": False,320 "thinking_mode": False,321 "known_issues": [322 "Systematic EOS token contamination: <|im_end|> appears as literal text. "323 "Strip before ALL downstream processing.",324 "json_multistep: 4/5 cases produce only <|im_end|> as entire response.",325 "mcq: all 5 cases fail with EOS token output — bucket_score=0.000.",326 "toolcall_only: 0/2 pass — prose output instead of JSON schema.",327 ],328 "scores": {329 "task_completion": 0.5976,330 "reasoning": 0.3648, # Lowest in series331 "coherence": 0.4921, # Lowest in series332 "instruction_following": 0.8658,333 },334 "series_notes": (335 "Lowest reasoning (0.3648) and coherence (0.4921) in the evaluated series. "336 "Systematic EOS token contamination drives failures across planning, MCQ, "337 "and tool dispatch families. Demonstrates what rigorous pre-deployment "338 "evaluation surfaces that casual testing does not."339 ),340 },341 342 "mistral-nemo": {343 "display_name": "Mistral-Nemo-Instruct-2407 Q4_K_M",344 "short_name": "Mistral-Nemo",345 "family": "Mistral",346 "params": "12B",347 "context_window": 128000,348 "file_size_gb": 7.48,349 "vram_gb": 10.0,350 "avg_inference_sec": 1.42,351 "hf_repo": "pbhappliedsystems/mistral-nemo-instruct-2407-gguf-Q4-K-M",352 "hf_filename": "mistral-nemo-instruct-2407-gguf-Q4-K-M.gguf",353 "sha256": "5765024ff3361f6dc5b590b963b378bd2e87ac95eabe5823a08a3ad336b498c9",354 "run_id": "20260211_022944",355 "license": "Apache 2.0",356 "solo_only": False,357 "thinking_mode": False,358 "known_issues": [359 "MCQ A-bias: mcq_02 and mcq_05 both produce 'A' (wrong).",360 "json_multistep: ms_hard_01 fails all four gating signals simultaneously.",361 "toolcall_only: args_ok=0.000 — add schema enforcement.",362 "toolcall tool_02: final answer wrong despite correct tool dispatch — "363 "validate post-execution.",364 ],365 "scores": {366 "task_completion": 0.6631,367 "reasoning": 0.7870,368 "coherence": 0.8836,369 "instruction_following": 0.9329,370 },371 "series_notes": (372 "128K context window (Tekken tokenizer) — second largest in series "373 "after Qwen2.5-14B-1M's 1M. Multilingual: 9 languages. "374 "Strong instruction-following at 0.9329."375 ),376 },377 378 "qwen3.6-27b": {379 "display_name": "Qwen3.6-27B Q4_K_M",380 "short_name": "Qwen3.6-27B",381 "family": "Qwen3",382 "params": "27B",383 "context_window": 32768,384 "file_size_gb": 16.5,385 "vram_gb": 22.0,386 "avg_inference_sec": 1.938,387 "hf_repo": "pbhappliedsystems/qwen3.6-27B-gguf-Q4-K-M",388 "hf_filename": "qwen3.6-27B-gguf-Q4-K-M.gguf",389 "sha256": "c863357b1b532a02c47ca363ab666dd623470a152a291dac6619ed7ce751d8c8",390 "run_id": "20260426_163540",391 "license": "Apache 2.0",392 "solo_only": False,393 "thinking_mode": True,394 "known_issues": [395 "Hybrid thinking mode: <think> blocks generated on medium/hard tasks. "396 "json_multistep medium and hard cases fail with schema_ok=0 because the "397 "extraction layer receives the think block before the JSON. "398 "Strip <think>...</think> blocks before extraction, or use /no_think "399 "in user message to suppress thinking mode for structured output tasks.",400 "toolcall_only: args_ok=0.000 — uses 'arguments' instead of 'args'. "401 "tool_name key IS correct without enforcement (only model in series to do so). "402 "Specify 'args' explicitly in system prompt to resolve.",403 "EOS token contamination on toolcall final answers — "404 "strip <|im_end|> before downstream processing.",405 ],406 "scores": {407 # Single-runner evaluation: F16 GGUF (53.8 GB) exceeds RTX 4090 VRAM.408 # Aggregate dimension scores are not computed without an F16 baseline.409 # Per-family pass rates are published on the model card.410 "task_completion": None,411 "reasoning": None,412 "coherence": None,413 "instruction_following": None,414 },415 "series_notes": (416 "First Qwen3-series model in the evaluated series. "417 "Hybrid adaptive thinking mode is the defining behavioral characteristic. "418 "json_multistep 0.400 is a pipeline compatibility finding, not a capability "419 "regression — easy cases pass cleanly; medium/hard require think-block stripping. "420 "Only model in the series to produce correct 'tool_name' key without enforcement. "421 "stateful_followup: 1.000. mixed_brief_json: 1.000. MCQ: 5/5 perfect. "422 "fuzz: 20/20 pass."423 ),424 },425}426 427# ---------------------------------------------------------------------------428# VRAM budget — ZeroGPU Nvidia H200 (141 GB HBM3e)429# All models in the evaluated series can be paired without restriction.430# ---------------------------------------------------------------------------431 432ZEROGPU_VRAM_GB = 141.0433VRAM_SAFETY_CEILING_GB = 130.0434 435 436def pair_is_feasible(key_a: str, key_b: str) -> tuple[bool, str]:437 """438 Returns (feasible: bool, reason: str).439 Checks for duplicate selection and combined VRAM against H200 ceiling.440 """441 if key_a == key_b:442 return False, "Select two different models for comparison."443 combined = MODELS[key_a]["vram_gb"] + MODELS[key_b]["vram_gb"]444 if combined > VRAM_SAFETY_CEILING_GB:445 return False, (446 f"Combined VRAM estimate ({combined:.1f} GB) exceeds safe ceiling "447 f"({VRAM_SAFETY_CEILING_GB} GB)."448 )449 return True, f"Estimated combined VRAM: {combined:.1f} GB / {ZEROGPU_VRAM_GB} GB"450