Premchan369/Q-TensorFormer

1"""2tests/test_baseline_comparator.py3Unit tests for the Comprehensive Baseline Comparison Engine.4"""5 6import pytest7import os8import json9import tempfile10from pathlib import Path11 12import sys13import os14sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))15 16from src.baseline_comparator import (17    ProvenanceTag,18    MetricDirection,19    MetricSpec,20    METRIC_CATALOG,21    BaselineEvaluationRecord,22    BaselineComparator,23)24 25 26def test_metric_catalog_integrity():27    assert len(METRIC_CATALOG) >= 1728    assert "parameters_total_m" in METRIC_CATALOG29    assert "tpot_ms" in METRIC_CATALOG30    assert "tokens_per_sec" in METRIC_CATALOG31    assert "cosine_fidelity" in METRIC_CATALOG32    assert "cost_per_million_tokens_usd" in METRIC_CATALOG33 34    # Verify every metric has valid direction and provenance35    for key, spec in METRIC_CATALOG.items():36        assert isinstance(spec.direction, MetricDirection)37        assert isinstance(spec.default_provenance, ProvenanceTag)38        assert len(spec.display_name) > 039        assert len(spec.unit) > 040 41 42def test_compute_percentage_improvement():43    # Lower is better: reduction is an improvement (+)44    # Latency 10ms -> 5ms: +50%45    res = BaselineComparator.compute_percentage_improvement(10.0, 5.0, MetricDirection.LOWER_IS_BETTER)46    assert res == pytest.approx(50.0)47 48    # Latency 10ms -> 15ms: -50%49    res = BaselineComparator.compute_percentage_improvement(10.0, 15.0, MetricDirection.LOWER_IS_BETTER)50    assert res == pytest.approx(-50.0)51 52    # Higher is better: increase is an improvement (+)53    # Tokens/sec 100 -> 150: +50%54    res = BaselineComparator.compute_percentage_improvement(100.0, 150.0, MetricDirection.HIGHER_IS_BETTER)55    assert res == pytest.approx(50.0)56 57    # Tokens/sec 100 -> 50: -50%58    res = BaselineComparator.compute_percentage_improvement(100.0, 50.0, MetricDirection.HIGHER_IS_BETTER)59    assert res == pytest.approx(-50.0)60 61    # Missing values return None62    assert BaselineComparator.compute_percentage_improvement(None, 10.0, MetricDirection.LOWER_IS_BETTER) is None63    assert BaselineComparator.compute_percentage_improvement(10.0, None, MetricDirection.HIGHER_IS_BETTER) is None64    assert BaselineComparator.compute_percentage_improvement(0.0, 10.0, MetricDirection.LOWER_IS_BETTER) is None65 66 67def test_comparator_matrix_and_formatting():68    rec_dense = BaselineEvaluationRecord(69        model_id="dense",70        display_name="Dense Baseline (FP32)",71        category="dense",72        metrics={73            "parameters_total_m": 0.237,74            "tpot_ms": 2.50,75            "tokens_per_sec": 400.0,76            "peak_ram_mb": 0.94,77            "cosine_fidelity": 1.000,78        },79    )80 81    rec_qtf = BaselineEvaluationRecord(82        model_id="qtf_edge",83        display_name="Q-TensorFormer (Edge-SLA)",84        category="adaptive_tt",85        metrics={86            "parameters_total_m": 0.112,87            "tpot_ms": 2.78,88            "tokens_per_sec": 359.7,89            "peak_ram_mb": 0.44,90            "cosine_fidelity": 0.948,91        },92    )93 94    comparator = BaselineComparator([rec_dense, rec_qtf])95    matrix = comparator.compute_improvement_matrix(96        target_model_id="qtf_edge",97        baseline_model_ids=["dense"],98        metric_keys=["peak_ram_mb", "tpot_ms", "tokens_per_sec"],99    )100 101    # RAM: 0.94 -> 0.44 => (0.94 - 0.44)/0.94 * 100 = +53.19%102    assert matrix["dense"]["peak_ram_mb"] == pytest.approx(53.19, abs=0.1)103 104    # TPOT: 2.50 -> 2.78 => (2.50 - 2.78)/2.50 * 100 = -11.2%105    assert matrix["dense"]["tpot_ms"] == pytest.approx(-11.2, abs=0.1)106 107    # Table generation108    md_table = comparator.to_markdown_table(metric_keys=["peak_ram_mb", "tpot_ms"])109    assert "| **Dense Baseline (FP32)** |" in md_table110    assert "| **Q-TensorFormer (Edge-SLA)** |" in md_table111 112    # Percentage matrix markdown113    pct_md = comparator.to_percentage_matrix_markdown(114        target_model_id="qtf_edge",115        baseline_model_ids=["dense"],116        metric_keys=["peak_ram_mb", "tpot_ms"],117    )118    assert "Relative % Improvement: Q-TensorFormer (Edge-SLA) vs Baselines" in pct_md119    assert "**+53.2%**" in pct_md or "+53.2%" in pct_md120 121 122def test_regime_analyzer():123    comparator = BaselineComparator()124    regimes = comparator.identify_regimes()125    assert "best_case_regimes" in regimes126    assert "worst_case_regimes" in regimes127    assert len(regimes["best_case_regimes"]) >= 3128    assert len(regimes["worst_case_regimes"]) >= 3129 130 131def test_json_and_csv_export(tmp_path):132    rec = BaselineEvaluationRecord(133        model_id="dense",134        display_name="Dense Baseline",135        category="dense",136        metrics={"parameters_total_m": 0.237, "tpot_ms": 2.50},137    )138    comparator = BaselineComparator([rec])139 140    json_path = tmp_path / "baseline_out.json"141    csv_path = tmp_path / "baseline_out.csv"142 143    comparator.to_json(json_path)144    comparator.to_csv(csv_path)145 146    assert json_path.exists()147    assert csv_path.exists()148 149    with open(json_path, "r") as f:150        data = json.load(f)151        assert data["metadata"]["system"] == "Q-TensorFormer Baseline Comparison System"152        assert len(data["models"]) == 1153