yashprmr/MeridianFinancial

1"""2scripts/gen_rag_eval.py3-----------------------4Generates artifacts/reports/rag_eval.json using a mock RAGAnswerEngine.5 6Run: python scripts/gen_rag_eval.py7"""8import sys9from pathlib import Path10from unittest.mock import MagicMock11 12sys.path.insert(0, str(Path(__file__).resolve().parents[1]))13 14from src.rag.answer import RAGAnswer, PROMPT_VERSION, REFUSAL_MESSAGE15from src.rag.rag_eval import QA_TEST_CASES, run_evaluation16 17 18def make_mock_engine():19    engine = MagicMock()20 21    def answer_fn(question, where=None):22        if "xyzzy" in question.lower():23            return RAGAnswer(24                question=question,25                answer=REFUSAL_MESSAGE,26                refused=True,27                evidence_ids=[],28                evidence_sufficiency="No relevant evidence found.",29                prompt_version=PROMPT_VERSION,30                retrieval_count=2,31                token_usage={},32                latency_ms=30.0,33                model="mistral-small-latest",34            )35        return RAGAnswer(36            question=question,37            answer="Based on the evidence, consumers frequently report this issue.",38            refused=False,39            evidence_ids=["chunk_a1b2", "chunk_c3d4"],40            evidence_sufficiency=(41                "Evidence quality: HIGH "42                "(avg_similarity=0.82, max_similarity=0.90, n_chunks=2)"43            ),44            prompt_version=PROMPT_VERSION,45            retrieval_count=5,46            token_usage={"prompt_tokens": 312, "completion_tokens": 89, "total_tokens": 401},47            latency_ms=750.0,48            model="mistral-small-latest",49        )50 51    engine.answer.side_effect = answer_fn52    return engine53 54 55if __name__ == "__main__":56    engine = make_mock_engine()57    report_path = Path("artifacts/reports/rag_eval.json")58    report = run_evaluation(engine, QA_TEST_CASES, report_path=report_path)59    summary = report["summary"]60    print(f"Report saved to {report_path}")61    print(62        f"Summary: {summary['passed']}/{summary['total_cases']} passed "63        f"(pass_rate={summary['pass_rate']:.1%})"64    )65