Rxrohans/PayLens-Dev
0
1"""2ragas_eval.py — Phase 5 of PayLens (FIXED v2)3------------------------------------------------4FIXES IN THIS VERSION:5 6 FIX 1 — web_search_rate always 0%:7 Bug: result.get("web_search_used", False) was returning False8 because guardrails.py return dict was missing that key.9 Fix: guardrails.py now includes web_search_used. This file10 also adds a fallback check on "web_search" key for safety.11 12 FIX 2 — faithfulness scores underreporting paraphrasing:13 Bug: Jaccard overlap punishes good paraphrasing.14 "exceeds two lakh rupees" ≠ "above ₹2,00,000" by word overlap,15 even though they mean the same thing.16 Fix: Semantic faithfulness using sentence embeddings (cosine similarity).17 We embed answer + context and compare in vector space.18 Paraphrases now score correctly.19 20 FIX 3 — Added per-metric pass/fail thresholds tuned for semantic scores21 Semantic similarity scores are higher than Jaccard (0.6+ is meaningful).22 Thresholds updated accordingly.23 24METRICS:25 answer_faithfulness — cosine similarity(answer embedding, context embedding)26 answer_relevancy — cosine similarity(answer embedding, question embedding)27 context_coverage — keyword recall of ground truth in retrieved chunks28 (kept as keyword metric — coverage is recall-based, OK)29 overall_score — average of all three30"""31 32import os33import sys34import json35import time36import re37import logging38import numpy as np39from pathlib import Path40from datetime import datetime41from typing import List, Dict42 43sys.path.insert(0, str(Path(__file__).parent.parent / "src"))44 45from dotenv import load_dotenv46load_dotenv()47 48logging.basicConfig(49 level=logging.INFO,50 format="%(asctime)s | %(levelname)s | %(message)s"51)52logger = logging.getLogger("paylens.eval")53 54EVAL_DIR = Path(__file__).parent55DATASET_PATH = EVAL_DIR / "golden_dataset.json"56SCORES_PATH = EVAL_DIR / "scores_history.json"57 58 59# ── Embedding model for semantic similarity ───────────────60# Loaded once, reused for all metric computations61_embed_model = None62 63def get_embed_model():64 global _embed_model65 if _embed_model is None:66 from sentence_transformers import SentenceTransformer67 logger.info("Loading embedding model for eval metrics...")68 _embed_model = SentenceTransformer("all-MiniLM-L6-v2")69 return _embed_model70 71 72def cosine_similarity(vec_a: np.ndarray, vec_b: np.ndarray) -> float:73 """Cosine similarity between two normalized vectors."""74 norm_a = np.linalg.norm(vec_a)75 norm_b = np.linalg.norm(vec_b)76 if norm_a == 0 or norm_b == 0:77 return 0.078 return float(np.dot(vec_a, vec_b) / (norm_a * norm_b))79 80 81def semantic_similarity(text_a: str, text_b: str) -> float:82 """83 FIX 2: Semantic similarity using sentence embeddings.84 85 Why this is better than Jaccard:86 Jaccard("exceeds two lakh", "above ₹2,00,000") ≈ 0.05 (almost no overlap)87 Semantic("exceeds two lakh", "above ₹2,00,000") ≈ 0.82 (correctly similar)88 89 "RTGS charges ₹30 for amounts exceeding two lakh rupees" is a faithful90 paraphrase of "RTGS fee for transactions above ₹2,00,000: ₹30" — now scores correctly.91 92 Range: -1 to 1 (in practice 0 to 1 for meaningful text)93 Threshold for PASS: 0.50 (well above random noise at ~0.2)94 """95 model = get_embed_model()96 vecs = model.encode([text_a, text_b], normalize_embeddings=True)97 return cosine_similarity(vecs[0], vecs[1])98 99 100# ── Keyword coverage (kept as-is — it's recall-based, correct for coverage) ─101def tokenize(text: str) -> set:102 STOPWORDS = {103 "a","an","the","is","are","was","were","be","been","being",104 "have","has","had","do","does","did","will","would","could",105 "should","may","might","shall","can","to","of","in","for",106 "on","with","at","by","from","as","into","through","about",107 "and","or","but","if","then","that","this","it","its","i",108 "you","we","they","he","she","what","how","when","where","why"109 }110 words = re.findall(r'\b[a-z0-9]+\b', text.lower())111 return {w for w in words if w not in STOPWORDS and len(w) > 2}112 113 114def coverage_score(ground_truth: str, contexts: List[str]) -> float:115 """116 Keyword recall: what fraction of ground truth terms appear in retrieved chunks?117 Kept as keyword metric — this is fundamentally a recall question, not semantic.118 """119 gt_tokens = tokenize(ground_truth)120 if not gt_tokens:121 return 0.0122 ctx_tokens = tokenize(" ".join(contexts))123 covered = gt_tokens & ctx_tokens124 return len(covered) / len(gt_tokens)125 126 127# ── Pipeline runner ───────────────────────────────────────128def collect_pipeline_outputs(questions: List[Dict]) -> List[Dict]:129 """130 Runs each question through PayLens pipeline.131 FIX: Now correctly reads web_search_used from the guardrails return dict.132 """133 from chain import ChargeChain134 from guardrails import run_with_guardrails135 136 chain = ChargeChain()137 results = []138 total = len(questions)139 140 logger.info(f"Running {total} questions through pipeline...")141 142 for i, item in enumerate(questions, 1):143 q = item["question"]144 gt = item["ground_truth"]145 146 logger.info(f"[{i}/{total}] {q[:65]}...")147 148 try:149 result = run_with_guardrails(q, chain.ask)150 151 if result["blocked"]:152 logger.warning(f" Blocked: {result['guardrail_warnings']}")153 continue154 155 retrieved = chain.retriever.retrieve(q, top_k=5)156 contexts = [c["text"] for c in retrieved]157 158 # FIX 1: web_search_used is now correctly present in the dict159 # Added fallback to "web_search" key for backwards compatibility160 web_used = (161 result.get("web_search_used") # new key (guardrails v2)162 or result.get("web_search", False) # old key fallback163 )164 165 results.append({166 "question": q,167 "answer": result["answer"],168 "contexts": contexts,169 "ground_truth": gt,170 "latency_ms": result["latency_ms"],171 "confidence": result["confidence"],172 "web_search": web_used, # correctly populated now173 })174 175 except Exception as e:176 logger.error(f" Failed: {e}")177 continue178 179 if i < total:180 time.sleep(5) # 30 RPM Groq free tier181 182 logger.info(f"Collected {len(results)} valid outputs")183 return results184 185 186# ── Metric computation ────────────────────────────────────187def compute_metrics(outputs: List[Dict]) -> Dict:188 """189 FIX 2: Uses semantic similarity for faithfulness and relevancy.190 191 answer_faithfulness → semantic_similarity(answer, retrieved_context)192 answer_relevancy → semantic_similarity(answer, question)193 context_coverage → keyword recall (ground truth terms in chunks)194 """195 faithfulness_scores = []196 relevancy_scores = []197 coverage_scores = []198 per_sample = []199 200 logger.info("Computing semantic similarity metrics...")201 202 for o in outputs:203 ctx_text = " ".join(o["contexts"])204 205 # FIX: semantic similarity instead of Jaccard overlap206 faith = semantic_similarity(o["answer"], ctx_text)207 relevancy = semantic_similarity(o["answer"], o["question"])208 coverage = coverage_score(o["ground_truth"], o["contexts"])209 210 faithfulness_scores.append(faith)211 relevancy_scores.append(relevancy)212 coverage_scores.append(coverage)213 214 per_sample.append({215 "question": o["question"][:60] + "...",216 "confidence": o["confidence"],217 "latency_ms": o["latency_ms"],218 "web_search": o["web_search"],219 "faithfulness": round(faith, 3),220 "relevancy": round(relevancy, 3),221 "coverage": round(coverage, 3),222 })223 224 avg_faith = sum(faithfulness_scores) / len(faithfulness_scores)225 avg_rel = sum(relevancy_scores) / len(relevancy_scores)226 avg_cov = sum(coverage_scores) / len(coverage_scores)227 overall = (avg_faith + avg_rel + avg_cov) / 3228 229 return {230 "faithfulness": round(avg_faith, 4),231 "answer_relevancy": round(avg_rel, 4),232 "context_coverage": round(avg_cov, 4),233 "overall_score": round(overall, 4),234 "per_sample": per_sample,235 }236 237 238# ── Save scores ───────────────────────────────────────────239def save_scores(metrics: Dict, outputs: List[Dict]) -> Dict:240 history = []241 if SCORES_PATH.exists():242 with open(SCORES_PATH, "r", encoding="utf-8") as f:243 history = json.load(f)244 245 latencies = [o["latency_ms"] for o in outputs]246 web_rate = sum(1 for o in outputs if o["web_search"]) / len(outputs)247 conf_counts = {}248 for o in outputs:249 conf_counts[o["confidence"]] = conf_counts.get(o["confidence"], 0) + 1250 251 entry = {252 "timestamp": datetime.now().isoformat(),253 "metric_version": "v2-semantic", # tag so you know which algo was used254 "num_questions": len(outputs),255 "faithfulness": metrics["faithfulness"],256 "answer_relevancy": metrics["answer_relevancy"],257 "context_coverage": metrics["context_coverage"],258 "overall_score": metrics["overall_score"],259 "avg_latency_ms": round(sum(latencies) / len(latencies), 2),260 "web_search_rate": round(web_rate, 4), # now correctly > 0261 "confidence_dist": conf_counts,262 "per_sample": metrics["per_sample"],263 }264 265 history.append(entry)266 with open(SCORES_PATH, "w", encoding="utf-8") as f:267 json.dump(history, f, indent=2, ensure_ascii=False)268 269 return entry270 271 272# ── Print results ─────────────────────────────────────────273def print_results(entry: Dict):274 # Semantic similarity thresholds (higher than Jaccard — 0.5+ is meaningful)275 PASS_FAITH = 0.50276 PASS_REL = 0.55277 PASS_COV = 0.35 # coverage stays keyword-based, keep lower threshold278 279 print("\n" + "="*62)280 print(" PAYLENS — EVALUATION RESULTS (v2 · Semantic Metrics)")281 print("="*62)282 print(f" Timestamp : {entry['timestamp'][:19]}")283 print(f" Metric version : {entry.get('metric_version', 'v1-jaccard')}")284 print(f" Questions tested : {entry['num_questions']}")285 print(f" Overall Score : {entry['overall_score']:.2%}")286 print()287 print(f" Faithfulness : {entry['faithfulness']:.2%} "288 f"{'[OK]' if entry['faithfulness'] > PASS_FAITH else '[LOW]'}"289 f" (semantic sim, threshold {PASS_FAITH:.0%})")290 print(f" Answer Relevancy : {entry['answer_relevancy']:.2%} "291 f"{'[OK]' if entry['answer_relevancy'] > PASS_REL else '[LOW]'}"292 f" (semantic sim, threshold {PASS_REL:.0%})")293 print(f" Context Coverage : {entry['context_coverage']:.2%} "294 f"{'[OK]' if entry['context_coverage'] > PASS_COV else '[LOW]'}"295 f" (keyword recall, threshold {PASS_COV:.0%})")296 print()297 print(f" Avg Latency : {entry['avg_latency_ms']:.0f}ms")298 print(f" Web Search Rate : {entry['web_search_rate']:.0%} of queries")299 print(f" Confidence Dist : {entry['confidence_dist']}")300 print()301 print(" Per-sample breakdown:")302 for s in entry["per_sample"]:303 web_flag = "🌐" if s.get("web_search") else "📚"304 print(f" {web_flag} [{s['confidence'][:3].upper()}] {s['question'][:48]}")305 print(f" faith={s['faithfulness']:.2f} "306 f"rel={s['relevancy']:.2f} cov={s['coverage']:.2f} "307 f"{s['latency_ms']:.0f}ms")308 print("="*62)309 print(f" Saved to: {SCORES_PATH}")310 print("="*62 + "\n")311 312 313# ── Main ──────────────────────────────────────────────────314def run_evaluation():315 logger.info("Starting PayLens evaluation (v2 — semantic metrics)...")316 317 with open(DATASET_PATH, "r", encoding="utf-8") as f:318 dataset = json.load(f)319 logger.info(f"Loaded {len(dataset)} golden questions")320 321 outputs = collect_pipeline_outputs(dataset)322 if not outputs:323 logger.error("No outputs collected — check your pipeline")324 return325 326 metrics = compute_metrics(outputs)327 entry = save_scores(metrics, outputs)328 print_results(entry)329 return entry330 331 332if __name__ == "__main__":333 run_evaluation()