Adityax-07/CodeSage

1"""2Standalone benchmark runner — runs all 50 reference questions through3Baseline LLM and RAG, computes all 8 metrics, saves to data/benchmark_cache.json.4Run once: python run_benchmark.py5"""6import os, json, time7import numpy as np8# rouge_score MUST be imported before heavy ML libs to avoid segfault9from rouge_score import rouge_scorer as rs10from dotenv import load_dotenv11 12load_dotenv()13 14# ── Import system modules (after rouge_score) ─────────────────────────────────15from langchain_huggingface import HuggingFaceEmbeddings16from langchain_community.vectorstores import FAISS17from openai import OpenAI18 19# ── Load reference answers ────────────────────────────────────────────────────20with open("data/reference_answers.json", encoding="utf-8") as f:21    ref_answers = json.load(f)22QUESTIONS = list(ref_answers.keys())23 24# ── Load vector store ─────────────────────────────────────────────────────────25INDEX_PATH = "data/faiss_index"26print("Loading vector store...")27emb = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")28vs  = FAISS.load_local(INDEX_PATH, emb, allow_dangerous_deserialization=True)29print("Vector store ready.\n")30 31# ── Groq client ───────────────────────────────────────────────────────────────32client = OpenAI(33    api_key=os.getenv("GROQ_API_KEY"),34    base_url="https://api.groq.com/openai/v1",35)36MODEL = "llama-3.1-8b-instant"37 38BASELINE_SYS = (39    "You are a programming tutor specializing in Data Structures, Algorithms, "40    "and Web Development. Answer questions clearly and concisely."41)42RAG_SYS = (43    "You are a programming tutor. Use only the provided context to answer. "44    "If the answer is not in the context, say 'I don't have that in my knowledge base.'"45)46 47def ask_baseline(q: str) -> dict:48    t = time.time()49    r = client.chat.completions.create(50        model=MODEL,51        messages=[{"role": "system", "content": BASELINE_SYS}, {"role": "user", "content": q}],52        max_tokens=300, temperature=0.3,53    )54    return {"answer": r.choices[0].message.content.strip(), "response_time": round(time.time()-t, 2)}55 56def ask_rag(q: str) -> dict:57    t = time.time()58    docs    = vs.similarity_search(q, k=3)59    context = "\n\n".join([d.page_content for d in docs])60    prompt  = f"Context:\n{context}\n\nQuestion: {q}\nAnswer:"61    r = client.chat.completions.create(62        model=MODEL,63        messages=[{"role": "system", "content": RAG_SYS}, {"role": "user", "content": prompt}],64        max_tokens=300, temperature=0.3,65    )66    return {"answer": r.choices[0].message.content.strip(),67            "response_time": round(time.time()-t, 2), "context": context}68 69# ── Metric helpers ────────────────────────────────────────────────────────────70scorer = rs.RougeScorer(["rougeL"], use_stemmer=True)71 72def _cosine(a, b):73    n = np.linalg.norm(a) * np.linalg.norm(b)74    return float(np.dot(a, b) / (n + 1e-8))75 76def compute_metrics(answer: str, question: str, context: str = "") -> dict:77    if not answer or not answer.strip():78        return {"accuracy": 0, "rouge_l": 0, "groundedness": 0,79                "answer_relevance": 0, "faithfulness": 0}80    try:81        a_emb = np.array(vs.embeddings.embed_query(answer))82        q_emb = np.array(vs.embeddings.embed_query(question))83        answer_relevance = round(max(0.0, _cosine(a_emb, q_emb)), 3)84 85        ref = ref_answers.get(question.strip().lower(), "")86        accuracy, rouge_l = 0.0, 0.087        if ref:88            rouge_l  = round(scorer.score(ref, answer)["rougeL"].fmeasure, 3)89            r_emb    = np.array(vs.embeddings.embed_query(ref))90            accuracy = round(max(0.0, _cosine(a_emb, r_emb)), 3)91 92        if context and context.strip():93            c_emb = np.array(vs.embeddings.embed_query(context[:1000]))94            groundedness = round(max(0.0, _cosine(a_emb, c_emb)), 3)95            faithfulness = round(scorer.score(context[:1000], answer)["rougeL"].fmeasure, 3)96        else:97            groundedness = accuracy98            faithfulness = rouge_l99 100        return {"accuracy": accuracy, "rouge_l": rouge_l,101                "groundedness": groundedness, "answer_relevance": answer_relevance,102                "faithfulness": faithfulness}103    except Exception as e:104        print(f"  [metric error] {e}")105        return {"accuracy": 0, "rouge_l": 0, "groundedness": 0,106                "answer_relevance": 0, "faithfulness": 0}107 108def _cost(answer: str, system: str) -> float:109    tokens = max(1, len(answer.split()))110    if   system == "r1": return round(0.001 + tokens * 0.0000059, 4)111    elif system == "r2": return round(0.0015 + tokens * 0.0000059 * 1.8, 4)112    else:                return round(tokens * 0.0000015, 4)113 114# ── Run benchmark ─────────────────────────────────────────────────────────────115# Load existing partial results to resume if interrupted116OUT_PATH = "data/benchmark_cache.json"117if os.path.exists(OUT_PATH):118    with open(OUT_PATH, encoding="utf-8") as f:119        results = json.load(f)120    done_qs = {r["question"] for r in results}121    print(f"Resuming — {len(results)} already done.\n")122else:123    results  = []124    done_qs  = set()125 126total = len(QUESTIONS)127print(f"Running benchmark on {total} questions...\n")128 129for i, q in enumerate(QUESTIONS):130    if q in done_qs:131        print(f"[{i+1:02d}/{total}] SKIP (cached): {q[:55]}")132        continue133 134    print(f"[{i+1:02d}/{total}] {q[:60]}")135 136    r1  = ask_baseline(q)137    r2  = ask_rag(q)138    ctx = r2.get("context", "")139 140    m1 = compute_metrics(r1["answer"], q)141    m2 = compute_metrics(r2["answer"], q, context=ctx)142 143    results.append({144        "question":  q,145        "r1_time":   r1["response_time"], "r2_time": r2["response_time"], "r3_time": 0,146        "r1_rouge":  m1["rouge_l"],   "r2_rouge":  m2["rouge_l"],   "r3_rouge":  0,147        "r1_sim":    m1["accuracy"],  "r2_sim":    m2["accuracy"],  "r3_sim":    0,148        "r1_ground": m1["groundedness"], "r2_ground": m2["groundedness"], "r3_ground": 0,149        "r1_relev":  m1["answer_relevance"], "r2_relev": m2["answer_relevance"], "r3_relev": 0,150        "r1_faith":  m1["faithfulness"], "r2_faith": m2["faithfulness"], "r3_faith": 0,151        "r1_cost":   _cost(r1["answer"], "r1"),152        "r2_cost":   _cost(r2["answer"], "r2"),153        "r3_cost":   0,154    })155    print(f"       r1_acc={m1['accuracy']:.2f}  r2_acc={m2['accuracy']:.2f} | "156          f"r1={r1['response_time']}s  r2={r2['response_time']}s")157 158    # Save after every question so we can resume if interrupted159    with open(OUT_PATH, "w", encoding="utf-8") as f:160        json.dump(results, f, indent=2)161 162# ── Summary ───────────────────────────────────────────────────────────────────163n = len(results)164r1_acc = round(sum(r["r1_sim"] for r in results) / n * 100, 1)165r2_acc = round(sum(r["r2_sim"] for r in results) / n * 100, 1)166r1_t   = round(sum(r["r1_time"] for r in results) / n, 2)167r2_t   = round(sum(r["r2_time"] for r in results) / n, 2)168print(f"\nDone! {n} rows saved to {OUT_PATH}")169print(f"  Baseline  — accuracy {r1_acc}%  avg_time {r1_t}s")170print(f"  RAG       — accuracy {r2_acc}%  avg_time {r2_t}s")171