VTdevelops/rag-vs-full-context-comparator
0
1#!/usr/bin/env python32"""RAG vs full-context comparator Gradio app."""3 4import asyncio5import json6import os7import math8import re9import uuid10import textwrap11from collections import Counter12from dataclasses import dataclass, field13from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple14 15import gradio as gr16 17try:18 from dotenv import load_dotenv19except ImportError: # pragma: no cover - optional dependency20 def load_dotenv(*_args, **_kwargs):21 return None22 23try:24 import chromadb25 from chromadb.config import Settings26except Exception: # pragma: no cover - optional dependency27 chromadb = None28 Settings = None29 30 31load_dotenv()32 33 34@dataclass35class Document:36 name: str37 text: str38 39 40@dataclass41class Chunk:42 text: str43 source: str44 tokens: Counter = field(default_factory=Counter)45 score: float = 0.046 embedding: Optional[List[float]] = None47 uid: Optional[str] = None48 49 50TOKEN_RE = re.compile(r"[A-Za-z0-9']+")51 52FULL_CONTEXT_LABEL = "Full-context answer"53RAG_LABEL = "RAG answer"54 55 56def _env_int(name: str, default: int) -> int:57 try:58 return int(os.getenv(name, str(default)))59 except ValueError:60 return default61 62 63FULL_CONTEXT_CHAR_LIMIT = max(0, _env_int("FULL_CONTEXT_CHAR_LIMIT", 12000))64 65 66def tokenize(text: str) -> List[str]:67 if not text:68 return []69 return TOKEN_RE.findall(text.lower())70 71 72def split_into_sentences(text: str) -> List[str]:73 if not text:74 return []75 sentences = re.split(r"(?<=[.!?])\s+", text.strip())76 return [sentence.strip() for sentence in sentences if sentence.strip()]77 78 79def cosine_similarity(vec_a: Counter, vec_b: Counter) -> float:80 if not vec_a or not vec_b:81 return 0.082 shared = set(vec_a.keys()) & set(vec_b.keys())83 if not shared:84 return 0.085 dot = sum(vec_a[token] * vec_b[token] for token in shared)86 norm_a = math.sqrt(sum(value * value for value in vec_a.values()))87 norm_b = math.sqrt(sum(value * value for value in vec_b.values()))88 if norm_a == 0.0 or norm_b == 0.0:89 return 0.090 value = dot / (norm_a * norm_b)91 return max(0.0, min(1.0, value))92 93 94def cosine_similarity_dense(vec_a: Sequence[float], vec_b: Sequence[float]) -> float:95 if not vec_a or not vec_b:96 return 0.097 if len(vec_a) != len(vec_b):98 return 0.099 dot = sum(a * b for a, b in zip(vec_a, vec_b))100 norm_a = math.sqrt(sum(a * a for a in vec_a))101 norm_b = math.sqrt(sum(b * b for b in vec_b))102 if norm_a == 0.0 or norm_b == 0.0:103 return 0.0104 value = dot / (norm_a * norm_b)105 return max(0.0, min(1.0, value))106 107 108def parse_json_block(text: str) -> Dict[str, Any]:109 candidate = (text or "").strip()110 if not candidate:111 raise ValueError("Empty response body")112 try:113 return json.loads(candidate)114 except json.JSONDecodeError:115 pass116 117 fenced = re.findall(r"```(?:json)?\s*(\{.*?\})\s*```", candidate, flags=re.IGNORECASE | re.DOTALL)118 for block in fenced:119 try:120 return json.loads(block)121 except json.JSONDecodeError:122 continue123 124 start = candidate.find("{")125 end = candidate.rfind("}")126 if start != -1 and end != -1 and end > start:127 snippet = candidate[start : end + 1]128 try:129 return json.loads(snippet)130 except json.JSONDecodeError:131 pass132 133 raise ValueError("Could not parse JSON from response")134 135 136def chunk_document(document: Document, chunk_size: int = 200, chunk_overlap: int = 40) -> List[Chunk]:137 sentences = split_into_sentences(document.text)138 if not sentences:139 stripped = (document.text or "").strip()140 if not stripped:141 return []142 tokens = tokenize(stripped)143 if not tokens:144 return []145 return [Chunk(text=stripped, source=document.name, tokens=Counter(tokens))]146 147 chunks: List[Chunk] = []148 current_sentences: List[str] = []149 current_tokens: List[List[str]] = []150 current_len = 0151 152 for sentence in sentences:153 tokens = tokenize(sentence)154 if not tokens:155 continue156 token_count = len(tokens)157 if current_len + token_count > chunk_size and current_sentences:158 chunk_text = " ".join(current_sentences).strip()159 if chunk_text:160 chunk_counter = Counter()161 for token_list in current_tokens:162 chunk_counter.update(token_list)163 chunks.append(Chunk(text=chunk_text, source=document.name, tokens=chunk_counter))164 if chunk_overlap > 0:165 retained_sentences: List[str] = []166 retained_tokens: List[List[str]] = []167 retained_len = 0168 for prev_sentence, prev_tokens in zip(reversed(current_sentences), reversed(current_tokens)):169 retained_sentences.insert(0, prev_sentence)170 retained_tokens.insert(0, prev_tokens)171 retained_len += len(prev_tokens)172 if retained_len >= chunk_overlap:173 break174 current_sentences = retained_sentences175 current_tokens = retained_tokens176 current_len = retained_len177 else:178 current_sentences = []179 current_tokens = []180 current_len = 0181 current_sentences.append(sentence)182 current_tokens.append(tokens)183 current_len += token_count184 185 if current_sentences:186 chunk_text = " ".join(current_sentences).strip()187 if chunk_text:188 chunk_counter = Counter()189 for token_list in current_tokens:190 chunk_counter.update(token_list)191 chunks.append(Chunk(text=chunk_text, source=document.name, tokens=chunk_counter))192 193 return chunks194 195 196def build_corpus_chunks(documents: Sequence[Document], chunk_size: int = 200, chunk_overlap: int = 40) -> List[Chunk]:197 chunks: List[Chunk] = []198 counter = 0199 for document in documents:200 for chunk in chunk_document(document, chunk_size=chunk_size, chunk_overlap=chunk_overlap):201 chunk.uid = f"{document.name}-{counter}"202 counter += 1203 chunks.append(chunk)204 return chunks205 206 207def assemble_full_context(documents: Sequence[Document], max_chars: Optional[int] = None) -> Tuple[str, bool]:208 if not documents:209 return "", False210 211 limit = max_chars if max_chars and max_chars > 0 else None212 combined_parts: List[str] = []213 running_total = 0214 truncated = False215 216 for document in documents:217 section = f"Source: {document.name}\n{document.text.strip()}\n\n"218 section_length = len(section)219 if limit is not None and running_total + section_length > limit:220 remaining = limit - running_total221 if remaining > 0:222 combined_parts.append(section[:remaining])223 running_total += remaining224 truncated = True225 break226 combined_parts.append(section)227 running_total += section_length228 229 combined_text = "".join(combined_parts).strip()230 return combined_text, truncated231 232 233def retrieve_chunks_lexical(query: str, chunks: Sequence[Chunk], top_k: int = 3) -> List[Chunk]:234 query_tokens = Counter(tokenize(query))235 if not query_tokens:236 return []237 scored: List[Chunk] = []238 for chunk in chunks:239 if not chunk.tokens:240 continue241 score = cosine_similarity(query_tokens, chunk.tokens)242 scored.append(243 Chunk(244 text=chunk.text,245 source=chunk.source,246 tokens=chunk.tokens,247 score=score,248 embedding=chunk.embedding,249 uid=chunk.uid,250 )251 )252 scored.sort(key=lambda item: item.score, reverse=True)253 return scored[:top_k]254 255 256def extract_text_from_upload(path: str, name: str) -> str:257 ext = os.path.splitext(name)[1].lower()258 if ext in {".txt", ".md", ".markdown", ".csv"}:259 with open(path, "r", encoding="utf-8", errors="ignore") as handle:260 return handle.read()261 if ext == ".json":262 import json263 264 with open(path, "r", encoding="utf-8", errors="ignore") as handle:265 data = json.load(handle)266 if isinstance(data, str):267 return data268 return json.dumps(data, indent=2)269 if ext == ".pdf":270 try:271 from pypdf import PdfReader272 except Exception as exc:273 raise RuntimeError("PDF support requires the 'pypdf' package to be installed in this environment.") from exc274 reader = PdfReader(path)275 pages = []276 for page in reader.pages:277 text = page.extract_text() or ""278 pages.append(text)279 return "\n".join(pages)280 raise RuntimeError(281 f"Unsupported file type: '{ext or 'unknown'}'. Upload text-based formats such as .txt, .md, .json, or .pdf."282 )283 284 285def read_uploaded_files(files: Optional[Sequence]) -> Tuple[List[Document], List[str]]:286 documents: List[Document] = []287 errors: List[str] = []288 if not files:289 return documents, errors290 for uploaded in files:291 path = getattr(uploaded, "name", None)292 if not path:293 errors.append("Encountered an upload without a file path.")294 continue295 display_name = getattr(uploaded, "orig_name", None) or os.path.basename(path)296 try:297 text = extract_text_from_upload(path, display_name)298 cleaned = text.strip()299 if not cleaned:300 errors.append(f"{display_name}: file was empty after stripping whitespace.")301 continue302 documents.append(Document(name=display_name, text=cleaned))303 except Exception as exc:304 errors.append(f"{display_name}: {exc}")305 return documents, errors306 307 308def compute_support_score(309 answer: str,310 corpus_embeddings: Sequence[Sequence[float]],311 embedder: "EmbeddingService",312) -> float:313 answer_sentences = split_into_sentences(answer)314 if not answer_sentences or not corpus_embeddings:315 return 0.0316 answer_embeddings = embedder.embed_texts(answer_sentences)317 if not answer_embeddings:318 return 0.0319 filtered_corpus = [vector for vector in corpus_embeddings if vector]320 if not filtered_corpus:321 return 0.0322 sentence_scores: List[float] = []323 for sentence_vector in answer_embeddings:324 if not sentence_vector:325 continue326 similarities = [cosine_similarity_dense(sentence_vector, corpus_vec) for corpus_vec in filtered_corpus]327 if similarities:328 sentence_scores.append(max(similarities))329 if not sentence_scores:330 return 0.0331 average = sum(sentence_scores) / len(sentence_scores)332 return max(0.0, min(1.0, average))333 334 335def compute_semantic_similarity(answer: str, reference: str, embedder: "EmbeddingService") -> Optional[float]:336 if not answer or not reference:337 return None338 embeddings = embedder.embed_texts([answer, reference])339 if len(embeddings) != 2:340 return None341 answer_vector, reference_vector = embeddings342 if not answer_vector or not reference_vector:343 return None344 return cosine_similarity_dense(answer_vector, reference_vector)345 346 347def compute_retrieval_effectiveness(348 question: str,349 question_embedding: Optional[Sequence[float]],350 retrieved: Sequence[Chunk],351 all_chunks: Sequence[Chunk],352) -> float:353 if question_embedding and any(chunk.embedding for chunk in retrieved):354 retrieved_scores = [355 cosine_similarity_dense(question_embedding, chunk.embedding)356 for chunk in retrieved357 if chunk.embedding358 ]359 baseline_scores = [360 cosine_similarity_dense(question_embedding, chunk.embedding)361 for chunk in all_chunks362 if chunk.embedding363 ]364 if not retrieved_scores or not baseline_scores:365 return 0.0366 top_avg = sum(retrieved_scores) / len(retrieved_scores)367 baseline_avg = sum(baseline_scores) / len(baseline_scores)368 return max(0.0, min(1.0, top_avg / (top_avg + baseline_avg + 1e-8)))369 370 # Fallback to lexical approximation when embeddings are unavailable371 query_vector = Counter(tokenize(question))372 if not query_vector or not retrieved:373 return 0.0374 retrieved_scores = [cosine_similarity(query_vector, chunk.tokens) for chunk in retrieved if chunk.tokens]375 baseline_scores = [cosine_similarity(query_vector, chunk.tokens) for chunk in all_chunks if chunk.tokens]376 if not retrieved_scores or not baseline_scores:377 return 0.0378 top_avg = sum(retrieved_scores) / len(retrieved_scores)379 baseline_avg = sum(baseline_scores) / len(baseline_scores)380 return max(0.0, min(1.0, top_avg / (top_avg + baseline_avg + 1e-8)))381 382 383def discounted_gain(score: float, rank: int) -> float:384 return (2 ** score - 1) / math.log2(rank + 1) if rank > 0 else 0.0385 386 387def compute_rank_metrics(388 retrieved: Sequence[Chunk],389 all_chunks: Sequence[Chunk],390 question_embedding: Optional[Sequence[float]],391 max_relevant: int = 5,392) -> Dict[str, float]:393 if not question_embedding:394 return {"precision": 0.0, "recall": 0.0, "mrr": 0.0, "ndcg": 0.0}395 396 scored: List[Tuple[str, float]] = []397 for chunk in all_chunks:398 if not chunk.embedding or not chunk.uid:399 continue400 score = cosine_similarity_dense(question_embedding, chunk.embedding)401 if score <= 0.0:402 continue403 scored.append((chunk.uid, score))404 405 if not scored:406 return {"precision": 0.0, "recall": 0.0, "mrr": 0.0, "ndcg": 0.0}407 408 scored.sort(key=lambda item: item[1], reverse=True)409 top_relevant = {uid for uid, _ in scored[: max_relevant]}410 relevant_scores = {uid: score for uid, score in scored}411 total_relevant = max(len(top_relevant), 1)412 413 retrieved_uids = [chunk.uid for chunk in retrieved if chunk.uid]414 if not retrieved_uids:415 return {"precision": 0.0, "recall": 0.0, "mrr": 0.0, "ndcg": 0.0}416 417 binary_relevance = [1 if uid in top_relevant else 0 for uid in retrieved_uids]418 graded_relevance = [relevant_scores.get(uid, 0.0) for uid in retrieved_uids]419 420 precision = sum(binary_relevance) / len(binary_relevance)421 recall = sum(binary_relevance) / total_relevant422 423 mrr = 0.0424 for idx, is_rel in enumerate(binary_relevance):425 if is_rel:426 mrr = 1.0 / (idx + 1)427 break428 429 dcg = 0.0430 for idx, rel in enumerate(graded_relevance):431 dcg += discounted_gain(rel, idx + 1)432 433 ideal_scores = [score for _, score in scored[: len(graded_relevance)]]434 idcg = 0.0435 for idx, rel in enumerate(ideal_scores):436 idcg += discounted_gain(rel, idx + 1)437 ndcg = dcg / idcg if idcg > 0 else 0.0438 439 return {440 "precision": max(0.0, min(1.0, precision)),441 "recall": max(0.0, min(1.0, recall)),442 "mrr": max(0.0, min(1.0, mrr)),443 "ndcg": max(0.0, min(1.0, ndcg)),444 }445 446 447def winner_label(score_full_context: Optional[float], score_rag: Optional[float], higher_is_better: bool) -> str:448 if score_full_context is None and score_rag is None:449 return "Tie"450 if score_full_context is None:451 return RAG_LABEL452 if score_rag is None:453 return FULL_CONTEXT_LABEL454 tolerance = 1e-6455 if abs(score_full_context - score_rag) <= tolerance:456 return "Tie"457 if higher_is_better:458 return RAG_LABEL if score_rag > score_full_context else FULL_CONTEXT_LABEL459 return RAG_LABEL if score_rag < score_full_context else FULL_CONTEXT_LABEL460 461 462def evaluate_answers(463 question: str,464 documents: Sequence[Document],465 all_chunks: Sequence[Chunk],466 retrieved_chunks: Sequence[Chunk],467 full_context_answer: str,468 rag_answer: str,469 reference_answer: str,470 embedder: "EmbeddingService",471 question_embedding: Optional[Sequence[float]],472) -> dict:473 corpus_sentences: List[str] = []474 for document in documents:475 corpus_sentences.extend(split_into_sentences(document.text))476 corpus_embeddings = embedder.embed_texts(corpus_sentences) if corpus_sentences else []477 support_full = compute_support_score(full_context_answer, corpus_embeddings, embedder)478 support_rag = compute_support_score(rag_answer, corpus_embeddings, embedder)479 hallucination_full = max(0.0, 1.0 - support_full)480 hallucination_rag = max(0.0, 1.0 - support_rag)481 reference_clean = reference_answer.strip()482 semantic_full = compute_semantic_similarity(full_context_answer, reference_clean, embedder) if reference_clean else None483 semantic_rag = compute_semantic_similarity(rag_answer, reference_clean, embedder) if reference_clean else None484 retrieval_score = compute_retrieval_effectiveness(question, question_embedding, retrieved_chunks, all_chunks)485 rank_metrics = compute_rank_metrics(retrieved_chunks, all_chunks, question_embedding)486 comparison = {487 "more_grounded": winner_label(support_full, support_rag, higher_is_better=True),488 "fewer_hallucinations": winner_label(hallucination_full, hallucination_rag, higher_is_better=False),489 "closest_to_reference": (490 winner_label(semantic_full, semantic_rag, higher_is_better=True)491 if reference_clean492 else "Reference answer not provided"493 ),494 "retrieval_effectiveness": retrieval_score,495 }496 return {497 "full_context": {498 "support_score": support_full,499 "hallucination_score": hallucination_full,500 "semantic_similarity": semantic_full,501 },502 "rag": {503 "support_score": support_rag,504 "hallucination_score": hallucination_rag,505 "semantic_similarity": semantic_rag,506 },507 "retrieval_effectiveness": retrieval_score,508 "retrieval_rank": rank_metrics,509 "retrieved_k": len(retrieved_chunks),510 "comparison": comparison,511 }512 513 514def format_score(score: Optional[float]) -> str:515 if score is None:516 return "—"517 return f"{score * 100:.1f}%"518 519 520def format_metrics_html(metrics: dict, reference_available: bool) -> str:521 rows = [522 (523 "Support score",524 format_score(metrics["full_context"]["support_score"]),525 format_score(metrics["rag"]["support_score"]),526 "Higher is better — measures how well each answer sentence embedding is backed by the uploaded sources.",527 ),528 (529 "Hallucination score",530 format_score(metrics["full_context"]["hallucination_score"]),531 format_score(metrics["rag"]["hallucination_score"]),532 "Lower is better — 1 minus the support score, indicating embedding-level gaps in support.",533 ),534 ]535 if reference_available:536 rows.append(537 (538 "Semantic similarity to reference",539 format_score(metrics["full_context"]["semantic_similarity"]),540 format_score(metrics["rag"]["semantic_similarity"]),541 "Higher is better — embedding similarity to the provided ground-truth answer.",542 )543 )544 rows.append(545 (546 "Retrieval effectiveness",547 "—",548 format_score(metrics["retrieval_effectiveness"]),549 "Higher is better — compares retrieved chunk embeddings to the average embedding match for this question.",550 )551 )552 553 html_parts = [554 "<div class='metric-summary'>",555 "<table class='metric-table'>",556 "<thead><tr><th>Metric</th><th>Full context</th><th>RAG</th><th>Interpretation</th></tr></thead>",557 "<tbody>",558 ]559 for metric_name, no_value, rag_value, note in rows:560 html_parts.append(561 f"<tr><td>{metric_name}</td><td>{no_value}</td><td>{rag_value}</td><td>{note}</td></tr>"562 )563 html_parts.append("</tbody></table>")564 565 comparison = metrics["comparison"]566 html_parts.append("<p class='comparison-note'>Automatic comparison</p>")567 html_parts.append("<ul>")568 html_parts.append(569 f"<li class='comparison-note'>More grounded: <strong>{comparison['more_grounded']}</strong></li>"570 )571 html_parts.append(572 f"<li class='comparison-note'>Fewer hallucinations: <strong>{comparison['fewer_hallucinations']}</strong></li>"573 )574 if reference_available:575 html_parts.append(576 f"<li class='comparison-note'>Closer to reference answer: <strong>{comparison['closest_to_reference']}</strong></li>"577 )578 else:579 html_parts.append(580 "<li>Closer to reference answer: Provide a reference answer to enable this comparison.</li>"581 )582 retrieval_percent = format_score(metrics["retrieval_effectiveness"])583 html_parts.append(584 f"<li class='comparison-note'>Retrieval effectiveness score: <strong>{retrieval_percent}</strong></li>"585 )586 html_parts.append("</ul>")587 rank_metrics = metrics.get("retrieval_rank", {})588 if rank_metrics:589 retrieved_k = metrics.get("retrieved_k") or len(rank_metrics)590 html_parts.append("<p class='comparison-note'>Retrieval ranking metrics</p>")591 html_parts.append("<ul>")592 html_parts.append(593 f"<li>Precision@{retrieved_k}: <strong>{format_score(rank_metrics.get('precision'))}</strong></li>"594 )595 html_parts.append(596 f"<li>Recall estimate: <strong>{format_score(rank_metrics.get('recall'))}</strong></li>"597 )598 html_parts.append(599 f"<li>MRR: <strong>{format_score(rank_metrics.get('mrr'))}</strong></li>"600 )601 html_parts.append(602 f"<li>nDCG: <strong>{format_score(rank_metrics.get('ndcg'))}</strong></li>"603 )604 html_parts.append("</ul>")605 judge = metrics.get("judge", {})606 if judge:607 html_parts.append("<p class='comparison-note'>LLM judge verdicts</p>")608 html_parts.append("<table class='metric-table'>")609 html_parts.append("<thead><tr><th>Criterion</th><th>Full context</th><th>RAG</th><th>Interpretation</th></tr></thead>")610 html_parts.append("<tbody>")611 judge_rows = [612 (613 "Accuracy",614 format_score(judge.get("full_context", {}).get("accuracy")),615 format_score(judge.get("rag", {}).get("accuracy")),616 "Higher means the answer aligns with ground truth/context.",617 ),618 (619 "Completeness",620 format_score(judge.get("full_context", {}).get("completeness")),621 format_score(judge.get("rag", {}).get("completeness")),622 "Higher means the answer covers all key points.",623 ),624 (625 "Relevance",626 format_score(judge.get("full_context", {}).get("relevance")),627 format_score(judge.get("rag", {}).get("relevance")),628 "Higher means the answer stays on topic for the question.",629 ),630 ]631 for metric_name, no_value, rag_value, note in judge_rows:632 html_parts.append(633 f"<tr><td>{metric_name}</td><td>{no_value}</td><td>{rag_value}</td><td>{note}</td></tr>"634 )635 html_parts.append("</tbody></table>")636 feedback_no = judge.get("full_context", {}).get("feedback")637 feedback_rag = judge.get("rag", {}).get("feedback")638 if feedback_no or feedback_rag:639 html_parts.append("<p class='metric-disclaimer'>LLM judge feedback:</p>")640 html_parts.append("<ul>")641 if feedback_no:642 html_parts.append(f"<li>Full context: {feedback_no}</li>")643 if feedback_rag:644 html_parts.append(f"<li>RAG: {feedback_rag}</li>")645 html_parts.append("</ul>")646 html_parts.append(647 "<p class='metric-disclaimer'>Scores are heuristic: embedding-based metrics use OpenAI or hashed fallback vectors, and the LLM judge relies on prompting best effort. "648 "Use them for rapid feedback rather than formal evaluation.</p>"649 )650 html_parts.append("</div>")651 return "".join(html_parts)652 653 654def format_context_markdown(retrieved: Sequence[Chunk]) -> str:655 if not retrieved:656 return "_No supporting context retrieved yet. Upload richer documents or rephrase the question._"657 parts: List[str] = []658 for idx, chunk in enumerate(retrieved, start=1):659 score_display = f"{chunk.score:.2f}"660 parts.append(661 f"**Chunk {idx} — {chunk.source} (similarity {score_display})**\n\n{chunk.text.strip()}"662 )663 return "\n\n---\n\n".join(parts)664 665 666def prepend_warnings(markdown: str, errors: Sequence[str]) -> str:667 if not errors:668 return markdown669 bullet_list = "\n".join(f"- {message}" for message in errors)670 warning_block = f"**Warnings while reading your files:**\n{bullet_list}\n\n"671 return warning_block + (markdown or "")672 673 674class EmbeddingService:675 """Utility for generating embeddings with OpenAI or a hashed fallback."""676 677 def __init__(self, existing_client: Optional[Any] = None) -> None:678 self.model = os.getenv("OPENAI_EMBED_MODEL", "text-embedding-3-large")679 self.dim = max(64, int(os.getenv("FALLBACK_EMBED_DIM", "512")))680 self.client = existing_client681 if self.client is None:682 api_key = os.getenv("OPENAI_API_KEY")683 if api_key:684 try:685 from openai import OpenAI686 687 self.client = OpenAI(api_key=api_key)688 except Exception:689 self.client = None690 self.fallback_mode = self.client is None691 692 def embed_texts(self, texts: Sequence[str]) -> List[List[float]]:693 cleaned = [(text or "").strip() for text in texts if text is not None]694 if not cleaned:695 return []696 normalized_inputs = [text if text else " " for text in cleaned]697 if self.client is not None:698 try:699 response = self.client.embeddings.create(model=self.model, input=normalized_inputs)700 embeddings = [record.embedding for record in response.data]701 # Reset fallback marker in case a prior request failed but this one succeeds702 self.fallback_mode = False703 return embeddings704 except Exception:705 # Deactivate the failing client to avoid repeated errors and fall back to hashing706 self.client = None707 self.fallback_mode = True708 return [self._fallback_embedding(text) for text in normalized_inputs]709 710 def embed_one(self, text: str) -> Optional[List[float]]:711 embeddings = self.embed_texts([text])712 return embeddings[0] if embeddings else None713 714 def _fallback_embedding(self, text: str) -> List[float]:715 vector = [0.0] * self.dim716 tokens = tokenize(text)717 if not tokens:718 return vector719 for token in tokens:720 idx = hash(token) % self.dim721 vector[idx] += 1.0722 norm = math.sqrt(sum(value * value for value in vector))723 if norm:724 vector = [value / norm for value in vector]725 return vector726 727 728class LLMService:729 """Simple wrapper that prefers OpenAI if configured, otherwise uses heuristics."""730 731 def __init__(self) -> None:732 self.client = None733 self.model = os.getenv("OPENAI_MODEL", "gpt-4o-mini")734 self.temperature = float(os.getenv("LLM_TEMPERATURE", "0.2"))735 api_key = os.getenv("OPENAI_API_KEY")736 if api_key:737 try:738 from openai import OpenAI739 740 self.client = OpenAI(api_key=api_key)741 except Exception:742 self.client = None743 744 def answer(self, question: str, context: Optional[str] = None) -> str:745 question = (question or "").strip()746 if not question:747 return "No question provided."748 if self.client:749 system_prompt = (750 "You are an assistant that compares retrieval augmented generation to plain LLM answers. "751 "Always ground responses in the supplied context when available. If context is missing, answer from "752 "general knowledge and call out uncertainty explicitly."753 )754 user_content: List[str] = []755 if context:756 user_content.append("Context:\n" + context.strip())757 user_content.append("Question:\n" + question)758 user_content.append("Answer:")759 try:760 response = self.client.responses.create(761 model=self.model,762 input=[763 {"role": "system", "content": system_prompt},764 {"role": "user", "content": "\n\n".join(user_content)},765 ],766 temperature=self.temperature,767 )768 return response.output_text.strip()769 except Exception as exc:770 return f"[LLM call failed: {exc}]"771 return self._fallback_answer(question, context)772 773 def _fallback_answer(self, question: str, context: Optional[str] = None) -> str:774 if context:775 sentences = split_into_sentences(context)776 if not sentences:777 return (778 "Context grounded answer (heuristic): The retrieved context did not contain enough readable sentences."779 )780 keywords = set(tokenize(question))781 scored: List[Tuple[int, int, str]] = []782 for sentence in sentences:783 tokens = tokenize(sentence)784 if not tokens:785 continue786 overlap = sum(1 for token in tokens if token in keywords)787 scored.append((overlap, -len(tokens), sentence))788 if not scored:789 return "Context grounded answer (heuristic): No overlapping signals were found in the retrieved context."790 scored.sort()791 top_sentences = [entry[2] for entry in scored[-3:]]792 snippet = " ".join(top_sentences).strip()793 if not snippet:794 snippet = (795 "The uploaded material does not include details that match the question closely, so this answer may be incomplete."796 )797 return (798 "Context grounded answer (heuristic, no external LLM configured): "799 f"{snippet}"800 )801 return (802 "Speculative answer without retrieval (heuristic): "803 f"I do not have supporting documents for '{question}', so this guess may hallucinate."804 )805 806 807LLM = LLMService()808EMBEDDER = EmbeddingService(existing_client=LLM.client)809 810 811async def async_llm_judge(812 client: Any,813 question: str,814 answer: str,815 reference: str,816 context: str,817) -> Dict[str, Any]:818 if client is None:819 return {820 "status": "unavailable",821 "accuracy": None,822 "completeness": None,823 "relevance": None,824 "feedback": "LLM judge unavailable (no API key configured).",825 }826 827 prompt = textwrap.dedent(828 f"""829 You are evaluating answers in a Retrieval-Augmented Generation (RAG) demo.830 Rate the given answer from 0 to 1 on accuracy, completeness, and relevance, using the reference answer and retrieved context.831 Return a compact JSON dictionary with keys accuracy, completeness, relevance, and a short feedback note.832 833 Context:834 {context}835 836 Reference answer (optional):837 {reference or '[not provided]'}838 839 User question:840 {question}841 842 Answer to evaluate:843 {answer}844 """845 ).strip()846 847 try:848 loop = asyncio.get_event_loop()849 response = await loop.run_in_executor(850 None,851 lambda: client.responses.create(852 model=os.getenv("OPENAI_JUDGE_MODEL", "gpt-4o-mini"),853 input=[854 {855 "role": "system",856 "content": "Return valid JSON only."857 },858 {859 "role": "user",860 "content": prompt,861 },862 ],863 temperature=0.2,864 ),865 )866 raw = response.output_text.strip()867 data = parse_json_block(raw)868 return {869 "status": "ok",870 "accuracy": float(data.get("accuracy", 0.0)),871 "completeness": float(data.get("completeness", 0.0)),872 "relevance": float(data.get("relevance", 0.0)),873 "feedback": data.get("feedback", ""),874 }875 except Exception as exc:876 raw_preview = (raw[:200] + "…") if "raw" in locals() and len(raw) > 200 else raw if "raw" in locals() else ""877 return {878 "status": "error",879 "accuracy": None,880 "completeness": None,881 "relevance": None,882 "feedback": f"Judge error: {exc}. Raw: {raw_preview}",883 }884 885 886METRIC_DETAILS_MD = """887**Hallucination Score** — 1 minus the support score. Lower is better; 0 means every answer sentence has strong embedding support in the uploaded sources.888 889**Support Score** — Average cosine similarity between each answer sentence embedding and its best matching sentence embedding from the uploaded documents. Higher is better and indicates factual grounding.890 891**Semantic Similarity** — Cosine similarity between the answer embedding and the provided reference answer embedding. Higher is better. This metric is only available when you supply a ground-truth answer.892 893**Retrieval Effectiveness** — Compares how well the retrieved chunk embeddings match the question embedding relative to the average chunk in the corpus. Higher is better; 0.5 means retrieval is performing on par with a random chunk.894 895**Precision / Recall / MRR / nDCG** — Ranking metrics derived from embedding similarity that estimate how well the retriever surfaces the most relevant chunks. Higher is better.896 897When the OpenAI embedding API is unavailable, the app falls back to hashed lexical vectors so scores remain approximate but still indicative. If the LLM judge cannot be reached, its scores are replaced with a warning.898 899The full-context baseline feeds the concatenated uploads directly to the LLM (trimmed to the `FULL_CONTEXT_CHAR_LIMIT` environment variable, default 12,000 characters) so you can see what retrieval adds over simply pasting the entire document set into the prompt.900"""901 902 903CUSTOM_CSS = """904.metric-table {905 width: 100%;906 border-collapse: collapse;907}908.metric-table th, .metric-table td {909 border: 1px solid #ddd;910 padding: 0.4rem 0.6rem;911 text-align: left;912}913.metric-table th {914 background-color: #f8f8f8;915}916.metric-summary ul {917 margin-top: 0.4rem;918}919.metric-summary li {920 margin-bottom: 0.2rem;921}922.metric-disclaimer {923 font-size: 0.85rem;924 color: #555;925}926.answer-section {927 display: flex;928 gap: 1rem;929 flex-wrap: wrap;930}931.answer-card {932 background-color: #f9fafc;933 border: 1px solid #d8dee9;934 border-radius: 8px;935 padding: 0.8rem;936 flex: 1 1 280px;937 box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.6);938}939.answer-card h3 {940 margin-top: 0;941}942.answer-card p {943 white-space: pre-wrap;944}945.comparison-note {946 font-weight: 600;947}948"""949 950 951async def compare(question: str, files, reference_answer: str):952 question = (question or "").strip()953 if not question:954 message = "Please provide a question to run the comparison."955 return (956 message,957 message,958 "_Awaiting question and documents._",959 "<p>Enter a question and upload documents to see the comparison.</p>",960 )961 962 documents, file_errors = read_uploaded_files(files)963 warnings = list(file_errors)964 if not documents:965 no_docs_message = (966 "No documents were loaded. Upload at least one text-friendly file (.txt, .md, .json, .pdf, .csv)."967 )968 if EMBEDDER.fallback_mode:969 warnings.append(970 "OpenAI embedding API not configured; using hashed fallback vectors until an API key is provided."971 )972 warnings.append(no_docs_message)973 full_context_answer = LLM.answer(question, context=None)974 rag_answer = LLM.answer(question, context=None)975 metrics_placeholder = "<p>Upload documents to compute grounding metrics.</p>"976 context_markdown = prepend_warnings("_No context available._", warnings)977 return full_context_answer, rag_answer, context_markdown, metrics_placeholder978 979 full_context_text, truncated_full_context = assemble_full_context(documents, max_chars=FULL_CONTEXT_CHAR_LIMIT)980 if truncated_full_context and FULL_CONTEXT_CHAR_LIMIT:981 warnings.append(982 f"Full-context prompt truncated to {FULL_CONTEXT_CHAR_LIMIT} characters to stay within model limits."983 )984 985 all_chunks = build_corpus_chunks(documents)986 chunk_texts = [chunk.text for chunk in all_chunks]987 chunk_embeddings = EMBEDDER.embed_texts(chunk_texts) if chunk_texts else []988 for index, chunk in enumerate(all_chunks):989 embedding = chunk_embeddings[index] if index < len(chunk_embeddings) else None990 chunk.embedding = embedding991 992 question_embedding = EMBEDDER.embed_one(question)993 retrieved: List[Chunk] = []994 vector_retrieval_used = False995 vector_attempted = False996 997 if (998 chromadb is not None999 and Settings is not None1000 and chunk_embeddings1001 and question_embedding is not None1002 ):1003 vector_attempted = True1004 try:1005 if hasattr(chromadb, "EphemeralClient"):1006 client = chromadb.EphemeralClient()1007 else:1008 client = chromadb.Client(1009 Settings(1010 anonymized_telemetry=False,1011 )1012 )1013 collection = client.create_collection(1014 name=f"rag-session-{uuid.uuid4()}",1015 metadata={"hnsw:space": "cosine"},1016 )1017 ids = [f"chunk-{idx}" for idx in range(len(all_chunks))]1018 collection.add(1019 ids=ids,1020 embeddings=chunk_embeddings,1021 documents=chunk_texts,1022 metadatas=[{"source": chunk.source} for chunk in all_chunks],1023 )1024 n_results = min(3, len(all_chunks))1025 results = collection.query(query_embeddings=[question_embedding], n_results=n_results)1026 id_lookup: Dict[str, Chunk] = {id_value: all_chunks[idx] for idx, id_value in enumerate(ids)}1027 retrieved_ids = results.get("ids", [[]])1028 for idx, chunk_id in enumerate(retrieved_ids[0] if retrieved_ids else []):1029 base_chunk = id_lookup.get(chunk_id)1030 if base_chunk is None:1031 continue1032 score = 0.01033 if question_embedding and base_chunk.embedding:1034 score = cosine_similarity_dense(question_embedding, base_chunk.embedding)1035 retrieved.append(1036 Chunk(1037 text=base_chunk.text,1038 source=base_chunk.source,1039 tokens=base_chunk.tokens,1040 score=score,1041 embedding=base_chunk.embedding,1042 uid=base_chunk.uid,1043 )1044 )1045 vector_retrieval_used = bool(retrieved)1046 except Exception as exc:1047 warnings.append(f"Chroma retrieval failed ({exc}); falling back to lexical search.")1048 else:1049 if chromadb is None or Settings is None:1050 warnings.append("Chroma not installed; using lexical retrieval as fallback.")1051 elif question_embedding is None:1052 warnings.append("Could not compute question embedding; using lexical retrieval as fallback.")1053 elif not chunk_embeddings:1054 warnings.append("No chunks were embedded; using lexical retrieval as fallback.")1055 1056 if not vector_retrieval_used:1057 if vector_attempted and not retrieved:1058 warnings.append("Vector retrieval returned no matches; using lexical retrieval as fallback.")1059 retrieved = retrieve_chunks_lexical(question, all_chunks, top_k=3)1060 1061 if EMBEDDER.fallback_mode:1062 warnings.append(1063 "OpenAI embedding API not configured; using hashed fallback vectors until an API key is provided."1064 )1065 1066 context_text = "\n\n".join(chunk.text for chunk in retrieved)1067 context_sections: List[str] = []1068 if context_text:1069 context_sections.append("Retrieved context:\n" + context_text)1070 if full_context_text:1071 context_sections.append("Full document context:\n" + full_context_text)1072 context_for_judge = (1073 "\n\n".join(context_sections)1074 if context_sections1075 else "No context was available for this question."1076 )1077 1078 full_context_answer = LLM.answer(1079 question,1080 context=full_context_text if full_context_text else None,1081 )1082 rag_answer = LLM.answer(question, context=context_text if context_text else None)1083 1084 reference_clean = (reference_answer or "").strip()1085 metrics = evaluate_answers(1086 question=question,1087 documents=documents,1088 all_chunks=all_chunks,1089 retrieved_chunks=retrieved,1090 full_context_answer=full_context_answer,1091 rag_answer=rag_answer,1092 reference_answer=reference_clean,1093 embedder=EMBEDDER,1094 question_embedding=question_embedding,1095 )1096 judge_tasks = [1097 asyncio.create_task(1098 async_llm_judge(1099 LLM.client,1100 question,1101 full_context_answer,1102 reference_clean,1103 context_for_judge,1104 )1105 ),1106 asyncio.create_task(1107 async_llm_judge(1108 LLM.client,1109 question,1110 rag_answer,1111 reference_clean,1112 context_for_judge,1113 )1114 ),1115 ]1116 judge_no, judge_rag = await asyncio.gather(*judge_tasks)1117 judge_results = {"full_context": judge_no, "rag": judge_rag}1118 for label, result in judge_results.items():1119 status = result.get("status")1120 if status == "unavailable":1121 if "LLM judge unavailable" not in " ".join(warnings):1122 warnings.append(1123 "LLM judge unavailable (no API key configured); judge scores omitted."1124 )1125 elif status == "error":1126 warnings.append(result.get("feedback", "LLM judge encountered an error."))1127 metrics["judge"] = judge_results1128 metrics_html = format_metrics_html(metrics, reference_available=bool(reference_clean))1129 context_markdown = format_context_markdown(retrieved)1130 context_markdown = prepend_warnings(context_markdown, warnings)1131 1132 return full_context_answer, rag_answer, context_markdown, metrics_html1133 1134 1135with gr.Blocks(title="RAG vs Full-Context Comparator", analytics_enabled=False) as demo:1136 gr.Markdown("# RAG vs Full-Context Comparator")1137 gr.HTML(f"<style>{CUSTOM_CSS}</style>")1138 gr.Markdown(1139 "Upload documents, ask a question, and compare a full-document LLM answer against a retrieval-augmented answer. "1140 "Vector retrieval runs on OpenAI text-embedding-3-large + Chroma, with automatic evaluations highlighting grounding, hallucination risk, and retrieval quality."1141 )1142 1143 with gr.Row():1144 question_input = gr.Textbox(1145 label="Question",1146 placeholder="e.g. What are the main obligations in the contract?",1147 lines=2,1148 )1149 reference_input = gr.Textbox(1150 label="Reference answer (optional)",1151 placeholder="Provide a ground-truth answer to compare against.",1152 lines=4,1153 )1154 1155 document_input = gr.File(1156 label="Upload documents",1157 file_count="multiple",1158 file_types=[".txt", ".md", ".markdown", ".json", ".pdf", ".csv"],1159 )1160 1161 run_button = gr.Button("Run comparison", variant="primary")1162 1163 gr.Markdown("## Answer comparison")1164 with gr.Row(elem_classes=["answer-section"]):1165 with gr.Group(elem_classes=["answer-card"]):1166 gr.Markdown("### Full-context answer (entire uploads)")1167 full_context_output = gr.Markdown(show_label=False)1168 with gr.Group(elem_classes=["answer-card"]):1169 gr.Markdown("### RAG answer (with retrieved context)")1170 rag_output = gr.Markdown(show_label=False)1171 1172 context_output = gr.Markdown(label="Retrieved context")1173 metrics_output = gr.HTML(label="Evaluation metrics")1174 1175 with gr.Accordion("What do these metrics mean?", open=False):1176 gr.Markdown(METRIC_DETAILS_MD)1177 1178 run_button.click(1179 fn=compare,1180 inputs=[question_input, document_input, reference_input],1181 outputs=[full_context_output, rag_output, context_output, metrics_output],1182 )1183 1184 1185if __name__ == "__main__":1186 demo.launch()1187