Israelbliz/User-Modeling-Agent
0
1"""Full evaluation harness — measures everything the rubric measures.2 3Runs Task A and Task B (warm, cold-start, cross-domain) across N users,4computes all automated metrics (RMSE, ROUGE-L, BERTScore, Hit Rate, NDCG)5plus LLM-as-judge proxies for the human-eval dimensions (behavioral6fidelity, contextual relevance, bridge quality), and prints a structured7report.8 9This is the baseline. Run it before any experimental change to lock the10current numbers; re-run after to see what moved.11 12Usage:13 # Default — 30/30/10/15 users14 python -m scripts.run_eval15 16 # Quick smoke test — 5/5/3/5 users (~3 min)17 python -m scripts.run_eval --smoke18 19 # Custom sample sizes20 python -m scripts.run_eval --n-task-a 50 --n-warm 50 --n-cold 20 --n-cross 2521 22 # Skip BERTScore (slow first-time, downloads ~400MB)23 python -m scripts.run_eval --no-bertscore24 25 # Skip LLM-judge calls (faster, automated metrics only)26 python -m scripts.run_eval --no-judges27 28 # Save the report to a file (markdown)29 python -m scripts.run_eval --out reports/baseline_stage2b.md30"""31from __future__ import annotations32 33import argparse34import json35import logging36import time37from dataclasses import dataclass, field38from datetime import datetime39from pathlib import Path40from typing import Optional41 42import pandas as pd43 44from core.config import settings45from core.llm import LLMClient46from core.nigerian import naija_persona_examples47from core.persona import PersonaEngine, UserPersona48from eval.metrics import (49 rmse, mae, rouge_l, bertscore_f1,50 ndcg_at_k, hit_rate_at_k, mean_skipping_nan,51)52from eval.judges import (53 judge_behavioral_fidelity, judge_contextual_relevance,54 judge_bridge_quality, title_quality_rate, domain_coverage,55)56from task_a_user_modeling.agent import ImpersonationAgent, ItemInput57from task_b_recommender.agent import RecommendationAgent58 59logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")60log = logging.getLogger(__name__)61 62 63# ──────────────────────────────────────────────────────────────────────────────64# Result containers65# ──────────────────────────────────────────────────────────────────────────────66 67@dataclass68class TaskAResult:69 n_users: int = 070 rmse: float = float("nan")71 mae: float = float("nan")72 rouge_l: float = float("nan")73 bertscore_f1: float = float("nan")74 behavioral_fidelity: float = float("nan")75 n_failed: int = 076 raw: list[dict] = field(default_factory=list)77 78 79@dataclass80class TaskBModeResult:81 mode: str = ""82 n_users: int = 083 hit_rate_at_10: float = float("nan")84 ndcg_at_10: float = float("nan")85 title_quality: float = float("nan")86 domain_coverage_avg: float = float("nan")87 contextual_relevance: float = float("nan")88 bridge_quality: float = float("nan") # cross-domain only89 n_failed: int = 090 raw: list[dict] = field(default_factory=list)91 92 93@dataclass94class FullEvalReport:95 provider: str = ""96 started_at: str = ""97 completed_at: str = ""98 elapsed_seconds: float = 0.099 task_a: TaskAResult = field(default_factory=TaskAResult)100 task_b_warm: TaskBModeResult = field(default_factory=TaskBModeResult)101 task_b_cold_start: TaskBModeResult = field(default_factory=TaskBModeResult)102 task_b_cross_domain: TaskBModeResult = field(default_factory=TaskBModeResult)103 104 105# ──────────────────────────────────────────────────────────────────────────────106# User sampling107# ──────────────────────────────────────────────────────────────────────────────108 109def _load_reviews() -> pd.DataFrame:110 path = settings.processed_dir / "reviews.parquet"111 if not path.exists():112 raise SystemExit(f"Reviews file not found at {path}. Run prepare_data.py first.")113 df = pd.read_parquet(path)114 log.info(f"Loaded {len(df):,} reviews")115 return df116 117 118def _sample_users_with_test_reviews(reviews: pd.DataFrame, n: int,119 require_min_train: int = 3,120 require_min_domains: int = 1,121 seed: int = 42) -> list[str]:122 """Sample N users who have both train history and test items.123 124 Constraints applied per user:125 - At least `require_min_train` training reviews (enough for a persona)126 - At least `require_min_domains` distinct domains in training127 - At least 1 held-out test item128 """129 train = reviews[reviews["split"] == "train"]130 test = reviews[reviews["split"] == "test"]131 users_with_test = set(test["user_id"].unique())132 133 counts = (train.groupby("user_id")134 .agg(n_train=("rating", "size"),135 n_domains=("domain", "nunique"))136 .reset_index())137 eligible = counts[138 (counts["user_id"].isin(users_with_test))139 & (counts["n_train"] >= require_min_train)140 & (counts["n_domains"] >= require_min_domains)141 ]142 if len(eligible) == 0:143 raise SystemExit("No eligible users found.")144 if len(eligible) < n:145 log.warning(f"Only {len(eligible)} eligible users (requested {n}); using all")146 n = len(eligible)147 sample = eligible.sample(n=n, random_state=seed)148 return sample["user_id"].tolist()149 150 151def _sample_cross_domain_users(reviews: pd.DataFrame, n: int, seed: int = 43) -> list[str]:152 """Sample N users with single-domain training history (cross-domain test targets).153 154 For cross-domain mode we want users whose training history is in 1 domain so155 we can recommend in the OTHER domains and measure if anything they engaged156 with in those other domains shows up.157 """158 train = reviews[reviews["split"] == "train"]159 test = reviews[reviews["split"] == "test"]160 users_with_test = set(test["user_id"].unique())161 162 counts = (train.groupby("user_id")163 .agg(n_train=("rating", "size"),164 n_domains=("domain", "nunique"))165 .reset_index())166 # users with single-domain training, at least 3 train reviews, with test items167 eligible = counts[168 (counts["user_id"].isin(users_with_test))169 & (counts["n_train"] >= 3)170 & (counts["n_domains"] == 1)171 ]172 if len(eligible) == 0:173 raise SystemExit("No eligible cross-domain users found.")174 if len(eligible) < n:175 log.warning(f"Only {len(eligible)} cross-domain users (requested {n})")176 n = len(eligible)177 return eligible.sample(n=n, random_state=seed)["user_id"].tolist()178 179 180# ──────────────────────────────────────────────────────────────────────────────181# Task A evaluation182# ──────────────────────────────────────────────────────────────────────────────183 184def run_task_a(reviews: pd.DataFrame, n_users: int, *,185 with_bertscore: bool, with_judges: bool,186 persona_engine: PersonaEngine, agent: ImpersonationAgent,187 judge_llm: LLMClient) -> TaskAResult:188 log.info(f"════ Task A evaluation: {n_users} users ════")189 user_ids = _sample_users_with_test_reviews(reviews, n_users)190 train = reviews[reviews["split"] == "train"]191 test = reviews[reviews["split"] == "test"]192 193 result = TaskAResult(n_users=0)194 predicted_ratings, actual_ratings = [], []195 predicted_reviews, actual_reviews = [], []196 fidelity_scores = []197 198 for i, user_id in enumerate(user_ids, 1):199 log.info(f" [{i}/{len(user_ids)}] Task A user={user_id[:12]}...")200 try:201 persona = persona_engine.from_dataframe(user_id, train)202 persona = persona_engine.enrich(persona)203 except Exception as e:204 log.warning(f" Persona build failed: {e}; skipping user")205 result.n_failed += 1206 continue207 208 user_test = test[test["user_id"] == user_id]209 if user_test.empty:210 result.n_failed += 1211 continue212 # Pick the first test review as the target213 target_row = user_test.iloc[0]214 target_item_id = target_row["parent_asin"]215 target_title = target_row["title"] if "title" in target_row else "(unknown)"216 target_domain = target_row["domain"]217 actual_rating = float(target_row["rating"])218 actual_review = str(target_row["text"])219 220 # Try to look up real item metadata (description) for the agent221 items_path = settings.processed_dir / "items.parquet"222 item_description = ""223 item_categories = ""224 if items_path.exists():225 items_df = pd.read_parquet(items_path)226 match = items_df[items_df["parent_asin"] == target_item_id]227 if not match.empty:228 row = match.iloc[0]229 target_title = str(row.get("title") or target_title)230 item_description = str(row.get("description") or "")231 item_categories = str(row.get("categories") or "")232 233 item_input = ItemInput(234 parent_asin=target_item_id,235 title=target_title,236 description=item_description,237 categories=item_categories,238 domain=target_domain,239 )240 241 try:242 output = agent.run(persona, item=item_input)243 pred_rating = float(output.rating)244 pred_review = str(output.review)245 except Exception as e:246 log.warning(f" Generation failed: {e}; skipping user")247 result.n_failed += 1248 continue249 250 # Auto metrics251 predicted_ratings.append(pred_rating)252 actual_ratings.append(actual_rating)253 predicted_reviews.append(pred_review)254 actual_reviews.append(actual_review)255 256 # LLM-judge257 fidelity = None258 if with_judges:259 try:260 score = judge_behavioral_fidelity(261 judge_llm, persona.to_prompt_block(),262 target_title, target_domain,263 pred_rating, pred_review,264 actual_rating, actual_review,265 )266 fidelity_scores.append(score.score)267 fidelity = score.score268 except Exception as e:269 log.warning(f" Judge failed: {e}")270 271 result.raw.append({272 "user_id": user_id,273 "item_id": target_item_id,274 "domain": target_domain,275 "actual_rating": actual_rating,276 "predicted_rating": pred_rating,277 "rouge_l": rouge_l(pred_review, actual_review),278 "fidelity": fidelity,279 })280 result.n_users += 1281 282 # Aggregate283 if predicted_ratings:284 result.rmse = rmse(predicted_ratings, actual_ratings)285 result.mae = mae(predicted_ratings, actual_ratings)286 result.rouge_l = float(287 sum(rouge_l(c, r) for c, r in zip(predicted_reviews, actual_reviews))288 / len(predicted_reviews)289 )290 291 if with_bertscore and predicted_reviews:292 log.info(" Computing BERTScore (may download model on first use)...")293 try:294 result.bertscore_f1 = bertscore_f1(predicted_reviews, actual_reviews)295 except Exception as e:296 log.warning(f" BERTScore failed: {e}")297 298 if fidelity_scores:299 result.behavioral_fidelity = float(sum(fidelity_scores) / len(fidelity_scores))300 301 return result302 303 304# ──────────────────────────────────────────────────────────────────────────────305# Task B evaluation306# ──────────────────────────────────────────────────────────────────────────────307 308def run_task_b_warm(reviews: pd.DataFrame, n_users: int, *,309 with_judges: bool, persona_engine: PersonaEngine,310 agent: RecommendationAgent, judge_llm: LLMClient) -> TaskBModeResult:311 log.info(f"════ Task B warm evaluation: {n_users} users ════")312 user_ids = _sample_users_with_test_reviews(reviews, n_users, require_min_domains=1)313 train = reviews[reviews["split"] == "train"]314 test = reviews[reviews["split"] == "test"]315 316 result = TaskBModeResult(mode="warm", n_users=0)317 hit_rates, ndcgs, title_qualities = [], [], []318 coverages, relevance_scores = [], []319 320 for i, user_id in enumerate(user_ids, 1):321 log.info(f" [{i}/{len(user_ids)}] Task B warm user={user_id[:12]}...")322 try:323 persona = persona_engine.from_dataframe(user_id, train)324 persona = persona_engine.enrich(persona)325 recs = agent.run(persona, k=10, cross_domain=False)326 except Exception as e:327 log.warning(f" Failed: {e}")328 result.n_failed += 1329 continue330 if not recs:331 result.n_failed += 1332 continue333 334 recs_dicts = [r.as_dict() for r in recs]335 user_test = test[test["user_id"] == user_id]336 gt_ids = list(user_test["parent_asin"].unique())337 pred_ids = [r.item_id for r in recs]338 339 hr = hit_rate_at_k(pred_ids, gt_ids, k=10)340 nd = ndcg_at_k(pred_ids, gt_ids, k=10)341 tq = title_quality_rate(recs_dicts)342 # Domain coverage: did we span the user's known domains?343 known_domains = list(persona.domains) if persona.domains else []344 dc = domain_coverage(recs_dicts, known_domains) if known_domains else 0.0345 normalized_dc = dc / max(1, len(known_domains))346 347 hit_rates.append(hr)348 ndcgs.append(nd)349 title_qualities.append(tq)350 coverages.append(normalized_dc)351 352 relevance = None353 if with_judges:354 try:355 score = judge_contextual_relevance(356 judge_llm, persona.to_prompt_block(), recs_dicts, mode="warm",357 )358 relevance_scores.append(score.score)359 relevance = score.score360 except Exception as e:361 log.warning(f" Judge failed: {e}")362 363 result.raw.append({364 "user_id": user_id,365 "hit_rate": hr,366 "ndcg": nd,367 "title_quality": tq,368 "domain_coverage": normalized_dc,369 "relevance": relevance,370 })371 result.n_users += 1372 373 if hit_rates:374 result.hit_rate_at_10 = float(sum(hit_rates) / len(hit_rates))375 result.ndcg_at_10 = float(sum(ndcgs) / len(ndcgs))376 result.title_quality = float(sum(title_qualities) / len(title_qualities))377 result.domain_coverage_avg = float(sum(coverages) / len(coverages))378 if relevance_scores:379 result.contextual_relevance = float(sum(relevance_scores) / len(relevance_scores))380 381 return result382 383 384def run_task_b_cold_start(n_personas: int, *,385 with_judges: bool, agent: RecommendationAgent,386 judge_llm: LLMClient) -> TaskBModeResult:387 """Cold-start eval uses synthetic Naija personas (no history).388 389 No held-out test items exist for synthetic personas, so we can't compute390 Hit Rate or NDCG — only title quality, domain coverage, and contextual391 relevance (LLM-judge).392 """393 log.info(f"════ Task B cold-start evaluation: {n_personas} personas ════")394 naija = naija_persona_examples()395 # Cycle through naija personas if n_personas > len396 personas_to_test = (naija * ((n_personas // len(naija)) + 1))[:n_personas]397 398 result = TaskBModeResult(mode="cold_start", n_users=0)399 title_qualities, coverages, relevance_scores = [], [], []400 401 for i, demo in enumerate(personas_to_test, 1):402 log.info(f" [{i}/{len(personas_to_test)}] Cold-start persona={demo['name']}")403 persona = UserPersona(404 user_id=f"cold_start_{i}",405 n_reviews=0, avg_rating=4.0, std_rating=0.5,406 avg_review_length=80.0, std_review_length=20.0,407 verified_rate=1.0, domains=["Books"], n_domains=1,408 rating_distribution={4: 0.6, 5: 0.3, 3: 0.1},409 top_terms=[],410 tone="", preferred_themes=demo["stated_preferences"],411 common_complaints=demo["deal_breakers"],412 voice_one_liner=demo["description"],413 history_samples=[],414 )415 try:416 recs = agent.run(persona, k=10, cross_domain=False)417 except Exception as e:418 log.warning(f" Generation failed: {e}")419 result.n_failed += 1420 continue421 if not recs:422 result.n_failed += 1423 continue424 425 recs_dicts = [r.as_dict() for r in recs]426 tq = title_quality_rate(recs_dicts)427 # Expect coverage across the 3 domains since cold-start often spans interests428 all_domains = ["Books", "Kindle_Store", "Movies_and_TV"]429 dc = domain_coverage(recs_dicts, all_domains)430 431 title_qualities.append(tq)432 coverages.append(dc)433 434 relevance = None435 if with_judges:436 try:437 score = judge_contextual_relevance(438 judge_llm, persona.to_prompt_block(), recs_dicts, mode="cold_start",439 )440 relevance_scores.append(score.score)441 relevance = score.score442 except Exception as e:443 log.warning(f" Judge failed: {e}")444 445 result.raw.append({446 "persona_name": demo["name"],447 "title_quality": tq,448 "domain_coverage": dc,449 "relevance": relevance,450 })451 result.n_users += 1452 453 if title_qualities:454 result.title_quality = float(sum(title_qualities) / len(title_qualities))455 result.domain_coverage_avg = float(sum(coverages) / len(coverages))456 if relevance_scores:457 result.contextual_relevance = float(sum(relevance_scores) / len(relevance_scores))458 459 return result460 461 462def run_task_b_cross_domain(reviews: pd.DataFrame, n_users: int, *,463 with_judges: bool, persona_engine: PersonaEngine,464 agent: RecommendationAgent,465 judge_llm: LLMClient) -> TaskBModeResult:466 log.info(f"════ Task B cross-domain evaluation: {n_users} users ════")467 user_ids = _sample_cross_domain_users(reviews, n_users)468 train = reviews[reviews["split"] == "train"]469 test = reviews[reviews["split"] == "test"]470 471 result = TaskBModeResult(mode="cross_domain", n_users=0)472 cross_hit_rates, title_qualities, coverages = [], [], []473 relevance_scores, bridge_scores = [], []474 475 for i, user_id in enumerate(user_ids, 1):476 log.info(f" [{i}/{len(user_ids)}] Task B cross-domain user={user_id[:12]}...")477 try:478 persona = persona_engine.from_dataframe(user_id, train)479 persona = persona_engine.enrich(persona)480 recs = agent.run(persona, k=10, cross_domain=True)481 except Exception as e:482 log.warning(f" Failed: {e}")483 result.n_failed += 1484 continue485 if not recs:486 result.n_failed += 1487 continue488 489 recs_dicts = [r.as_dict() for r in recs]490 # Cross-domain hit rate: test items in NEW domains (not in user's training)491 user_test = test[test["user_id"] == user_id]492 known = set(persona.domains)493 cross_gt = user_test[~user_test["domain"].isin(known)]494 if cross_gt.empty:495 # User has no test items in unknown domains; can't measure HR496 hr = float("nan")497 else:498 gt_ids = list(cross_gt["parent_asin"].unique())499 pred_ids = [r.item_id for r in recs]500 hr = hit_rate_at_k(pred_ids, gt_ids, k=10)501 502 tq = title_quality_rate(recs_dicts)503 all_domains = {"Books", "Kindle_Store", "Movies_and_TV"}504 expected_unknown = list(all_domains - known)505 dc = domain_coverage(recs_dicts, expected_unknown)506 normalized_dc = dc / max(1, len(expected_unknown))507 508 cross_hit_rates.append(hr)509 title_qualities.append(tq)510 coverages.append(normalized_dc)511 512 relevance = None513 bridge = None514 if with_judges:515 try:516 rscore = judge_contextual_relevance(517 judge_llm, persona.to_prompt_block(), recs_dicts,518 mode="cross_domain",519 )520 relevance_scores.append(rscore.score)521 relevance = rscore.score522 except Exception as e:523 log.warning(f" Relevance judge failed: {e}")524 try:525 bscore = judge_bridge_quality(526 judge_llm, persona.to_prompt_block(),527 list(persona.domains), recs_dicts,528 )529 bridge_scores.append(bscore.score)530 bridge = bscore.score531 except Exception as e:532 log.warning(f" Bridge judge failed: {e}")533 534 result.raw.append({535 "user_id": user_id,536 "known_domains": list(known),537 "cross_hit_rate": hr,538 "title_quality": tq,539 "domain_coverage": normalized_dc,540 "relevance": relevance,541 "bridge_quality": bridge,542 })543 result.n_users += 1544 545 if cross_hit_rates:546 result.hit_rate_at_10 = mean_skipping_nan(cross_hit_rates)547 if title_qualities:548 result.title_quality = float(sum(title_qualities) / len(title_qualities))549 result.domain_coverage_avg = float(sum(coverages) / len(coverages))550 if relevance_scores:551 result.contextual_relevance = float(sum(relevance_scores) / len(relevance_scores))552 if bridge_scores:553 result.bridge_quality = float(sum(bridge_scores) / len(bridge_scores))554 555 return result556 557 558# ──────────────────────────────────────────────────────────────────────────────559# Report formatting560# ──────────────────────────────────────────────────────────────────────────────561 562def _f(v: float, fmt: str = ".3f") -> str:563 """Format a metric — return 'n/a' for NaN."""564 import math565 if v is None or (isinstance(v, float) and math.isnan(v)):566 return " n/a"567 return f"{v:{fmt}}"568 569 570def format_report(report: FullEvalReport) -> str:571 lines = [572 "═" * 65,573 "NaijaTaste AI — Full Evaluation Report",574 f"Provider: {report.provider}",575 f"Started: {report.started_at}",576 f"Completed: {report.completed_at}",577 f"Elapsed: {report.elapsed_seconds:.1f}s ({report.elapsed_seconds/60:.1f} min)",578 "═" * 65,579 "",580 f"TASK A — User Modeling (N={report.task_a.n_users}, failed={report.task_a.n_failed})",581 f" Rating accuracy (RMSE): {_f(report.task_a.rmse)}",582 f" Rating accuracy (MAE): {_f(report.task_a.mae)}",583 f" Review text (ROUGE-L F1): {_f(report.task_a.rouge_l)}",584 f" Review text (BERTScore F1): {_f(report.task_a.bertscore_f1)}",585 f" Behavioral fidelity (judge 1-5): {_f(report.task_a.behavioral_fidelity, '.2f')}",586 "",587 "TASK B — Recommendation",588 "",589 f" Warm mode (N={report.task_b_warm.n_users}, failed={report.task_b_warm.n_failed})",590 f" Hit Rate@10: {_f(report.task_b_warm.hit_rate_at_10)}",591 f" NDCG@10: {_f(report.task_b_warm.ndcg_at_10)}",592 f" Title quality (real titles %): {_f(report.task_b_warm.title_quality)}",593 f" Domain coverage (known): {_f(report.task_b_warm.domain_coverage_avg)}",594 f" Contextual relevance (judge): {_f(report.task_b_warm.contextual_relevance, '.2f')}",595 "",596 f" Cold-start (N={report.task_b_cold_start.n_users}, failed={report.task_b_cold_start.n_failed})",597 f" Title quality (real titles %): {_f(report.task_b_cold_start.title_quality)}",598 f" Domain coverage (of 3): {_f(report.task_b_cold_start.domain_coverage_avg, '.1f')}",599 f" Contextual relevance (judge): {_f(report.task_b_cold_start.contextual_relevance, '.2f')}",600 "",601 f" Cross-domain (N={report.task_b_cross_domain.n_users}, failed={report.task_b_cross_domain.n_failed})",602 f" Cross-domain Hit Rate@10: {_f(report.task_b_cross_domain.hit_rate_at_10)}",603 f" Title quality (real titles %): {_f(report.task_b_cross_domain.title_quality)}",604 f" Domain coverage (unknown): {_f(report.task_b_cross_domain.domain_coverage_avg)}",605 f" Contextual relevance (judge): {_f(report.task_b_cross_domain.contextual_relevance, '.2f')}",606 f" Bridge quality (judge): {_f(report.task_b_cross_domain.bridge_quality, '.2f')}",607 "",608 "═" * 65,609 "Higher is better for: ROUGE-L, BERTScore, Hit Rate, NDCG,",610 " Title quality, Domain coverage, all judge scores.",611 "Lower is better for: RMSE, MAE.",612 "═" * 65,613 ]614 return "\n".join(lines)615 616 617# ──────────────────────────────────────────────────────────────────────────────618# Main619# ──────────────────────────────────────────────────────────────────────────────620 621def main():622 ap = argparse.ArgumentParser()623 ap.add_argument("--smoke", action="store_true",624 help="Quick smoke test: 5/5/3/5 users (~3 min)")625 ap.add_argument("--n-task-a", type=int, default=30)626 ap.add_argument("--n-warm", type=int, default=30)627 ap.add_argument("--n-cold", type=int, default=10)628 ap.add_argument("--n-cross", type=int, default=15)629 ap.add_argument("--no-bertscore", action="store_true",630 help="Skip BERTScore (first run downloads ~400MB)")631 ap.add_argument("--no-judges", action="store_true",632 help="Skip LLM-judge calls (faster, automated metrics only)")633 ap.add_argument("--out", type=str, default=None,634 help="Save report markdown to this file path")635 ap.add_argument("--json-out", type=str, default=None,636 help="Save raw per-user results as JSON to this path")637 args = ap.parse_args()638 639 if args.smoke:640 args.n_task_a, args.n_warm, args.n_cold, args.n_cross = 5, 5, 3, 5641 642 started = time.time()643 started_dt = datetime.now()644 645 reviews = _load_reviews()646 log.info(f"Provider: {settings.llm_provider}")647 log.info(f"Sample sizes: Task A={args.n_task_a}, "648 f"Warm={args.n_warm}, Cold={args.n_cold}, Cross={args.n_cross}")649 log.info(f"BERTScore: {'OFF' if args.no_bertscore else 'ON'}, "650 f"Judges: {'OFF' if args.no_judges else 'ON'}")651 652 # Build shared resources (load once, reuse across tasks)653 persona_engine = PersonaEngine()654 task_a_agent = ImpersonationAgent()655 task_b_agent = RecommendationAgent()656 judge_llm = LLMClient()657 658 report = FullEvalReport(659 provider=settings.llm_provider,660 started_at=started_dt.strftime("%Y-%m-%d %H:%M:%S"),661 )662 663 # ── Task A ───────────────────────────────────────────────────────────664 if args.n_task_a > 0:665 report.task_a = run_task_a(666 reviews, args.n_task_a,667 with_bertscore=not args.no_bertscore,668 with_judges=not args.no_judges,669 persona_engine=persona_engine,670 agent=task_a_agent,671 judge_llm=judge_llm,672 )673 674 # ── Task B warm ──────────────────────────────────────────────────────675 if args.n_warm > 0:676 report.task_b_warm = run_task_b_warm(677 reviews, args.n_warm,678 with_judges=not args.no_judges,679 persona_engine=persona_engine,680 agent=task_b_agent,681 judge_llm=judge_llm,682 )683 684 # ── Task B cold-start ────────────────────────────────────────────────685 if args.n_cold > 0:686 report.task_b_cold_start = run_task_b_cold_start(687 args.n_cold,688 with_judges=not args.no_judges,689 agent=task_b_agent,690 judge_llm=judge_llm,691 )692 693 # ── Task B cross-domain ──────────────────────────────────────────────694 if args.n_cross > 0:695 report.task_b_cross_domain = run_task_b_cross_domain(696 reviews, args.n_cross,697 with_judges=not args.no_judges,698 persona_engine=persona_engine,699 agent=task_b_agent,700 judge_llm=judge_llm,701 )702 703 completed = time.time()704 report.completed_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")705 report.elapsed_seconds = completed - started706 707 # ── Print + save ─────────────────────────────────────────────────────708 text = format_report(report)709 print("\n" + text + "\n")710 711 if args.out:712 out_path = Path(args.out)713 out_path.parent.mkdir(parents=True, exist_ok=True)714 out_path.write_text(text)715 log.info(f"Report saved to {out_path}")716 717 if args.json_out:718 json_path = Path(args.json_out)719 json_path.parent.mkdir(parents=True, exist_ok=True)720 json_blob = {721 "provider": report.provider,722 "started_at": report.started_at,723 "completed_at": report.completed_at,724 "elapsed_seconds": report.elapsed_seconds,725 "task_a": {**report.task_a.__dict__},726 "task_b_warm": {**report.task_b_warm.__dict__},727 "task_b_cold_start": {**report.task_b_cold_start.__dict__},728 "task_b_cross_domain": {**report.task_b_cross_domain.__dict__},729 }730 json_path.write_text(json.dumps(json_blob, indent=2, default=str))731 log.info(f"Raw results saved to {json_path}")732 733 734if __name__ == "__main__":735 main()736