Raniahossam33/knowledge-drift-experiments

1"""2Data Diagnostic: Answers all reviewer questions about the dataset.3Run: python data_diagnostic.py4"""5import json6from collections import Counter, defaultdict7import numpy as np8 9# Load both datasets10print("=" * 70)11print("  DATA DIAGNOSTIC — Answering All Reviewer Questions")12print("=" * 70)13 14with open("data/knowledge_drift_unified_tier1.json") as f:15    tier1 = json.load(f)16samples = tier1["samples"]17print(f"\nTier 1: {len(samples)} samples")18 19# ============================================================20# Q1: Class imbalance per model21# ============================================================22print("\n" + "=" * 70)23print("  Q1: CLASS IMBALANCE PER MODEL")24print("=" * 70)25models = ["llama2", "mistral", "llama31", "qwen25", "gemma2"]26for m in models:27    key = f"is_drifted_{m}"28    d = sum(1 for s in samples if s.get(key, False))29    s_ = len(samples) - d30    ratio = d / s_ if s_ > 0 else 031    print(f"  {m:10s}: {d:5d} drifted / {s_:5d} stable  (ratio 1:{s_/d:.1f})" if d > 0 else f"  {m:10s}: 0 drifted")32 33# ============================================================34# Q2: What categories map to "drifted" for each model?35# ============================================================36print("\n" + "=" * 70)37print("  Q2: CATEGORY → DRIFT LABEL MAPPING (for Qwen2.5)")38print("=" * 70)39cat_drift = defaultdict(lambda: {"drifted": 0, "stable": 0})40for s in samples:41    cat = s.get("category", "unknown")42    if s.get("is_drifted_qwen25", False):43        cat_drift[cat]["drifted"] += 144    else:45        cat_drift[cat]["stable"] += 146 47print(f"  {'Category':20s} {'Drifted':>8s} {'Stable':>8s} {'%Drifted':>10s}")48print("  " + "-" * 50)49for cat in sorted(cat_drift.keys()):50    d = cat_drift[cat]["drifted"]51    s_ = cat_drift[cat]["stable"]52    pct = d / (d + s_) * 100 if (d + s_) > 0 else 053    print(f"  {cat:20s} {d:8d} {s_:8d} {pct:9.1f}%")54 55print("\n  KEY QUESTION: Is known_drift labeled as drifted?")56kd_drifted = sum(1 for s in samples if s.get("category") == "known_drift" and s.get("is_drifted_qwen25", False))57kd_total = sum(1 for s in samples if s.get("category") == "known_drift")58print(f"  known_drift samples labeled drifted for Qwen: {kd_drifted}/{kd_total}")59 60# ============================================================61# Q3: YEAR LEAKAGE — distribution of query years by drift label62# ============================================================63print("\n" + "=" * 70)64print("  Q3: YEAR LEAKAGE CHECK (Qwen2.5)")65print("=" * 70)66 67# Extract year from query or from 'year' field68year_by_drift = defaultdict(lambda: {"drifted": 0, "stable": 0})69for s in samples:70    yr = s.get("year", "unknown")71    if s.get("is_drifted_qwen25", False):72        year_by_drift[yr]["drifted"] += 173    else:74        year_by_drift[yr]["stable"] += 175 76print(f"  {'Year':>6s} {'Drifted':>8s} {'Stable':>8s} {'%Drifted':>10s}")77print("  " + "-" * 36)78for yr in sorted(year_by_drift.keys(), key=lambda x: str(x)):79    d = year_by_drift[yr]["drifted"]80    s_ = year_by_drift[yr]["stable"]81    pct = d / (d + s_) * 100 if (d + s_) > 0 else 082    print(f"  {str(yr):>6s} {d:8d} {s_:8d} {pct:9.1f}%")83 84# Check if "In 20XX" appears in queries85year_in_query = Counter()86for s in samples:87    q = s.get("query", "")88    for y in range(2010, 2027):89        if str(y) in q:90            year_in_query[y] += 191            break92    else:93        year_in_query["no_year"] += 194 95print(f"\n  Year mentioned in query text:")96for yr, n in sorted(year_in_query.items(), key=lambda x: str(x)):97    print(f"    {yr}: {n}")98 99# CRITICAL: For drifted vs stable, what years appear in the query?100print(f"\n  Query year distribution for DRIFTED vs STABLE (Qwen):")101drifted_years = Counter()102stable_years = Counter()103for s in samples:104    q = s.get("query", "")105    yr_found = None106    for y in range(2010, 2027):107        if str(y) in q:108            yr_found = y109            break110    if yr_found is None:111        yr_found = "no_year"112    if s.get("is_drifted_qwen25", False):113        drifted_years[yr_found] += 1114    else:115        stable_years[yr_found] += 1116 117all_years = sorted(set(list(drifted_years.keys()) + list(stable_years.keys())), key=lambda x: str(x))118print(f"  {'Year':>8s} {'Drifted':>8s} {'Stable':>8s}")119print("  " + "-" * 28)120for yr in all_years:121    print(f"  {str(yr):>8s} {drifted_years.get(yr, 0):8d} {stable_years.get(yr, 0):8d}")122 123# ============================================================124# Q4: What does temporal_zone filter actually include?125# ============================================================126print("\n" + "=" * 70)127print("  Q4: TEMPORAL ZONE DISTRIBUTION")128print("=" * 70)129tz_counts = Counter(s.get("temporal_zone", "none") for s in samples)130for tz, n in tz_counts.most_common():131    print(f"  {str(tz):20s}: {n:6d}")132 133# ============================================================134# Q5: expected_answer and model_likely_answer135# ============================================================136print("\n" + "=" * 70)137print("  Q5: MODEL_LIKELY_ANSWER FIELD")138print("=" * 70)139has_mla = sum(1 for s in samples if s.get("model_likely_answer") and str(s.get("model_likely_answer")).strip())140print(f"  Samples with model_likely_answer: {has_mla}/{len(samples)}")141if has_mla > 0:142    # Show a few examples143    count = 0144    for s in samples:145        mla = s.get("model_likely_answer", "")146        if mla and str(mla).strip():147            ea = s.get("expected_answer", "")148            q = s.get("query", "")[:60]149            print(f"    Query: {q}")150            print(f"    Expected: {ea}")151            print(f"    Model likely: {mla}")152            print(f"    Drifted (qwen): {s.get('is_drifted_qwen25', False)}")153            print()154            count += 1155            if count >= 3:156                break157 158# ============================================================159# Q6: Noise samples (Arabic, empty relation)160# ============================================================161print("\n" + "=" * 70)162print("  Q6: NOISE SAMPLES IN TIER 1")163print("=" * 70)164empty_rel = sum(1 for s in samples if not s.get("relation", "").strip())165arabic = sum(1 for s in samples if any(ord(c) > 0x0600 and ord(c) < 0x06FF for c in s.get("relation", "")))166tiny_rels = [(r, n) for r, n in Counter(s.get("relation", "") for s in samples).items() if n < 20]167print(f"  Empty relation: {empty_rel}")168print(f"  Arabic relation: {arabic}")169print(f"  Relations with <20 samples: {tiny_rels}")170 171# ============================================================172# Q7: Differential facts distribution across relations173# ============================================================174print("\n" + "=" * 70)175print("  Q7: DIFFERENTIAL FACTS BY RELATION")176print("=" * 70)177diff_by_rel = Counter()178total_by_rel = Counter()179for s in samples:180    rel = s.get("relation", "unknown")181    total_by_rel[rel] += 1182    labels = set()183    for m in models:184        labels.add(s.get(f"is_drifted_{m}", False))185    if len(labels) > 1:186        diff_by_rel[rel] += 1187 188n_diff = sum(diff_by_rel.values())189print(f"  Total differential facts: {n_diff}")190print(f"\n  {'Relation':30s} {'Differential':>12s} {'Total':>8s} {'%Diff':>8s}")191print("  " + "-" * 60)192for rel in sorted(total_by_rel.keys()):193    d = diff_by_rel.get(rel, 0)194    t = total_by_rel[rel]195    pct = d / t * 100 if t > 0 else 0196    print(f"  {rel:30s} {d:12d} {t:8d} {pct:7.1f}%")197 198# ============================================================199# Q8: Sample query format examples per category200# ============================================================201print("\n" + "=" * 70)202print("  Q8: SAMPLE QUERIES PER CATEGORY")203print("=" * 70)204for cat in ["stable", "no_drift", "known_drift", "unknown_drift"]:205    cat_samples = [s for s in samples if s.get("category") == cat]206    print(f"\n  [{cat}] ({len(cat_samples)} samples)")207    for s in cat_samples[:3]:208        d_labels = " | ".join(f"{m}={'D' if s.get(f'is_drifted_{m}', False) else 'S'}" for m in models)209        print(f"    Q: {s.get('query', '')[:80]}")210        print(f"    A: {s.get('expected_answer', '')[:40]}")211        print(f"    Year: {s.get('year', '?')}, Drift date: {str(s.get('drift_date', ''))[:10]}")212        print(f"    Labels: {d_labels}")213        print()214 215# ============================================================216# SUMMARY: Is year leakage a real problem?217# ============================================================218print("\n" + "=" * 70)219print("  VERDICT: YEAR LEAKAGE RISK")220print("=" * 70)221# Check if drifted samples are concentrated in recent years222drifted_recent = sum(1 for s in samples if s.get("is_drifted_qwen25", False) and int(s.get("year", 0)) >= 2024)223drifted_total = sum(1 for s in samples if s.get("is_drifted_qwen25", False))224stable_recent = sum(1 for s in samples if not s.get("is_drifted_qwen25", False) and int(s.get("year", 0)) >= 2024)225stable_total = sum(1 for s in samples if not s.get("is_drifted_qwen25", False))226 227print(f"  Drifted in 2024+: {drifted_recent}/{drifted_total} ({drifted_recent/drifted_total*100:.1f}%)" if drifted_total > 0 else "  No drifted samples")228print(f"  Stable in 2024+:  {stable_recent}/{stable_total} ({stable_recent/stable_total*100:.1f}%)" if stable_total > 0 else "  No stable samples")229 230if drifted_total > 0 and stable_total > 0:231    d_pct = drifted_recent / drifted_total232    s_pct = stable_recent / stable_total233    if d_pct > 0.8 and s_pct < 0.3:234        print("\n  ⚠️  HIGH RISK: Drifted samples are concentrated in recent years.")235        print("     The probe may be learning YEAR, not DRIFT.")236    elif d_pct > s_pct + 0.2:237        print("\n  ⚠️  MODERATE RISK: Some year-drift correlation exists.")238        print("     Paraphrase test + year-controlled subset needed.")239    else:240        print("\n  ✅ LOW RISK: Year distribution is similar across drifted/stable.")241 242print(f"\n{'=' * 70}")243print("  Run: python data_diagnostic.py")244print(f"{'=' * 70}")