Scandium-Labs/Scandium-Dataset

1"""Phase 1: Raw Data Audit — source integrity, schema, statistics.2 3Critical check: every source entry has all required fields.4"""5import json, os, time6from pathlib import Path7from collections import Counter, defaultdict8 9OUT = Path("scripts/audit_reports")10OUT.mkdir(parents=True, exist_ok=True)11 12DATASET = "dataset/entries_final_v3.json"13AUDIT_DIR = Path.cwd() if Path.cwd().name == "Scandium-Dataset" else Path("/home/shamique/Scandium Labs SSB/Scandium-Dataset")14 15REQUIRED_FIELDS = [16    "formula", "source", "source_id", "formation_energy_per_atom",17    "elements", "nsites", "structure_json",18]19 20REQUIRED_STRUCTURE_KEYS = ["lattice", "sites", "@module", "@class"]21 22severity_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "PASS": 0}23findings = []24 25def finding(severity, phase, check, status, detail):26    severity_counts[severity] += 127    findings.append({28        "severity": severity, "phase": phase, "check": check,29        "status": status, "detail": detail,30        "timestamp": time.strftime("%Y-%m-%d %H:%M:%S")31    })32    s = "🔴" if severity == "CRITICAL" else "🟠" if severity == "HIGH" else "🟡" if severity == "MEDIUM" else "🔵" if severity == "LOW" else "✅"33    print(f"  {s} [{severity:8s}] {check}: {detail[:120]}")34 35def main():36    print("=" * 60)37    print("  PHASE 1: RAW DATA AUDIT")38    print("=" * 60)39 40    with open(AUDIT_DIR / DATASET) as f:41        entries = json.load(f)42    N = len(entries)43    print(f"\n  Loaded {N:,} entries")44 45    # 1. Source integrity46    print("\n--- Source Integrity ---")47    sources = Counter(e.get("source", "unknown") for e in entries)48    for src, cnt in sources.most_common():49        finding("PASS", "source_integrity", f"source_{src}", f"{cnt:,} entries", "")50    51    # Check for unknown sources52    unknown = [e for e in entries if e.get("source") not in ("mp", "oqmd", "jarvis")]53    if unknown:54        finding("CRITICAL", "source_integrity", "unknown_sources",55                f"{len(unknown)} entries", f"sources: {set(e.get('source') for e in unknown)}")56    else:57        finding("PASS", "source_integrity", "all_sources_known", "3 valid sources", "")58 59    # 2. Schema consistency60    print("\n--- Schema Consistency ---")61    missing_fields = defaultdict(set)62    wrong_types = []63    corrupted_json = []64    duplicate_ids = defaultdict(list)65 66    for i, e in enumerate(entries):67        for field in REQUIRED_FIELDS:68            if e.get(field) is None:69                src = e.get("source", "?")70                missing_fields[field].add(src)71        72        # Check structure_json parseable73        sj = e.get("structure_json")74        if sj:75            if isinstance(sj, str):76                try:77                    sd = json.loads(sj)78                    if not all(k in sd for k in REQUIRED_STRUCTURE_KEYS):79                        corrupted_json.append((i, "missing_keys"))80                except json.JSONDecodeError:81                    corrupted_json.append((i, "parse_error"))82            elif isinstance(sj, dict):83                if not all(k in sj for k in REQUIRED_STRUCTURE_KEYS):84                    corrupted_json.append((i, "missing_keys_dict"))85 86        # Check duplicate IDs per source87        sid = e.get("source_id")88        src = e.get("source")89        if sid and src:90            duplicate_ids[(src, sid)].append(i)91 92    # Report missing fields93    for field, srcs in missing_fields.items():94        finding("HIGH" if field in ("formula", "source", "structure_json") else "MEDIUM",95                "schema", f"missing_field_{field}", 96                f"missing in {', '.join(sorted(srcs))}",97                f"{field} should never be None")98 99    # Report corrupted JSON100    if corrupted_json:101        finding("CRITICAL", "schema", "corrupted_structure_json",102                f"{len(corrupted_json)} entries", "")103    else:104        finding("PASS", "schema", "structure_json_valid", "all parseable", "")105 106    # Report duplicate IDs107    dup_ids_found = {k: v for k, v in duplicate_ids.items() if len(v) > 1}108    if dup_ids_found:109        finding("HIGH", "schema", "duplicate_source_ids",110                f"{len(dup_ids_found)} groups", 111                "same (source, source_id) pairs exist — potential dedup gap")112    else:113        finding("PASS", "schema", "no_duplicate_ids", "all source IDs unique", "")114 115    # 3. Source statistics116    print("\n--- Source Statistics ---")117    for src in ["mp", "oqmd", "jarvis"]:118        subset = [e for e in entries if e.get("source") == src]119        print(f"\n  {src.upper()}: {len(subset):,} entries")120        121        # Missing labels122        for prop in ["formation_energy_per_atom", "energy_above_hull", "band_gap", "space_group", "volume", "density"]:123            missing = sum(1 for e in subset if e.get(prop) is None)124            if missing > 0:125                pct = 100 * missing / len(subset)126                sev = "CRITICAL" if pct > 50 else "HIGH" if pct > 10 else "MEDIUM" if pct > 0 else "PASS"127                finding(sev, "source_stats", f"{src}_{prop}_missing",128                        f"{missing:,} / {len(subset):,} ({pct:.1f}%)", "")129            else:130                finding("PASS", "source_stats", f"{src}_{prop}_present",131                        f"0 missing (100% coverage)", "")132 133    # 4. License compatibility check134    print("\n--- License Compatibility ---")135    oqmd = [e for e in entries if e.get("source") == "oqmd"]136    finding("PASS", "license", "oqmd_license",137            f"{len(oqmd):,} entries: non-commercial use OK",138            "OQMD allows non-commercial use with attribution")139    finding("PASS", "license", "mp_license",140            f"{sources.get('mp', 0):,} entries: CC BY 4.0",141            "MP requires attribution")142    finding("PASS", "license", "jarvis_license",143            f"{sources.get('jarvis', 0):,} entries: CC0",144            "No restrictions")145 146    # Summary147    print(f"\n{'=' * 60}")148    print(f"  PHASE 1 SUMMARY")149    print(f"  CRITICAL: {severity_counts['CRITICAL']}")150    print(f"  HIGH:     {severity_counts['HIGH']}")151    print(f"  MEDIUM:   {severity_counts['MEDIUM']}")152    print(f"  LOW:      {severity_counts['LOW']}")153    print(f"  PASS:     {severity_counts['PASS']}")154    print(f"{'=' * 60}")155 156    report = {157        "phase": "Phase 1: Raw Data Audit",158        "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),159        "total_entries": N,160        "sources": dict(sources.most_common()),161        "findings": findings,162        "summary": dict(severity_counts),163    }164    with open(OUT / "phase1_raw_data_audit.json", "w") as f:165        json.dump(report, f, indent=2)166    print(f"\n  Report: {OUT / 'phase1_raw_data_audit.json'}")167 168if __name__ == "__main__":169    main()170