Scandium-Labs/Scandium-Dataset

1"""Integrate experimental Li solid-electrolyte conductivity data.2 3Two separate, independently curated databases are supported:4 51. **Hargreaves et al. 2023** — npj Computational Materials6   ~820 entries, 403 compositions, 214 sources7   https://doi.org/10.1038/s41524-023-01137-38 92. **OBELiX (Therrien et al. 2025, NRC-Mila)**10   ~599 entries, curated with leakage-resistant splits11   pip install obelix-data12   https://github.com/nrc-mila/OBELiX13 14These are complementary — not duplicates — and are tracked as two separate15provenance sources with distinct citations.16 17Usage:18    # Hargreaves 202319    python scripts/integrate_experimental_data.py --ransom-path path/to/ransom2023.csv20 21    # OBELiX via pip package22    python scripts/integrate_experimental_data.py --obelix23 24    # Both25    python scripts/integrate_experimental_data.py --ransom-path ... --obelix26 27    # Dry run28    python scripts/integrate_experimental_data.py --dry-run29"""30import json, os, sys, time, argparse, csv, io, re, subprocess31from pathlib import Path32from collections import defaultdict33import numpy as np34import pandas as pd35import warnings36warnings.filterwarnings("ignore")37 38WIDTH = 6039 40RANSOM_URLS = [41    "https://raw.githubusercontent.com/nrc-cnrc/ransom2023-conductivity/main/data/conductivity_database.csv",42]43 44HARGREAVES_DOI = "https://doi.org/10.1038/s41524-022-00951-z"45OBELIX_DOI = "https://github.com/nrc-mila/OBELiX"46 47 48def parse_formula(formula):49    parts = re.findall(r'([A-Z][a-z]*)(\d*\.?\d*)', formula)50    return {el: float(cnt) if cnt else 1.0 for el, cnt in parts}51 52 53def formula_similarity(f1, f2):54    d1 = parse_formula(f1)55    d2 = parse_formula(f2)56    if set(d1.keys()) != set(d2.keys()):57        return False58    total1, total2 = sum(d1.values()), sum(d2.values())59    for el in d1:60        r1 = d1[el] / total161        r2 = d2[el] / total262        if abs(r1 - r2) > 0.05:63            return False64    return True65 66 67def try_fetch_ransom():68    """Try to download Hargreaves 2023 database."""69    import urllib.request70    for url in RANSOM_URLS:71        try:72            req = urllib.request.Request(url, headers={"User-Agent": "Scandium-Labs/1.0"})73            with urllib.request.urlopen(req, timeout=30) as resp:74                data = resp.read().decode("utf-8")75                print(f"  Downloaded {len(data):,} bytes")76                return data77        except Exception as e:78            print(f"  Failed: {str(e)[:80]}")79    return None80 81 82def try_fetch_obelix_package():83    """Try to install obelix-data package and load data."""84    try:85        import obelix86        ob = obelix.OBELiX(data_path="/tmp/obelix_rawdata", no_cifs=True)87        n = len(ob.dataframe)88        print(f"  OBELiX package loaded: {n} entries")89        return ob90    except ImportError:91        print("  obelix-data not installed. Attempting pip install...")92        result = subprocess.run(93            [sys.executable, "-m", "pip", "install", "obelix-data"],94            capture_output=True, text=True, timeout=6095        )96        if result.returncode == 0:97            try:98                import obelix99                ob = obelix.OBELiX(data_path="/tmp/obelix_rawdata", no_cifs=True)100                n = len(ob.dataframe)101                print(f"  OBELiX installed and loaded: {n} entries")102                return ob103            except Exception as e:104                print(f"  Load failed after install: {e}")105                return None106        else:107            print(f"  Install failed: {result.stderr[-200:]}")108            return None109 110 111def parse_ransom_csv(csv_data):112    """Parse Hargreaves 2023 CSV into entry dicts."""113    reader = csv.DictReader(io.StringIO(csv_data))114    entries = []115    for i, row in enumerate(reader):116        entry = {117            "source": "Hargreaves2023",118            "source_id": f"Hargreaves2023-{i:04d}",119            "is_experimental": True,120            "experimental_database": "Hargreaves2023",121            "provenance": {122                "source": "Hargreaves2023",123                "source_id": f"Hargreaves2023-{i:04d}",124                "doi": HARGREAVES_DOI,125                "integrated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),126            },127        }128        formula = row.get("Formula", row.get("formula", "")).strip()129        if formula:130            entry["formula"] = formula131            entry["structured_formula"] = formula132            entry["elements"] = list(parse_formula(formula).keys())133            entry["carrier_elements"] = ["Li"]134 135        for field in ["Conductivity_S_cm", "conductivity_S_cm", "Conductivity (S/cm)"]:136            val = row.get(field, "").strip()137            if val:138                try:139                    entry["conductivity_S_cm"] = float(val)140                except ValueError:141                    pass142 143        for field in ["Ea_eV", "activation_energy_eV", "Activation energy (eV)"]:144            val = row.get(field, "").strip()145            if val:146                try:147                    entry["activation_energy_eV"] = float(val)148                except ValueError:149                    pass150 151        for field in ["Temperature_K", "temperature_K", "Temperature (K)"]:152            val = row.get(field, "").strip()153            if val:154                try:155                    entry["temperature_K"] = float(val)156                except ValueError:157                    pass158 159        ref = row.get("Reference", row.get("reference", "")).strip()160        if ref:161            entry["reference"] = ref162            entry["provenance"]["experimental_reference"] = ref163 164        entries.append(entry)165 166    return entries167 168 169def parse_obelix_via_package(obelix_obj):170    """Parse OBELiX data via pandas DataFrame."""171    entries = []172    try:173        df = obelix_obj.dataframe174        for idx, row in df.iterrows():175            formula = str(row.get("Reduced Composition", ""))176            true_comp = str(row.get("True Composition", ""))177            conductivity = row.get("Ionic conductivity (S cm-1)")178            doi = str(row.get("DOI", ""))179            family = str(row.get("Family", ""))180            icsd = row.get("ICSD ID")181            sg = str(row.get("Space group", ""))182 183            entry = {184                "source": "OBELiX",185                "source_id": f"OBELiX-{idx}",186                "is_experimental": True,187                "experimental_database": "OBELiX_Therrien2025",188                "formula": formula,189                "structured_formula": true_comp if (true_comp and true_comp != "nan") else formula,190                "elements": list(parse_formula(formula).keys()) if formula else [],191                "carrier_elements": ["Li"],192                "conductivity_S_cm": float(conductivity) if pd.notna(conductivity) else None,193                "space_group": sg if sg != "nan" else "",194                "sse_family": family if family != "nan" else "",195                "reference": doi if doi != "nan" else "",196                "provenance": {197                    "source": "OBELiX_Therrien2025",198                    "source_id": f"OBELiX-{idx}",199                    "doi": "https://github.com/nrc-mila/OBELiX",200                    "icsd_id": str(icsd) if pd.notna(icsd) else "",201                    "integrated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),202                },203            }204            entries.append(entry)205    except Exception as e:206        print(f"  OBELiX DataFrame parse error: {e}")207 208    return entries209 210 211def cross_reference_and_add(exp_entries, all_dataset_entries):212    """Cross-reference experimental entries with the existing dataset."""213    formula_index = defaultdict(list)214    for e in all_dataset_entries:215        sf = e.get("structured_formula", e.get("formula", ""))216        formula_index[sf].append(e)217 218    matched = 0219    unmatched = 0220    conductivity_added = 0221    new_entries = []222 223    for exp_e in exp_entries:224        exp_formula = exp_e.get("formula", "")225        matched_entries = formula_index.get(exp_formula, [])226 227        if not matched_entries:228            for sf, existing in formula_index.items():229                if formula_similarity(exp_formula, sf):230                    matched_entries = existing231                    break232 233        db_name = exp_e.get("experimental_database", "unknown")234 235        if matched_entries:236            matched += 1237            for existing_e in matched_entries:238                if "ssb_screening" not in existing_e:239                    existing_e["ssb_screening"] = {}240 241                cond = exp_e.get("conductivity_S_cm")242                ea = exp_e.get("activation_energy_eV")243 244                if cond is not None:245                    existing_e["ssb_screening"]["estimated_ionic_conductivity_S_cm"] = cond246                    existing_e["ssb_screening"]["conductivity_source"] = f"experimental_{db_name}"247                    conductivity_added += 1248 249                if ea is not None:250                    existing_e["ssb_screening"]["experimental_activation_energy_eV"] = ea251 252                existing_e["is_experimental"] = True253                if "provenance" not in existing_e:254                    existing_e["provenance"] = {}255                existing_e["provenance"]["experimental_confirmed"] = True256                existing_e["provenance"]["experimental_database"] = db_name257                existing_e["provenance"]["experimental_reference"] = exp_e.get("reference", "")258        else:259            unmatched += 1260            new_entry = {261                "source": exp_e.get("source", "experimental"),262                "source_id": exp_e.get("source_id", f"exp-{unmatched}"),263                "formula": exp_formula,264                "structured_formula": exp_formula,265                "elements": exp_e.get("elements", []),266                "nsites": len(exp_e.get("elements", [])),267                "band_gap": None,268                "formation_energy_per_atom": None,269                "energy_above_hull": None,270                "is_experimental": True,271                "families": ["experimental_SSE"],272                "sse_family": "experimental",273                "mobile_ion": "Li",274                "carrier_elements": ["Li"],275                "tier": "experimental_gold",276                "quality_score": 95,277                "quality_flags": ["experimental_data", "has_conductivity"],278                "ssb_screening": {279                    "estimated_ionic_conductivity_S_cm": exp_e.get("conductivity_S_cm"),280                    "conductivity_source": f"experimental_{db_name}",281                    "experimental_activation_energy_eV": exp_e.get("activation_energy_eV"),282                    "measurement_temperature_K": exp_e.get("temperature_K"),283                    "mobile_ion": "Li",284                    "sse_family": "experimental",285                    "gates_passed": ["experimental"],286                    "sse_candidate_score": 100,287                },288                "provenance": exp_e.get("provenance", {}),289                "license": "CC-BY-4.0",290            }291            new_entries.append(new_entry)292 293    return matched, unmatched, conductivity_added, new_entries294 295 296def main():297    parser = argparse.ArgumentParser(description="Integrate experimental conductivity data")298    parser.add_argument("--ransom-path", type=str, default=None,299                        help="Path to Hargreaves 2023 CSV file")300    parser.add_argument("--obelix", action="store_true",301                        help="Try to load OBELiX via obelix-data package")302    parser.add_argument("--dry-run", action="store_true")303    parser.add_argument("--cross-ref-only", action="store_true")304    args = parser.parse_args()305 306    if not args.ransom_path and not args.obelix:307        print("Specify at least one data source:")308        print("  --ransom-path <file.csv>  Hargreaves et al. 2023 database")309        print("  --obelix                  OBELiX via obelix-data package")310        sys.exit(1)311 312    BASE_DIR = Path(__file__).resolve().parent.parent313    DATASET_PATH = BASE_DIR / "dataset"314 315    print("=" * WIDTH)316    print("  EXPERIMENTAL DATA INTEGRATION")317    print("=" * WIDTH)318 319    all_experimental = []320 321    # --- Hargreaves 2023 ---322    if args.ransom_path:323        source_label = "Hargreaves et al. 2023 (npj Comput. Mater.)"324        print(f"\n  [{source_label}]")325 326        ransom_data = None327        path = Path(args.ransom_path)328        if path.exists():329            with open(path) as f:330                ransom_data = f.read()331            print(f"  Loaded from {path}")332        else:333            print(f"  File not found: {path}")334            print("  Attempting download...")335            ransom_data = try_fetch_ransom()336 337        if ransom_data:338            entries = parse_ransom_csv(ransom_data)339            print(f"  Parsed {len(entries):,} entries")340            for e in entries:341                e["experimental_database"] = "Hargreaves2023"342            all_experimental.extend(entries)343            with_cond = sum(1 for e in entries if e.get("conductivity_S_cm") is not None)344            with_ea = sum(1 for e in entries if e.get("activation_energy_eV") is not None)345            print(f"    With conductivity: {with_cond}")346            print(f"    With activation energy: {with_ea}")347        else:348            print(f"  Could not load Hargreaves 2023 data.")349            print(f"  Download manually from: {HARGREAVES_DOI}")350 351    # --- OBELiX Therrien 2025 ---352    if args.obelix:353        source_label = "OBELiX (Therrien et al. 2025, NRC-Mila)"354        print(f"\n  [{source_label}]")355        print("  Attempting obelix-data package...")356        ob_data = try_fetch_obelix_package()357        if ob_data is not None:358            entries = parse_obelix_via_package(ob_data)359            print(f"  Parsed {len(entries):,} entries")360            for e in entries:361                e["experimental_database"] = "OBELiX_Therrien2025"362            all_experimental.extend(entries)363            with_cond = sum(1 for e in entries if e.get("conductivity_S_cm") is not None)364            with_ea = sum(1 for e in entries if e.get("activation_energy_eV") is not None)365            print(f"    With conductivity: {with_cond}")366            print(f"    With activation energy: {with_ea}")367        else:368            print(f"  Could not load OBELiX via package.")369            print(f"  Try: pip install obelix-data")370            print(f"  Or:  https://github.com/nrc-mila/OBELiX")371 372    if not all_experimental:373        print("\n  No experimental data loaded. Nothing to integrate.")374        sys.exit(1)375 376    # --- Cross-reference with existing dataset ---377    print(f"\n  Loading Scandium-Dataset...")378    t0 = time.time()379    with open(DATASET_PATH / "entries_final_v3.json") as f:380        all_entries = json.load(f)381    print(f"  {len(all_entries):,} entries ({time.time()-t0:.1f}s)")382 383    print(f"\n{'─' * WIDTH}")384    print("  Cross-referencing...")385    print(f"{'─' * WIDTH}")386 387    matched, unmatched, conductivity_added, new_entries = cross_reference_and_add(388        all_experimental, all_entries389    )390 391    print(f"\n  Results:")392    print(f"    Matched existing entries: {matched}")393    print(f"    Unmatched (new compositions): {unmatched}")394    print(f"    Conductivity labels added: {conductivity_added}")395    print(f"    New experimental entries: {len(new_entries)}")396 397    if new_entries:398        cond_entries = [(e.get("formula", "?"),399                         e.get("ssb_screening", {}).get("estimated_ionic_conductivity_S_cm"))400                        for e in new_entries401                        if e.get("ssb_screening", {}).get("estimated_ionic_conductivity_S_cm")]402        for formula, cond in sorted(cond_entries, key=lambda x: -abs(x[1] or 0))[:5]:403            if cond:404                print(f"    {formula:30s} σ={cond:.2e} S/cm")405 406    if not args.dry_run:407        if new_entries:408            all_entries.extend(new_entries)409            print(f"\n  Added {len(new_entries):,} experimental entries")410 411        output_path = DATASET_PATH / "entries_final_v3.json"412        print(f"  Writing to {output_path}...")413        t_write = time.time()414        with open(output_path, "w") as f:415            json.dump(all_entries, f)416        print(f"  Done ({time.time()-t_write:.1f}s)")417 418        experimental_count = sum(1 for e in all_entries if e.get("is_experimental"))419        with_conductivity_total = sum(420            1 for e in all_entries421            if e.get("ssb_screening", {}).get("estimated_ionic_conductivity_S_cm")422        )423 424        print(f"\n{'─' * WIDTH}")425        print("  INTEGRATION SUMMARY")426        print(f"{'─' * WIDTH}")427        db_sources = set(e.get("experimental_database", "unknown") for e in all_experimental)428        for db in sorted(db_sources):429            count = sum(1 for e in all_experimental if e.get("experimental_database") == db)430            print(f"  {db}: {count} entries")431        print(f"  Total experimental entries in dataset: {experimental_count}")432        print(f"  Entries with conductivity labels: {with_conductivity_total}")433    else:434        print(f"\n  (dry-run)")435 436    print("=" * WIDTH)437 438 439if __name__ == "__main__":440    main()441