fabsssss/ies4-turtle-instruct

1"""Text2KGBench-style eval of the fine-tuned model on held-out test.jsonl.2Metrics per task family:3  IES Turtle   : syntactic validity, ontology conformance (our validator), hallucinated-term rate4  multi-standard: syntactic validity, relation conformance (predicates within the given ontology)5Usage: python eval_ies.py --model <base> --adapter adapters   (omit --adapter for baseline)6Runs under .venv (mlx_lm)."""7import sys, json, argparse, pathlib, re8sys.path.insert(0, str(pathlib.Path(__file__).parent))9from iesval import validate_turtle, structural_conformance, REAL, IES10from rdflib import Graph11from mlx_lm import load, generate12 13ROOT = pathlib.Path("/Users/fabio/projects/qwen-ies-ft")14 15def extract_turtle(txt):16    m = re.search(r"```(?:turtle|ttl)?\s*(.*?)```", txt, re.S)17    if m: txt = m.group(1)18    return txt.strip()19 20def ies_metrics(ttl, user_turn=""):21    ok, reason, n, used = validate_turtle(ttl, min_triples=3)22    # syntactic validity independent of conformance23    syn = True24    try: Graph().parse(data=ttl, format="turtle")25    except Exception: syn = False26    # hallucinated term rate among ies: terms27    terms = set(re.findall(r"ies:(\w+)", ttl))28    halluc = (len(terms - REAL)/len(terms)) if terms else 1.029    # structural conformance (domain/range with subclass closure)30    sc, checked, _ = structural_conformance(ttl) if syn else (0.0, 0, [])31    # namespace fidelity: if the prompt demanded a namespace, the output must use it32    m = re.search(r"Use <(\S+)> as the namespace", user_turn)33    ns_ok = (m.group(1) in ttl) if m else None34    return {"syntactic": syn, "conformant": ok, "halluc_rate": halluc,35            "struct_conf": sc, "ns_ok": ns_ok, "triples": n}36 37def multistd_metrics(ttl, allowed_slugs):38    syn = True39    try: Graph().parse(data=ttl, format="turtle")40    except Exception: syn = False41    # extract predicates properly via RDF parsing (regex falsely captured subjects)42    flat = set()43    try:44        _g = Graph(); _g.parse(data=ttl, format="turtle")45        EXNS = "http://example.org/kg#"46        flat = {str(p).split("#")[-1] for _, p, _ in _g if str(p).startswith(EXNS)}47    except Exception:48        pass49    # IES bleed check: an ontology-conditioned multistd answer must not emit ies: terms50    ies_bleed = bool(re.search(r"\bies:\w+", ttl))51    bad = flat - allowed_slugs52    return {"syntactic": syn, "rel_conformant": len(bad)==0, "ies_bleed": ies_bleed,53            "off_ontology": sorted(bad)[:5]}54 55def main():56    ap = argparse.ArgumentParser()57    ap.add_argument("--model", default="mlx-community/Qwen3-Coder-30B-A3B-Instruct-8bit")58    ap.add_argument("--adapter", default=None)59    ap.add_argument("--max", type=int, default=200)60    a = ap.parse_args()61    kw = {"adapter_path": a.adapter} if a.adapter else {}62    model, tok = load(a.model, **kw)63 64    test = [json.loads(l) for l in (ROOT/"data"/"mlx"/"test.jsonl").open()][:a.max]65    ood_p = ROOT/"data"/"mlx"/"ood_test.jsonl"66    ood_rows = [json.loads(l) for l in ood_p.open()] if ood_p.exists() else []67    ies, multi, ood = [], [], []68    for r, bucket_ood in [(x, False) for x in test] + [(x, True) for x in ood_rows]:69        msgs = r["messages"]70        sysp = msgs[0]["content"]71        prompt = tok.apply_chat_template(msgs[:-1], add_generation_prompt=True, tokenize=False)72        out = generate(model, tok, prompt=prompt, max_tokens=1400, verbose=False)73        ttl = extract_turtle(out)74        if sysp.startswith("You extract ontology-conformant"):75            um = msgs[1]["content"]76            rels = re.search(r"Allowed relations: (.+?)\.", um)77            allowed = set()78            if rels:79                for x in rels.group(1).split(","):80                    allowed.add(re.sub(r"[^0-9A-Za-z]+","_",x.strip()).strip("_"))81            multi.append(multistd_metrics(ttl, allowed))82        elif "@prefix ies" in msgs[-1]["content"] or "a ies:" in msgs[-1]["content"]:83            (ood if bucket_ood else ies).append(ies_metrics(ttl, msgs[1]["content"]))84 85    def pct(rows, key): return 100.0*sum(1 for r in rows if r[key])/max(1,len(rows))86    def mean(rows, key): return sum(r[key] for r in rows)/max(1,len(rows))87    def block(name, rows):88        print(f"=== {name} (n={len(rows)}) ===")89        if not rows: return90        print(f"  syntactic validity : {pct(rows,'syntactic'):.1f}%")91        print(f"  term conformance   : {pct(rows,'conformant'):.1f}%")92        print(f"  structural conf    : {mean(rows,'struct_conf'):.3f}")93        print(f"  halluc-term rate   : {mean(rows,'halluc_rate'):.3f}")94        ns = [r for r in rows if r["ns_ok"] is not None]95        if ns: print(f"  namespace fidelity : {pct(ns,'ns_ok'):.1f}% (n={len(ns)})")96    print()97    block("IES Turtle (in-distribution)", ies)98    block("IES Turtle (OUT-OF-DISTRIBUTION, gold dstl)", ood)99    print(f"=== Multi-standard (n={len(multi)}) ===")100    if multi:101        print(f"  syntactic validity : {pct(multi,'syntactic'):.1f}%")102        print(f"  relation conformance: {pct(multi,'rel_conformant'):.1f}%")103        print(f"  IES-bleed rate     : {pct(multi,'ies_bleed'):.1f}% (should be 0)")104    out = {105        "ies_n": len(ies), "ood_n": len(ood), "multi_n": len(multi),106        "ies_syn": f"{pct(ies,'syntactic'):.1f}%" if ies else "-",107        "ies_conf": f"{pct(ies,'conformant'):.1f}%" if ies else "-",108        "ies_struct": f"{mean(ies,'struct_conf'):.3f}" if ies else "-",109        "ies_hall": f"{mean(ies,'halluc_rate'):.3f}" if ies else "-",110        "ood_syn": f"{pct(ood,'syntactic'):.1f}%" if ood else "-",111        "ood_conf": f"{pct(ood,'conformant'):.1f}%" if ood else "-",112        "ood_struct": f"{mean(ood,'struct_conf'):.3f}" if ood else "-",113        "multi_syn": f"{pct(multi,'syntactic'):.1f}%" if multi else "-",114        "multi_rel": f"{pct(multi,'rel_conformant'):.1f}%" if multi else "-",115        "multi_bleed": f"{pct(multi,'ies_bleed'):.1f}%" if multi else "-",116        "adapter": a.adapter or "baseline",117    }118    (ROOT/"data"/("eval.json" if a.adapter else "eval_baseline.json")).write_text(json.dumps(out, indent=2))119    print("wrote", "eval.json" if a.adapter else "eval_baseline.json")120 121if __name__=="__main__":122    main()123