ML0037/ClosureBench
ClosureBench ClosureBench is a controlled benchmark for evaluating if LLMs respect explicit semantic contracts about missing information. It tests whether models distinguish absence-as-unknown, absence-as-false, and absence-as-false-only-in-complete-scopes under explicit open-world, closed-world, and locally closed-world contracts. The dataset includes the base benchmark and three extensions: Config full rows Description base 960 Main OWA/CWA/LCWA benchmark with… See the full description on the dataset page: https://huggingface.co/datasets/ML0037/ClosureBench.
074
1{2 "dataset": "closurebench_base.jsonl",3 "version": "closurebench-v1",4 "num_items": 960,5 "num_base_scenarios": 320,6 "domains": {7 "cloud_deployment": 120,8 "hospital_access": 120,9 "library_services": 120,10 "university_registration": 120,11 "finance_controls": 120,12 "procurement_review": 120,13 "manufacturing_quality": 120,14 "robotics_operations": 12015 },16 "families": {17 "entailed_open_conclusion": 96,18 "explicit_negative_closed": 96,19 "explicit_negative_open": 96,20 "closed_missing_with_open_distractor": 96,21 "open_missing_with_closed_distractor": 96,22 "open_missing_direct": 96,23 "entailed_closed_conclusion": 96,24 "closed_missing_direct": 96,25 "closed_derived_missing_antecedent": 96,26 "open_derived_missing_antecedent": 9627 },28 "subsets": {29 "control_entailed": 192,30 "control_explicit_negative": 192,31 "core_contrastive": 57632 },33 "splits": {34 "dev": 93,35 "train": 648,36 "test": 21937 },38 "semantics": {39 "owa": 320,40 "cwa": 320,41 "lcwa": 32042 },43 "gold_answers": {44 "true": 192,45 "false": 480,46 "unknown": 28847 },48 "gold_reason_types": {49 "entailed_by_fact_or_rule": 192,50 "explicit_negative_fact": 192,51 "global_closed_world_underivable": 192,52 "local_open_world_underivable": 96,53 "open_world_underivable": 192,54 "local_closed_world_underivable": 9655 },56 "semantic_switch_patterns": {57 "false|false|false": 64,58 "true|true|true": 64,59 "unknown|false|false": 96,60 "unknown|false|unknown": 9661 },62 "primary_metrics": [63 "semantic_switch_accuracy",64 "lcwa_closed_scope_accuracy",65 "lcwa_open_scope_accuracy",66 "cwa_accuracy",67 "owa_accuracy"68 ],69 "notes": [70 "V3 preserves pilot v1/v2 files and should be treated as the publication candidate.",71 "Each base scenario has OWA, CWA, and LCWA variants with identical facts, rules, and target atom.",72 "Each item contains natural-language text plus symbolic atoms/rules and a gold reason type.",73 "Closure is explicit: closed predicates make unstated and underivable atoms false after rule closure.",74 "Controls are retained but marked separately from core contrastive cases."75 ]76}