Datasetpublic

notmehul/memory-bench

memory-bench v1.0.0, public release A screened benchmark for organizational memory in agent harnesses: does a memory system keep a rule that was stated once, drop a fact that was superseded, and pick the right one when tiers conflict? 371 valid paired probe instances across 3 simulated organizations, drawn from 486 probes over 612 events. Scored as pair credit: an instance counts only if the base task and its counterfactual twin both pass, so anything answerable from priors… See the full description on the dataset page: https://huggingface.co/datasets/notmehul/memory-bench.

sourceHugging Facecc-by-4.0updated 7d agoView on Hugging Face

0likes31downloads

rootMANIFEST.json

MANIFEST.json159 linesDownload Raw Back to root

1{2 "name": "memory-bench",3 "version": "1.0.0",4 "built": "2026-09-14",5 "counts": {6  "public_seeds": 3,7  "holdout_seeds": 2,8  "probes": 486,9  "events": 612,10  "valid_instances": 371,11  "facts_withheld": 74612 },13 "public_seeds": [14  "org-00001",15  "org-00002",16  "org-00003"17 ],18 "withheld": {19  "ground_truth_ledger": "org.json `facts` on every released seed",20  "probe_plans": "plan.json",21  "realization_provenance": "realization-map.json",22  "annotated_scoring_stream": "events.annotated.jsonl",23  "generator": "not released pending the v2 decision",24  "holdout_seeds": [25   "org-00004",26   "org-00005"27  ],28  "rater_keys": "packet-key.json, KEY-do-not-open.json"29 },30 "screening_worker": {31  "model": "gpt-5.4",32  "effort": "medium",33  "harness": "codex-cli 0.144.5",34  "status": "deprecated by the provider 2026-09; see MAINTENANCE.md"35 },36 "judge": {37  "model": "claude-sonnet-5",38  "rubric": "v2",39  "blinded": true40 },41 "specs": {42  "probe_spec": "v0.4.3",43  "judge_rubric": "v2"44 },45 "license": {46  "data": "CC BY 4.0",47  "code": "MIT"48 },49 "files": [50  {51   "path": "data/org-00001/events.jsonl",52   "withheld": null,53   "sha256": "53c42b91b94c533c979489c25c893fda0d89eae01ba05b70cf5a3a7c24215c4b",54   "bytes": 9604455  },56  {57   "path": "data/org-00001/probes.jsonl",58   "withheld": null,59   "sha256": "b91dae78a67ef3d52178b6707c0a290401e6ba7cec86add3459186bbb5234fa9",60   "bytes": 25086961  },62  {63   "path": "data/org-00001/g3-report.json",64   "withheld": null,65   "sha256": "aec99a6aee4224ad6c5b7906c019dbfd218cde62bc70952b81aa0bfd2bf0a23a",66   "bytes": 3559767  },68  {69   "path": "data/org-00001/org.json",70   "withheld": "126 ground-truth facts",71   "sha256": "4b88e22c8adf12f97157ce6026f2454fb524750c9364cd3c1eb99949bbcc3eca",72   "bytes": 4707273  },74  {75   "path": "data/org-00001-twin/events.jsonl",76   "withheld": null,77   "sha256": "3d44e8a7894aafcbc241b385b46afaff8ad35d2c292c603c7743a9c63115d43b",78   "bytes": 9434779  },80  {81   "path": "data/org-00001-twin/org.json",82   "withheld": "126 ground-truth facts",83   "sha256": "39101efab76416ae190bdcd828c9996c4ff228f4d20a7a3736e33f11278055e9",84   "bytes": 4707785  },86  {87   "path": "data/org-00002/events.jsonl",88   "withheld": null,89   "sha256": "cef0c9c6bd31357589507e0f4863715c4a0e699f7407154fc67a1e06d4a4555d",90   "bytes": 10338191  },92  {93   "path": "data/org-00002/probes.jsonl",94   "withheld": null,95   "sha256": "fa4eaf041231f4b6e9250487c272ad5bdbf74f9a3986de30526a4db90de1bb5e",96   "bytes": 24729397  },98  {99   "path": "data/org-00002/g3-report.json",100   "withheld": null,101   "sha256": "33f2e8f561ef6bce542c3ae1f29648e1bb969d668b93629710b9b9d2a358c629",102   "bytes": 35431103  },104  {105   "path": "data/org-00002/org.json",106   "withheld": "123 ground-truth facts",107   "sha256": "5ed2841db77824fe4a4af1721f49982e0907bd26ae2ef2a9dda96ff57146d5f8",108   "bytes": 46622109  },110  {111   "path": "data/org-00002-twin/events.jsonl",112   "withheld": null,113   "sha256": "919be6143a397578782c756cc00cf524a716160855872dde9b3c0f538a65440f",114   "bytes": 98234115  },116  {117   "path": "data/org-00002-twin/org.json",118   "withheld": "123 ground-truth facts",119   "sha256": "2ce0c1e7b284dfd8752879875264fce97de52a3977690ee5fc05852d26de3117",120   "bytes": 46627121  },122  {123   "path": "data/org-00003/events.jsonl",124   "withheld": null,125   "sha256": "e10ab94fbf83b3c9c22a29db4f3d31458b820b97f2c61f7a5ac9b94305a785ae",126   "bytes": 103774127  },128  {129   "path": "data/org-00003/probes.jsonl",130   "withheld": null,131   "sha256": "e6783d3f9ff7f01f4b5ab04c77d4b0a93de745c419e43da94d7ddfd81ed18f1d",132   "bytes": 241752133  },134  {135   "path": "data/org-00003/g3-report.json",136   "withheld": null,137   "sha256": "3a26b8c9c092101fb872f2dd8791d9190ec527e8a7ad91e232cf572d29380080",138   "bytes": 35440139  },140  {141   "path": "data/org-00003/org.json",142   "withheld": "124 ground-truth facts",143   "sha256": "62e24d5de723ed82ac0d22df890cb3b2c880c61f071ec9615d610f9547588de6",144   "bytes": 45865145  },146  {147   "path": "data/org-00003-twin/events.jsonl",148   "withheld": null,149   "sha256": "aab0e65a689d6f5fc0d1412fbb022f12c6acea932fa359613468564fe6602a30",150   "bytes": 100023151  },152  {153   "path": "data/org-00003-twin/org.json",154   "withheld": "124 ground-truth facts",155   "sha256": "b68a215ff1a3b9b96d854cec57a766e3c3732e49b3a8357e8abefca0b8732165",156   "bytes": 45870157  }158 ]159}