wakamex/github

1[{"node": {"oid": "f3b7917091afba325af3980a35d8a6dcba03dc3f", "messageHeadline": "Update README.md (#1430)", "author": {"name": "David Hoffmann", "email": "105221763+adavidho@users.noreply.github.com", "user": {"login": "adavidho"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2d0a646020db092a0e994a254bb9e5557a02e63b", "messageHeadline": "Refactor utilities into a separate model utils file. (#1429)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 554, "deletions": 527}}, {"node": {"oid": "620d6a152b2eb9216138495bfd6db4ea770dec26", "messageHeadline": "Fix: task weighting by subtask size ; update Pooled Stderr formula sl\u2026", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 53, "deletions": 8}}, {"node": {"oid": "bfbd0325d3beea7482b866919bc6d1ab05dd7578", "messageHeadline": "Added seeds to `evaluator.simple_evaluate` signature (#1412)", "author": {"name": "Amine Elhattami", "email": "35821254+Am1n3e@users.noreply.github.com", "user": {"login": "Am1n3e"}}, "additions": 65, "deletions": 6}}, {"node": {"oid": "b69c67c178ed101e857275f8d4d1a5769ced7210", "messageHeadline": "[m_mmul] added multilingual evaluation from alexandrainst/m_mmlu (#1358)", "author": {"name": "Alessandro Ercolani", "email": "ale.ercolani@gmail.com", "user": {"login": "giux78"}}, "additions": 178, "deletions": 0}}, {"node": {"oid": "7397b9652027fb5bdc24b1b2ded0d811f3d272dd", "messageHeadline": "Add multilingual TruthfulQA task (#1420)", "author": {"name": "Uanu", "email": "92366232+uanu2002@users.noreply.github.com", "user": {"login": "uanu2002"}}, "additions": 567, "deletions": 0}}, {"node": {"oid": "0256c68258d9977ee7a7491e863d2320e946986b", "messageHeadline": "Add multilingual ARC task (#1419)", "author": {"name": "Uanu", "email": "92366232+uanu2002@users.noreply.github.com", "user": {"login": "uanu2002"}}, "additions": 313, "deletions": 0}}, {"node": {"oid": "1ff84897a8cf156b80779ed8118621535c3b62fe", "messageHeadline": "Evaluate (#1385)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 107, "deletions": 99}}, {"node": {"oid": "1e6825da447275f2949c8a33f1aca7485a54ce2f", "messageHeadline": "Fix watchdog timeout (#1404)", "author": {"name": "Jeevan", "email": "64039772+JeevanBhoot@users.noreply.github.com", "user": {"login": "JeevanBhoot"}}, "additions": 9, "deletions": 2}}, {"node": {"oid": "921eab86b276e156eefa2a6343761fb422c5c01b", "messageHeadline": "Fixes https://github.com/EleutherAI/lm-evaluation-harness/issues/1416\u2026", "author": {"name": "Pasquale Minervini", "email": "p.minervini@gmail.com", "user": {"login": "pminervini"}}, "additions": 5, "deletions": 0}}, {"node": {"oid": "ab4dba8fa4d23b38f2cb03eb747b505f93f33f1c", "messageHeadline": "use reversed task hierarchy for print (#1414)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4c17c55c9ae3b22280daf4046f70c176e62706d4", "messageHeadline": "`batch_size` with `auto` defaults to 1 if `No executable batch size f\u2026", "author": {"name": "Pasquale Minervini", "email": "p.minervini@gmail.com", "user": {"login": "pminervini"}}, "additions": 7, "deletions": 1}}, {"node": {"oid": "756eeb6f0aee59fc624c81dcb0e334c1263d80e3", "messageHeadline": "adding hf_transfer (#1400)", "author": {"name": "Michael Feil", "email": "63565275+michaelfeil@users.noreply.github.com", "user": {"login": "michaelfeil"}}, "additions": 14, "deletions": 1}}, {"node": {"oid": "94cc18500fd3bd165b532d5e36349758f1221f30", "messageHeadline": "Use Pooled rather than Combined Variance for calculating stderr of ta\u2026", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 91, "deletions": 59}}, {"node": {"oid": "df01adf64434d6081e6139445da6c70b564d431d", "messageHeadline": "Fix confusing `write_out.py` instructions in README (#1371)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4d7d2f64576205105318fd12a622b6f0b7c70464", "messageHeadline": "Update README.md (#1398)", "author": {"name": "Michael Chen", "email": "chen.michael.1116@gmail.com", "user": {"login": "mycoalchen"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "d17dcea0822c542a5e75259e11cdb995f32dac44", "messageHeadline": "Support for Inf2 optimum class [WIP] (#1364)", "author": {"name": "Michael Feil", "email": "63565275+michaelfeil@users.noreply.github.com", "user": {"login": "michaelfeil"}}, "additions": 784, "deletions": 1}}, {"node": {"oid": "7411947112117e0339fe207fb620a70bcec22690", "messageHeadline": "fix on --task list (#1387)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "9a902155bc83bfd19231f2e08e2470adb6700a23", "messageHeadline": "Fix for https://github.com/EleutherAI/lm-evaluation-harness/issues/13\u2026", "author": {"name": "Pasquale Minervini", "email": "p.minervini@gmail.com", "user": {"login": "pminervini"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "d714fc951b719f53616da95c83ec4179ec8f7d83", "messageHeadline": "Faster Task and Group Loading, Allow Recursive Groups (#1321)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 927, "deletions": 573}}, {"node": {"oid": "17191063c2cf01c2c1514ed7bbe3598bacb5ff51", "messageHeadline": "Enable override of printed `n-shot` in table (#1379)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 13, "deletions": 12}}, {"node": {"oid": "994bdb3fd3780110e45f479f5f409cae05e089bd", "messageHeadline": "Hf: minor egde cases (#1380)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 3}}, {"node": {"oid": "f5408b6b7b96f587e4f936776bb8c8c5ddac6ff8", "messageHeadline": "Expand docs, update CITATION.bib (#1227)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 84, "deletions": 0}}, {"node": {"oid": "f8203de1ab017fb3868a5511340d202255f29818", "messageHeadline": "add bypass metric (#1156)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 129, "deletions": 43}}, {"node": {"oid": "084b70502cf12aae5e42867dca7490e45c265f59", "messageHeadline": "Add support for RWKV models with World tokenizer (#1374)", "author": {"name": "Eugene Cheah", "email": "PicoCreator@users.noreply.github.com", "user": {"login": "PicoCreator"}}, "additions": 10, "deletions": 0}}, {"node": {"oid": "a0a2fec8dc0c7da009029640d3316cb1091ee1c1", "messageHeadline": "Make dependencies compatible with PyPI (#1378)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 6}}, {"node": {"oid": "0da0dcbaf42d623aaec4f521977a0e504c41955d", "messageHeadline": "Publish to pypi (#1194)", "author": {"name": "Anjor Kanekar", "email": "anjor@umd.edu", "user": {"login": "anjor"}}, "additions": 75, "deletions": 0}}, {"node": {"oid": "bd7d265af6a8a01ca1c1c2ca34da4598a2344c2e", "messageHeadline": "Fix unintuitive `--gen_kwargs` behavior (#1329)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 13, "deletions": 7}}, {"node": {"oid": "1554066c9532a7a1cf171b02a5ddaa5bb38f1b78", "messageHeadline": "delay filter init; remove `*args` (#1369)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 27, "deletions": 15}}, {"node": {"oid": "7fc436562c3640a4140ca4a99ddb14d28c4051b0", "messageHeadline": "serialize callable functions in config (#1367)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 28, "deletions": 10}}, {"node": {"oid": "488759d2c9a55ad3ed55d55417b7733e95e37667", "messageHeadline": "Apply some best practices and guideline recommendations to code (#1363)", "author": {"name": "LSinev", "email": "LSinev@users.noreply.github.com", "user": {"login": "LSinev"}}, "additions": 83, "deletions": 74}}, {"node": {"oid": "97a67d27c09857e5698cbae730750cf84cd987f3", "messageHeadline": "Add causalLM OpenVino models (#1290)", "author": {"name": "NoushNabi", "email": "33136068+NoushNabi@users.noreply.github.com", "user": {"login": "NoushNabi"}}, "additions": 154, "deletions": 5}}, {"node": {"oid": "5b0b8a56dcaf3df0c1d3787d2d9ca8a491241b66", "messageHeadline": "Refix issue regarding stderr (#1357)", "author": {"name": "thnkinbtfly", "email": "70014488+thnkinbtfly@users.noreply.github.com", "user": {"login": "thnkinbtfly"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "52f48e8ca835c5d9a57295a856b8dc4450427eb0", "messageHeadline": "Add FAQ on `lm_eval.tasks.initialize_tasks()` to README (#1330)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "a0f1cacd9830874a1a8ef068220cc9a6c159825a", "messageHeadline": "`Filter` docs not offset by `doc_id`  (#1349)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 81, "deletions": 85}}, {"node": {"oid": "34cded30e6c0f13ad682f37f932e1573a36cc410", "messageHeadline": "update links to task_guide.md (#1348)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "38c8d02f790061f5af5e94dc01b0a80b77ef3ad5", "messageHeadline": "modified default gen_kwargs to work better with CLI; changed prompt_l\u2026", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "081deb8beb6989912d8107b196ee6ad69ca78f5a", "messageHeadline": "manage default (greedy) gen_kwargs in vllm (#1341)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 14, "deletions": 7}}, {"node": {"oid": "969b48bf6a998e9a84b2a7d03804686c4f5b1c7d", "messageHeadline": "Don't use `get_task_dict()` in task registration / initialization (#1\u2026", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "45a8f709cd3fa903a0e2ff7275694d441bcf0cac", "messageHeadline": "Update migrated HF dataset paths (#1332)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 5}}, {"node": {"oid": "6e65ef382b5fba83088b0c703d010a37467a2a1d", "messageHeadline": "skip \"benchmarks\" in changed_tasks (#1336)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "84357a46777cf9b4b6c27b4342f652ecfad5756f", "messageHeadline": "fix a trailing whitespace that breaks a lint job (#1335)", "author": {"name": "Brian Vaughan", "email": "nairbv@users.noreply.github.com", "user": {"login": "nairbv"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "607d7da551d09200c490229b87fb2356e2b5d5c5", "messageHeadline": "fallback to classname when LM doesnt have config (#1334)", "author": {"name": "Brian Vaughan", "email": "nairbv@users.noreply.github.com", "user": {"login": "nairbv"}}, "additions": 8, "deletions": 3}}, {"node": {"oid": "5c25dd5514eb8d34a385fa500d34d1249730723f", "messageHeadline": "Add `local-completions` support using OpenAI interface (#1277)", "author": {"name": "Michael Goin", "email": "michael@neuralmagic.com", "user": {"login": "mgoin"}}, "additions": 62, "deletions": 49}}, {"node": {"oid": "076372ee9ee81e25c4e2061256400570354a8d1a", "messageHeadline": "Fix Issue regarding stderr (#1327)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 20, "deletions": 12}}, {"node": {"oid": "181ccf4362fc945771fde525967ad52df42c2573", "messageHeadline": "don't pass extra kwargs to mamba any more (#1328)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "4477d572d5c2dc5f2ee903ab7e3c1fdda835af15", "messageHeadline": "Update polemo2_in.yaml (#1318)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "b93c3bcbf30109c40cd19ee862161702920b9c27", "messageHeadline": "Update task_guide.md (#1316)", "author": {"name": "kwrobel.eth", "email": "djstrong@gmail.com", "user": {"login": "djstrong"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "72ea626edde940f58ecb57e461fca772af1a5b7d", "messageHeadline": "Fix group register (#1315)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 5}}, {"node": {"oid": "b4c6bdb72ee15ece254f48b8229410aa5e3d416c", "messageHeadline": "Update pyproject.toml (#1314)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b8cbc425e7c71d07fdbf58e3b105f9112bf71291", "messageHeadline": "Fix polemo2_in.yaml config name (#1313)", "author": {"name": "Quentin Lhoest", "email": "42851186+lhoestq@users.noreply.github.com", "user": {"login": "lhoestq"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6f60a92436a49e77e3b75e8f7aa96eb215415b54", "messageHeadline": "Update pyproject.toml (#1312)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "92cc10ef27a2929b16db513ebcb5c7a3ff0fb764", "messageHeadline": "Update task_guide.md (#1306)", "author": {"name": "Danielle Pintz", "email": "38207072+daniellepintz@users.noreply.github.com", "user": {"login": "daniellepintz"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "10488d0d2cfbfd2d73416e92eb30c4a74ac3920b", "messageHeadline": "Update nq_open.yaml (#1305)", "author": {"name": "Hannibal046", "email": "38466901+Hannibal046@users.noreply.github.com", "user": {"login": "Hannibal046"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "ada4a31d69fd58ce794b60944900bd1471fbad3e", "messageHeadline": "Update README.md with custom integration doc (#1298)", "author": {"name": "Mark Saroufim", "email": "marksaroufim@meta.com", "user": {"login": "msaroufim"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "032e879bf5ff39c08ae0db1f622a5b382a42eaa2", "messageHeadline": "Update nq_open.yaml (#1289)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "588a493c41b58d048d3994f99800edd27223bd3a", "messageHeadline": "Update CITATION.bib (#1285)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 23}}, {"node": {"oid": "39a465ca0d770f3bac839140e5a8fae37b1b2843", "messageHeadline": "Re-add citation", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 9, "deletions": 5}}, {"node": {"oid": "b074ccb601c45ba921b7a3f6e1748131a80185ac", "messageHeadline": "Rework documentation for explaining local dataset (#1284)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 30, "deletions": 10}}, {"node": {"oid": "ef66508838b4658b24e7243f0014b67c3aa7f234", "messageHeadline": "Fix data-parallel evaluation with quantized models (#1270)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 19, "deletions": 4}}, {"node": {"oid": "03e7df519ea2505fa0890847cdf23a6d45393efc", "messageHeadline": "Allow parameter edits for registered tasks when listed in a benchmark\u2026", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 51, "deletions": 6}}, {"node": {"oid": "39e7b26402a9413640638106271072bcb0ad4819", "messageHeadline": "Make `parallelize=True` vs. `accelerate launch` distinction clearer i\u2026", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 24, "deletions": 12}}, {"node": {"oid": "ace4393e66b3f36aba4f06ccd60736b8fd05ec94", "messageHeadline": "fix whitespace in target + prompt for CoT gsm8k (#1275)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 12, "deletions": 14}}, {"node": {"oid": "89618bf8421d27c8cf28004d616b33fc5b305ceb", "messageHeadline": "apply process_docs() to fewshot_split too (#1276)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "653217a7edd712c484e5c17345ec2a7124c9bec2", "messageHeadline": "add Kobest (#1263)", "author": {"name": "jp", "email": "jp42maru@gmail.com", "user": {"login": "jp1924"}}, "additions": 193, "deletions": 0}}, {"node": {"oid": "75dc2b873a3a17306421ec26f711b327f59c9dbe", "messageHeadline": "update versioning logging (#1271)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "eed2d3a6e40b60cf9df0bf52d3d6f572a3dc5fb8", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ff739414af28f4ddb3887687b28af63cc71af08a", "messageHeadline": "Fix bug in multi-token Stop Sequences (#1268)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 49, "deletions": 42}}, {"node": {"oid": "818c056bce35d717d0c46ca11ab4080c284d7af2", "messageHeadline": "MultiMedQA (#1198)", "author": {"name": "Tanishq Abraham", "email": "37097934+tmabraham@users.noreply.github.com", "user": {"login": "tmabraham"}}, "additions": 115, "deletions": 0}}, {"node": {"oid": "692e0f83b5341b543fa288f84289617f793e4e93", "messageHeadline": "Call \"exact_match\" once for each multiple-target sample (#1266)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 30, "deletions": 596}}, {"node": {"oid": "9b0b15b1ccace3534ffbd13298c569869ce8eaf3", "messageHeadline": "fixed belebele (#1267)", "author": {"name": "James A. Michaelov", "email": "32554945+jmichaelov@users.noreply.github.com", "user": {"login": "jmichaelov"}}, "additions": 254, "deletions": 129}}, {"node": {"oid": "7264a2e03871733760c640acc3aab0e181be843d", "messageHeadline": "specify utf-8 encoding to save samples to file. (#1265)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ecb1df28f6de2495da560c21b891a00133372337", "messageHeadline": "Revert citation (#1257)", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 5, "deletions": 9}}, {"node": {"oid": "cf6a832143427f7b5e243c9fd4c5da5ff728be6f", "messageHeadline": "fixed fewshot loading for multiple input tasks (#1255)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 16, "deletions": 11}}, {"node": {"oid": "28ec7fa950346b5a895e85e1f3edd5648168acc4", "messageHeadline": "Do not escape ascii is logging outputs (#1246)", "author": {"name": "Sam Passaglia", "email": "8333102+passaglia@users.noreply.github.com", "user": {"login": "passaglia"}}, "additions": 7, "deletions": 2}}, {"node": {"oid": "28bb45fbfc6872372a220759c36a0b36b05831de", "messageHeadline": "Add multilingual HellaSwag task (#1228)", "author": {"name": "JorgeDeCorte", "email": "91887115+JorgeDeCorte@users.noreply.github.com", "user": {"login": "JorgeDeCorte"}}, "additions": 276, "deletions": 3}}, {"node": {"oid": "e7c03d0c73587e6617eb98a7e5234df09d48b3c3", "messageHeadline": "Remove self.dataset_path post_init process (#1243)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 7}}, {"node": {"oid": "eca6926be92e3349aef4e4214d43817b24108c8d", "messageHeadline": "vllm: handle max_length better and substitute Collator (#1241)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 118, "deletions": 105}}, {"node": {"oid": "25a15379676c8a2fa0b93ca9c4742b156e1fec39", "messageHeadline": "Update openai_completions.py (#1238)", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "4d10ad56b1ffe569467eee2297e2317c99313118", "messageHeadline": "batch_schedular bug in Collator (#1229)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "a12ef445d444676f29a420f8fa92192e5be8136e", "messageHeadline": "Update README.md (#1230)", "author": {"name": "Pasquale Minervini", "email": "p.minervini@gmail.com", "user": {"login": "pminervini"}}, "additions": 19, "deletions": 1}}, {"node": {"oid": "1229862a3450d31e8a7cdf3818ddfedfc029f210", "messageHeadline": "Update README.md (#1195)", "author": {"name": "Anjor Kanekar", "email": "anjor@umd.edu", "user": {"login": "anjor"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "34b563b18b6073b7cd4e040896c3363d321b641a", "messageHeadline": "Don't silence errors when loading tasks (#1148)", "author": {"name": "Paul McCann", "email": "132827365+polm-stability@users.noreply.github.com", "user": {"login": "polm-stability"}}, "additions": 13, "deletions": 3}}, {"node": {"oid": "46c796644913cd99da7eee868e64f9ed6af33407", "messageHeadline": "add length of strings and answer options to metadata (#1222)", "author": {"name": "Alex B\u00e4uerle", "email": "alex@a13x.io", "user": {"login": "Sparkier"}}, "additions": 11, "deletions": 6}}, {"node": {"oid": "6a1c19ed7406184da887c78fa755e647ce6febcf", "messageHeadline": "nits + fix siqa (#1216)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 23, "deletions": 11}}, {"node": {"oid": "f28539959d1bd18373446d6e0ee8fd926b89432e", "messageHeadline": "fix unbounded local variable (#1218)", "author": {"name": "Jaewoo Yang", "email": "sweyjw@gmail.com", "user": {"login": "onnoo"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "af74a93d1abd9a35fee9d628c2637ae922a82541", "messageHeadline": "pin vllm at < 0.2.6 (#1212)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e4970d817ae1f8ad1fccab3d77e9ef844d332239", "messageHeadline": "fix: incorrect argument order in `utils.divide` doc (#1208)", "author": {"name": "Yuliang Li", "email": "40186387+xTayEx@users.noreply.github.com", "user": {"login": "xTayEx"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "8ffbe58a63d2ce89002a788c3ea733c4d983d778", "messageHeadline": "Add remove_whitespace to FLD benchmark (#1206)", "author": {"name": "MorishT", "email": "106973776+MorishT@users.noreply.github.com", "user": {"login": "MorishT"}}, "additions": 6, "deletions": 1}}, {"node": {"oid": "9fb2ebababd78986f445cbe10e97c499213c407a", "messageHeadline": "Consolidate batching (#1197)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 346, "deletions": 109}}, {"node": {"oid": "b12bb1d4c0718a3f001fc49b91a46d261921775d", "messageHeadline": "Fix documentation in API table (#1203)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8286b1d99289c62b059150e44455c9dcec993dff", "messageHeadline": "Fixes https://github.com/EleutherAI/lm-evaluation-harness/issues/437 \u2026", "author": {"name": "Anjor Kanekar", "email": "anjor@umd.edu", "user": {"login": "anjor"}}, "additions": 1, "deletions": 3}}, {"node": {"oid": "5503b274e0b3f494ee13d740e29833330436ea03", "messageHeadline": "Upstream Mamba Support (`mamba_ssm`) (#1110)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 152, "deletions": 17}}, {"node": {"oid": "b69ca72ec3a0294638382e0f90cf32f90d761b44", "messageHeadline": "Update minerva_math_algebra.yaml (#1189)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "25cefbc1b37a9c4d10acab51a9ec283f2ff34772", "messageHeadline": "Refer in README to main branch (#1200)", "author": {"name": "Bram Vanroy", "email": "2779410+BramVanroy@users.noreply.github.com", "user": {"login": "BramVanroy"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "046ea6e242f29d8afa44f5b0a072d065c84a9d7f", "messageHeadline": "Generic decorator for handling rate limit errors (#1109)", "author": {"name": "Zach Schillaci", "email": "zschillaci@brandeis.edu", "user": {"login": "zachschillaci27"}}, "additions": 118, "deletions": 55}}, {"node": {"oid": "09493fd24d5c5947dfae07853b05d19c79bc16a4", "messageHeadline": "disable `mypy` (#1193)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 18, "deletions": 16}}, {"node": {"oid": "9267354e651a406a5314e16bcb9bbbe28170f9f4", "messageHeadline": "Update README.md (#1181)", "author": {"name": "Anjor Kanekar", "email": "anjor@umd.edu", "user": {"login": "anjor"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "da0a5e36e238f41ad5d0f3bba921fcfbf21022ca", "messageHeadline": "Remove tokenizer for openai chat completions (#1191)", "author": {"name": "Anjor Kanekar", "email": "anjor@umd.edu", "user": {"login": "anjor"}}, "additions": 4, "deletions": 72}}, {"node": {"oid": "84790e9931dfc453bab4c0dc17c2f69d6bdce6aa", "messageHeadline": "update Zeno example and reference in README (#1190)", "author": {"name": "Alex B\u00e4uerle", "email": "alex@a13x.io", "user": {"login": "Sparkier"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "9cd798974542f361960b164f0e41cc14cb61d436", "messageHeadline": "Correctly Print Task Versioning (#1173)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 150, "deletions": 125}}, {"node": {"oid": "a0cfe3f631a5a889ded400f7bc7bc514cc01fb5d", "messageHeadline": "Add tokenizer backend (#1186)", "author": {"name": "Anjor Kanekar", "email": "anjor@umd.edu", "user": {"login": "anjor"}}, "additions": 9, "deletions": 3}}, {"node": {"oid": "2b0b6fd823345dab6db9ee1c341023b50c8d7dd5", "messageHeadline": "Update README.md (#1183)", "author": {"name": "Anjor Kanekar", "email": "anjor@umd.edu", "user": {"login": "anjor"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e548d94d4c57bc5bd1ed6efea3203edb9e22eaee", "messageHeadline": "Update README.md (#1184)", "author": {"name": "Anjor Kanekar", "email": "anjor@umd.edu", "user": {"login": "anjor"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "fcfc0c6044480d4caccdcbcfd1c221eb48fcccef", "messageHeadline": "Implementing local OpenAI API-style chat completions on any given inf\u2026", "author": {"name": "Vicki Boykis", "email": "vicki@mozilla.ai", "user": {"login": "veekaybee"}}, "additions": 60, "deletions": 40}}, {"node": {"oid": "12f2c5ea15d793b7f1aa8f3611ccc8db5eb1a9a5", "messageHeadline": "Error in --num_fewshot option for K-MMLU Evaluation Harness (#1178)", "author": {"name": "GUIJIN SON", "email": "spthsrbwls123@yonsei.ac.kr", "user": {"login": "guijinSON"}}, "additions": 4, "deletions": 25}}, {"node": {"oid": "65b8761db922513dada0320b860fabb1b4f01dc3", "messageHeadline": "Switch Linting to `ruff` (#1166)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 389, "deletions": 446}}, {"node": {"oid": "21d4ae98c1e5bab15ccd71a239fc08c8fbf02c03", "messageHeadline": "feat: add option to upload results to Zeno (#990)", "author": {"name": "Alex B\u00e4uerle", "email": "alex@a13x.io", "user": {"login": "Sparkier"}}, "additions": 366, "deletions": 0}}, {"node": {"oid": "9e03d9d024be9bc3e92f8c63b5595c1e12c119da", "messageHeadline": "Fix Column Naming and Dataset Naming Conventions in K-MMLU Evaluation\u2026", "author": {"name": "seungduk.kim.2304", "email": "seungduk.kim@yanolja.com", "user": {"login": "seungduk-yanolja"}}, "additions": 29, "deletions": 35}}, {"node": {"oid": "78545d42f2ca95c6fe0ed220d456eeb94f4485e9", "messageHeadline": "self.device in huggingface.py line 210 treated as torch.device but mi\u2026", "author": {"name": "Pasquale Minervini", "email": "p.minervini@gmail.com", "user": {"login": "pminervini"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "3f0a3611bbcd86ed373935e4d8335d9d6804dfe4", "messageHeadline": "generalize qwen pad token fix (#1146)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "42730d90b388931e336c3071f8b0bbe0fcb69493", "messageHeadline": "Simplify evaluator (#1126)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 69, "deletions": 89}}, {"node": {"oid": "8e87eff487c5ae2f7572058a8c92cd6ff345b48b", "messageHeadline": "Add docs on adding a multiple choice metric (#1147)", "author": {"name": "Paul McCann", "email": "132827365+polm-stability@users.noreply.github.com", "user": {"login": "polm-stability"}}, "additions": 43, "deletions": 0}}, {"node": {"oid": "13fbfef7c8d8bfb0137f54d184b528b37f48eb32", "messageHeadline": "Update CODEOWNERS", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c2fad099237f388039c0bfe84a7878e032be5254", "messageHeadline": "Remove GooseAI docs and change no-commit-to-branch precommit hook (#1\u2026", "author": {"name": "Vicki Boykis", "email": "vicki@mozilla.ai", "user": {"login": "veekaybee"}}, "additions": 12, "deletions": 12}}, {"node": {"oid": "6f9630c8ebd5277af45cef3201f8e15e84c9be76", "messageHeadline": "bugfix (#1150)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "3ab5a262ce32af71a3c9e17959a61a56242652d9", "messageHeadline": "Add shorthand flags (#1149)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 12, "deletions": 2}}, {"node": {"oid": "08fcf1fed1de85c4ecba3ebdf0e6618495bba178", "messageHeadline": "set `--gen_kwargs` arg to None  (#1145)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 27, "deletions": 14}}, {"node": {"oid": "aa61f940598c8e979e8c5632aa61d7d23db7d5c0", "messageHeadline": "[WIP] Add IFEval / Instruction-Following Eval (#1087)", "author": {"name": "Wis Kojohnjaratkul", "email": "wkojohnj@gmail.com", "user": {"login": "wiskojo"}}, "additions": 3682, "deletions": 0}}, {"node": {"oid": "8f5b22955266ed7288b531b8def7509060685114", "messageHeadline": "openai nits (#1139)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 29, "deletions": 18}}, {"node": {"oid": "f7c67f0e81ed9823bed679c8f9a7a05bc068e088", "messageHeadline": "`use_tqdm=False` if batch_size != auto (#1144)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "bd0f2414585df9209db2f4ab0430912ed000f6a0", "messageHeadline": "Enabling OpenAI completions via gooseai (#1141)", "author": {"name": "Vicki Boykis", "email": "vicki.boykis@gmail.com", "user": {"login": "veekaybee"}}, "additions": 9, "deletions": 9}}, {"node": {"oid": "35a65ba05f8c479cb292533bb0301fa1f4084dcd", "messageHeadline": "add utils.clear_torch_cache() (#1142)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 12, "deletions": 2}}, {"node": {"oid": "b0d155d37c89e51c247fc07fefe05fb9bc9196ca", "messageHeadline": "Update Linter CI Job (#1130)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e65e5bbd3419f81c9ef432359a52ed746eb82dd5", "messageHeadline": "add correct openai api key to README.md (#1138)", "author": {"name": "Lenni Justen", "email": "65619456+lennijusten@users.noreply.github.com", "user": {"login": "lennijusten"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "38c36613d06af7af60244959fc182181c871ffe0", "messageHeadline": "fix typo in README.md (#1136)", "author": {"name": "Lenni Justen", "email": "65619456+lennijusten@users.noreply.github.com", "user": {"login": "lennijusten"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "755bf6e80544a9c062743a045399d1a6cd0ab6a5", "messageHeadline": "Add benchmark FLD (#1122)", "author": {"name": "MorishT", "email": "106973776+MorishT@users.noreply.github.com", "user": {"login": "MorishT"}}, "additions": 81, "deletions": 0}}, {"node": {"oid": "57c3b1a2a8c01a7dbb13d6ad2dab99f5459c6ec4", "messageHeadline": "place device onto `mps` (#1133)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "04707a2d4cf01f21efede709978ce2e301d3d83e", "messageHeadline": "fixed how to check if dataset_path is a local directory or not (#1127)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "2a47159caff00135b026f724ace2a2011f3c7621", "messageHeadline": "fix: passing max_length to vllm engine args (#1124)", "author": {"name": "NanoCode012", "email": "kevinvong@rocketmail.com", "user": {"login": "NanoCode012"}}, "additions": 7, "deletions": 1}}, {"node": {"oid": "c4f8c40ea8846586026f116aa6a1fffbe2cfeb90", "messageHeadline": "Fix vllm `batch_size` type (#1128)", "author": {"name": "Yuliang Li", "email": "40186387+xTayEx@users.noreply.github.com", "user": {"login": "xTayEx"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "fcb39a5a8ae2c95e0800ced15da4eb81464187e9", "messageHeadline": "doc_to_decontamination_query can use function (#1082)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 15, "deletions": 7}}, {"node": {"oid": "a2ed953f3843c4ba4bf0d8e0c987602459042e7a", "messageHeadline": "Additional process for doc_to_choice (#1093)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 2}}, {"node": {"oid": "e0eda4d3ffa10e5f65e0976161cd134bec61983a", "messageHeadline": "Refactor `hf` modeling code (#1096)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 379, "deletions": 192}}, {"node": {"oid": "5133c9c4f5f3bf221d37dec2827aac0d5829d112", "messageHeadline": "Merge pull request #1118 from Momo-Tori/main", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 32, "deletions": 33}}, {"node": {"oid": "c314246d8a1f6997f5f8110f82cc9c629b347ed6", "messageHeadline": "fix: _generate_configs.py", "author": {"name": "momotori", "email": "momotori@mail.ustc.edu.cn", "user": {"login": "Momo-Tori"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "057dc2d7e2188d6d12c9b5a77030e0adf1de5915", "messageHeadline": "`qqp`, `mnli_mismatch`: remove unlabled test sets (#1114)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "a7707c764e229de56eaaee6e7ffa35e285b71aeb", "messageHeadline": "bump version on cot_fewshot tasks", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "3fbdfea1c86b66438324004a47436629fcedb540", "messageHeadline": "update regeneration script, bump bbh_cot_fewshot version", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "9ef853acb5f7f629d1d0c24037692c7bde9f399a", "messageHeadline": "Revert \"Simplified `evaluator.py`\" (#1116)", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 51, "deletions": 23}}, {"node": {"oid": "7ec421657ef78e839bf4ee1fdc8e13007e473053", "messageHeadline": "fix: enlarge max_gen_toks to make output of bbh_cot_fewshot complete", "author": {"name": "momotori", "email": "momotori@mail.ustc.edu.cn", "user": {"login": "Momo-Tori"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "72e583d5fa3c38b2fad871362f2fd0ae904b9459", "messageHeadline": "Unpack group in `write_out` (#1113)", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "33dcbd49033aaa97fa762e3ee949216e7b835100", "messageHeadline": "fix: fix bug in the \"doc_to_text\" of BBH_cot_fewshot", "author": {"name": "momotori", "email": "momotori@mail.ustc.edu.cn", "user": {"login": "Momo-Tori"}}, "additions": 27, "deletions": 27}}, {"node": {"oid": "aed907735b7a39dc4ca0ad75e1b1b3fb87e5bec9", "messageHeadline": "Merge pull request #1035 from baberabb/big-refactor_dp", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 270, "deletions": 48}}, {"node": {"oid": "d588a4669052302e1d2a80331099cd0c384481a1", "messageHeadline": "Update vllm_causallms.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "86311c23f18e61b5df5787261d8bf00de07de28c", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6409fe6ca1b1c280603b06517dae353684446e8a", "messageHeadline": "data_parallel -> data_parallel_size", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "de2a60e30824ffb7603316c224c461d3b10207c4", "messageHeadline": "Describe model_comparator.py in readme", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "61b0cd29681206d16e3551ab229ff55f3287a12d", "messageHeadline": "Merge pull request #1104 from EleutherAI/simplify_evaluator", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 23, "deletions": 51}}, {"node": {"oid": "da320e91f65729f16cfd31a961399f4ac864c11a", "messageHeadline": "Merge branch 'main' into simplify_evaluator", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 3}}, {"node": {"oid": "6ec258cca5b922ebee2bcab61ae68ddea1540e87", "messageHeadline": "Merge pull request #1106 from EleutherAI/fix_stderr", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "d6f8faad5ea5eb2e1fddb3d3a4b2e5779c9ef675", "messageHeadline": "allow N/A as a possible value for Stderr", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "91279a6e8675281701c557c9f85775768e182153", "messageHeadline": "calculate std if items > 1", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "62e4ea10b6885199b794f7f1cdf6886500747847", "messageHeadline": "simplified evaluator by removing unused variables", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 25, "deletions": 51}}, {"node": {"oid": "f8c2cfcbe1cde622ceed933055f60cc102c8f4e4", "messageHeadline": "Merge pull request #1089 from h-albert-lee/K-MMLU", "author": {"name": "Hanwool Albert Lee", "email": "88315152+h-albert-lee@users.noreply.github.com", "user": {"login": "h-albert-lee"}}, "additions": 216, "deletions": 0}}, {"node": {"oid": "c86f527f45047e31da2307098ff362b955370889", "messageHeadline": "Merge pull request #1099 from shiweijiezero/main", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "ce4d9c68040c3f747f25c677633053f4ed4bef8c", "messageHeadline": "Delete lm_eval/tasks/glue/sst directory", "author": {"name": "weijie", "email": "34210233+shiweijiezero@users.noreply.github.com", "user": {"login": "shiweijiezero"}}, "additions": 0, "deletions": 14}}, {"node": {"oid": "dc5568379257024246d32350908c98f83aa27d9f", "messageHeadline": "Change the sub-task name from sst to sst2 in glue", "author": {"name": "weijie", "email": "34210233+shiweijiezero@users.noreply.github.com", "user": {"login": "shiweijiezero"}}, "additions": 14, "deletions": 0}}, {"node": {"oid": "09703d57058f9ac9a51c8e76cd8fe7c7017eb258", "messageHeadline": "Merge pull request #1097 from shiweijiezero/main", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "5dbdf58219a518b2ff14441058c6db34ec13a955", "messageHeadline": "Update template of qqp dataset", "author": {"name": "weijie", "email": "34210233+shiweijiezero@users.noreply.github.com", "user": {"login": "shiweijiezero"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c256eda81396fd5364dfba5abd3d0fa4efb9aaaa", "messageHeadline": "Debugging", "author": {"name": "h-albert-lee", "email": "gksdnf424@gmail.com", "user": {"login": "h-albert-lee"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "489814b76957d5d061fa9ec5a906517fc563ebfa", "messageHeadline": "Change spaces to underline", "author": {"name": "h-albert-lee", "email": "gksdnf424@gmail.com", "user": {"login": "h-albert-lee"}}, "additions": 29, "deletions": 29}}, {"node": {"oid": "452d49e0c2b1e3165083d96124d574d1a5690c7b", "messageHeadline": "fix `add_special_tokens=False` in generate_until + decoding fixes", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 6, "deletions": 1}}, {"node": {"oid": "4edd9c4cfd9953d45fe736582c5a1e39cca84feb", "messageHeadline": "Remove blanks at the end of a file", "author": {"name": "h-albert-lee", "email": "gksdnf424@gmail.com", "user": {"login": "h-albert-lee"}}, "additions": 0, "deletions": 3}}, {"node": {"oid": "520d8ce2d4417bef89b6096fa1c9c7f600e64cd2", "messageHeadline": "reformatted with Black", "author": {"name": "h-albert-lee", "email": "gksdnf424@gmail.com", "user": {"login": "h-albert-lee"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "95908d8bd808c73142eb00fd54005af0ee34fa6b", "messageHeadline": "Modify code for flake8", "author": {"name": "h-albert-lee", "email": "gksdnf424@gmail.com", "user": {"login": "h-albert-lee"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "9221cb57f2902a307e6a5b68d6d14c813e945d9a", "messageHeadline": "bugfix #2 remove 'context'", "author": {"name": "h-albert-lee", "email": "gksdnf424@gmail.com", "user": {"login": "h-albert-lee"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "7bc9b82bbfc1f8293518d24a648002bd484985e3", "messageHeadline": "bug fix (f-string)", "author": {"name": "h-albert-lee", "email": "gksdnf424@gmail.com", "user": {"login": "h-albert-lee"}}, "additions": 9, "deletions": 6}}, {"node": {"oid": "c64da8ac083e7da6a86fd87a9ef5fd9e427cebff", "messageHeadline": "update yaml config", "author": {"name": "h-albert-lee", "email": "gksdnf424@gmail.com", "user": {"login": "h-albert-lee"}}, "additions": 135, "deletions": 135}}, {"node": {"oid": "7003cddff774f7101cacabc0a780634792c621ed", "messageHeadline": "Merge pull request #1077 from EleutherAI/patch-scrolls", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1b14602e2a7bd767cd2a4bcc8222b291305736d7", "messageHeadline": "implementing kmmlu", "author": {"name": "h-albert-lee", "email": "gksdnf424@gmail.com", "user": {"login": "h-albert-lee"}}, "additions": 217, "deletions": 0}}, {"node": {"oid": "eb834c989bd0b530b6c9b669f839b662af481a99", "messageHeadline": "Merge branch 'main' of https://github.com/EleutherAI/lm-evaluation-ha\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 1}}, {"node": {"oid": "12f260cfc9ec9d26d9a5733a00636112135e7ecf", "messageHeadline": "fixed enumeration", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e5dfd0304efa6668bcf61aa3eb22ce8abe337eaf", "messageHeadline": "Merge pull request #1074 from EleutherAI/lintangsutawika-patch-4", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 1}}, {"node": {"oid": "965c5330d9124709225f5ce297cf26bb2f183977", "messageHeadline": "formatting", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ce079c961536897297d629d9d414beab3e2a21d3", "messageHeadline": "Update minerva_math_algebra.yaml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "ba7ba91035ed710a86fad4618e2bd0525314741a", "messageHeadline": "Update _cot_zeroshot_template_yaml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "a2cc877bc1070155f607839891ef42a14c31c7c2", "messageHeadline": "Update _fewshot_template_yaml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "361ba192a47ce1e2676f8ca3dbe968831b02fdac", "messageHeadline": "Update _zeroshot_template_yaml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "8c05c6be999690bfaa456d7bcd29149fd5d3aaac", "messageHeadline": "Update _mmlu_flan_cot_zeroshot_template_yaml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "4e34a6e86de3826c3e463d1dbece14abce8eed56", "messageHeadline": "Update _mmlu_flan_cot_fewshot_template_yaml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "a6d28ea763161f058d4b7f3555d3d256d49f5b54", "messageHeadline": "Update _cot_fewshot_template_yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "1c62da1dedafc83e7460bc7c845346a52a317630", "messageHeadline": "fix `batch`", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "38e3b73f0e84325c523afb7704a2e0c7b1b01a7f", "messageHeadline": "nits", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8b74beaaa78b0d77bc0893af15becdb801a1986c", "messageHeadline": "fix z-score and print; rename script", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 59, "deletions": 19}}, {"node": {"oid": "b99ad796d08b273899c8cf41fdee773698a65045", "messageHeadline": "fix errors", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 19, "deletions": 13}}, {"node": {"oid": "a4188e1d4b812573aeaceda41c11e6f539955b67", "messageHeadline": "nits", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 3}}, {"node": {"oid": "f44aa85fb957db9a2084bc028186ee551643927a", "messageHeadline": "add script to check vllm, hf equiv", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 78, "deletions": 0}}, {"node": {"oid": "f0b9649146b30c966bf7fa159e3ad45133acc672", "messageHeadline": "Merge pull request #1063 from EleutherAI/haileyschoelkopf-patch-2", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "dc1c816b089b42964c31e6de260e154a24f6c674", "messageHeadline": "added `_encode_pair`", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 16, "deletions": 7}}, {"node": {"oid": "f7fb109c2a49ac5285f23baa6dcb7b686ba751db", "messageHeadline": "Merge branch 'main' into haileyschoelkopf-patch-2", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 16, "deletions": 17}}, {"node": {"oid": "f721c0f0f608b3af525723f239f094053b3aec40", "messageHeadline": "update README.md", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "19f745aa1e853fe61fddb66e6cf1f1c4a79caeb7", "messageHeadline": "typo", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "e86e7b2767e0bbe2413708197641e28d04f9feaa", "messageHeadline": "Merge branch 'main' into big-refactor_dp", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 11, "deletions": 11}}, {"node": {"oid": "f74c0feb2667bdd5e67372095791e27f6a44b470", "messageHeadline": "Unit tests run on `main` branch", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "eeb99721899bfb3a4a394b5532dc6563f4629518", "messageHeadline": "run CI on `main` branch PRs", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "b957a080325a6db1a8df47c43b18009b33cf0a7b", "messageHeadline": "Merge pull request #1066 from EleutherAI/haileyschoelkopf-patch-4", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 6}}, {"node": {"oid": "d83fc511873cc32b8a3282feaf91943d0a6cdf22", "messageHeadline": "Update model_guide.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "f73c2bc8d12a9fd1cfa8ff5e42847cb6e27f59bf", "messageHeadline": "Update new_task_guide.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 4}}, {"node": {"oid": "6f76ee0eb07887f3a310d609e5c237574faa2480", "messageHeadline": "Merge pull request #1064 from EleutherAI/haileyschoelkopf-patch-3", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f55595c869ec75a51a84531b42b6447c3236e2de", "messageHeadline": "Merge pull request #1065 from EleutherAI/haileyschoelkopf-patch-4", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "05835d410b1b741be8d4f24e2123e0864c3b5277", "messageHeadline": "Update interface.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "ab7e8192e522d0c7fdc36561894f70d4d23dba67", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 5}}, {"node": {"oid": "4efa4a54be3e0598e0aa531e3c4874218669da24", "messageHeadline": "Update pyproject.toml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c1c446511fa99b5039f23be744169e5be5e9d532", "messageHeadline": "test", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 21, "deletions": 19}}, {"node": {"oid": "5c9fb98079e81ada920833bb515a30929670994b", "messageHeadline": "Merge branch 'main' into big-refactor_dp", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 77, "deletions": 77}}, {"node": {"oid": "296732d4b0df7f9c3c5ca7b60a49092f10eada60", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "3c75a2c77375769f5c483f084e95ea8b8514c9bb", "messageHeadline": "Merge pull request #1062 from EleutherAI/haileyschoelkopf-patch-2", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 19}}, {"node": {"oid": "15953e422db9b9e56b44f38633e9b35fe98b629a", "messageHeadline": "minor readme updates due to refacotr", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 7, "deletions": 5}}, {"node": {"oid": "3512f742b3a36f4531ffaa0f70d96203f13c6020", "messageHeadline": "Update samplers.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 19}}, {"node": {"oid": "fde2b77efc7bfe77a5c36a9a9d0997da2eaedc53", "messageHeadline": "Merge pull request #1061 from EleutherAI/haileyschoelkopf-patch-2", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 19, "deletions": 0}}, {"node": {"oid": "dea8d7bdfb87175f2ce00211b58579f81cbafcbb", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 19, "deletions": 0}}, {"node": {"oid": "c9bbec6e7de418b9082379da82797522eb173054", "messageHeadline": "Merge pull request #1060 from EleutherAI/fix-mmlu", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 61, "deletions": 61}}, {"node": {"oid": "57e017ff6dfcc191ad3f12c23e4414369b283003", "messageHeadline": "formatting", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "c30b71c39757d6d8db5ee0ce5093881cec930e90", "messageHeadline": "add \\n\\n to end of description", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b32b37932f7929a9899947c149eb2a15a72c5086", "messageHeadline": "add \\n\\n to end of description", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 58, "deletions": 58}}, {"node": {"oid": "3bb230d3d8d85e40944a58196b87190d40404a59", "messageHeadline": "add tokenizer options to model_args", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "d0dc5bf99f094174642b3fa274bb2076cad4f412", "messageHeadline": "nits", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 6, "deletions": 9}}, {"node": {"oid": "1a0f57d737b30138caaaca623b8977f290cfd246", "messageHeadline": "fix tokenizer", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 12, "deletions": 6}}, {"node": {"oid": "2312c2630882f4448307922c0b22ada0151fb9c6", "messageHeadline": "add link", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "d4d22e473474c134431607e5e3d4bd8bbee2ee0c", "messageHeadline": "repetition + nit", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 3}}, {"node": {"oid": "dcf330120805ea0ffeffca40e81c6eeb61271c55", "messageHeadline": "add vllm", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "82dafb34ff176c46f7ca5e65e1809f509c4fedf8", "messageHeadline": "nits", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f45f67e4e262e179cb47e732dce072268820c866", "messageHeadline": "nits", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 3}}, {"node": {"oid": "7d5364a58e2836593585e116c80233ad84aff937", "messageHeadline": "fixup for md files", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "98fba5f3564a1f27a0b54f9819919157eb3266a5", "messageHeadline": "Merge branch 'big-refactor' into big-refactor_dp", "author": {"name": "Baber Abbasi", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1515, "deletions": 101}}, {"node": {"oid": "7afae7b5e8dbd9da6f41cdf43bdd309840673911", "messageHeadline": "Merge pull request #1051 from EleutherAI/haileyschoelkopf-patch-2", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 15}}, {"node": {"oid": "553db3d0cd4ab1b5c15697531311782c880cd672", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ca3afc2efc4d39903694bfae042ea0ccb90b440c", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 15}}, {"node": {"oid": "f1b64f688fa4e8ab23873293e6d7ee43b7826b97", "messageHeadline": "Merge pull request #1008 from EleutherAI/openai_completions", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 220, "deletions": 5}}, {"node": {"oid": "c3d97e40b5a65325f5b5efb5f36fabf29fe4c2ce", "messageHeadline": "Merge pull request #1049 from EleutherAI/StellaAthena-patch-3", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 10, "deletions": 18}}, {"node": {"oid": "2a5e49fe97fb35a1e6f681553937cfbe7a1ea4a9", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 10, "deletions": 18}}, {"node": {"oid": "fb1c4a8a654e7cf5ad82f494f7febe0334d9d7b9", "messageHeadline": "Merge pull request #1025 from EleutherAI/more-docs", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1235, "deletions": 2}}, {"node": {"oid": "fd0ae5a1b5a08f01fcc9ebee85efb348432bc0f9", "messageHeadline": "Merge pull request #1047 from EleutherAI/StellaAthena-patch-3", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 36, "deletions": 42}}, {"node": {"oid": "6c029a236f938ff726e652fb46bb40fa325bb84c", "messageHeadline": "Merge pull request #1048 from EleutherAI/even-more-docs", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 97, "deletions": 20}}, {"node": {"oid": "b75ac3778a1955d32766634e5f79b56da45d63ee", "messageHeadline": "add more text, add process_docs example", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 97, "deletions": 20}}, {"node": {"oid": "23f8d1e323f00514b94f39f6829ce92c6defa14b", "messageHeadline": "Merge pull request #1037 from baberabb/big-refactor-mps", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 14}}, {"node": {"oid": "cca3d79cf39fb01a6931ffe5def22d1235760e88", "messageHeadline": "Merge branch 'big-refactor' into StellaAthena-patch-3", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 11, "deletions": 11}}, {"node": {"oid": "11176214d6eb51851ff9c95e93f3894d664f8fa0", "messageHeadline": "Merge pull request #1046 from EleutherAI/lintangsutawika-patch-4", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 12, "deletions": 12}}, {"node": {"oid": "994a2ebf4e8757be444d7371192999c73b46c9b9", "messageHeadline": "modified note on lm-eval/lm_eval command alias", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "aea0c59e60c23728a0e9253ec104fb16938b5bfc", "messageHeadline": "Miscellaneous documentation updates", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 36, "deletions": 42}}, {"node": {"oid": "61bb86367897d183e172bec032195c87a27cb4ab", "messageHeadline": "[Refactor] Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 9}}, {"node": {"oid": "6432933d8a1f6e9daa865219a3029ef7917c30a5", "messageHeadline": "updated notebook", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 308, "deletions": 315}}, {"node": {"oid": "d40938864338d8e6a125f0654a69763d45f09e1a", "messageHeadline": "nits", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "d4edd280384112ad2877aeef510c278acd41f7e3", "messageHeadline": "fixup", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "da2d160b94abebd533318360c7e5009667c198c9", "messageHeadline": "fixed chunking", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 53, "deletions": 2}}, {"node": {"oid": "b97c561f53e444e52e34a3c412f9d755fcf98e5a", "messageHeadline": "simpler request chunking", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 8}}, {"node": {"oid": "c57ca81a36d5b1b832d126765dbe8e74af15b307", "messageHeadline": "Merge remote-tracking branch 'origin/big-refactor' into big-refactor_dp", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 11}}, {"node": {"oid": "336cc4553521eb0c8f737d1357912506bcbdb2d6", "messageHeadline": "Merge remote-tracking branch 'origin/big-refactor' into big-refactor-mps", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 11}}, {"node": {"oid": "42f486ee49b65926a444cb0620870a39a5b4b0a8", "messageHeadline": "Merge pull request #1038 from baberabb/big-refactor-lintang", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 11}}, {"node": {"oid": "38ac7b2a1a57b0295a7e926d1f1e1df6bef8b017", "messageHeadline": "fixed sampler issue with new default num_fewshot value; n_shot value \u2026", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 11}}, {"node": {"oid": "7bb147b53f406f94b2f6e1e5b0601ded263fe8a3", "messageHeadline": "change torch req for mps", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 7, "deletions": 14}}, {"node": {"oid": "a0787a9f763866e9dad587b4ae526aa932ffd479", "messageHeadline": "Merge remote-tracking branch 'origin/big-refactor' into big-refactor_dp", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 361, "deletions": 97}}, {"node": {"oid": "6359f08385fbe013a4f32a2cb9d40d3ad2da0177", "messageHeadline": "fix imports", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "055d99b5a947a974737b37010f7a467c88e5dcc6", "messageHeadline": "rename function", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "9850c74586f8ceb90b4f7516a65948fe523f6844", "messageHeadline": "update readme for vllm", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "1f504aa42c42d0628706896ba4247319fe32f6da", "messageHeadline": "remove unused imports", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 0, "deletions": 4}}, {"node": {"oid": "694dc64215f5a6e562600db54c5452796a752a53", "messageHeadline": "fix data + tensor parallel", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 11, "deletions": 7}}, {"node": {"oid": "5075de6063c4e4761db160c70df5b01d582eed1f", "messageHeadline": "fix chunking of inputs", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 13, "deletions": 10}}, {"node": {"oid": "dc5b3d5df022bd7f24fe01911b3c8df179473040", "messageHeadline": "Merge pull request #1031 from EleutherAI/versioning", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 253, "deletions": 6}}, {"node": {"oid": "39c2bb4e41ee6ebaf2d385510427cac0674af209", "messageHeadline": "Merge pull request #1033 from EleutherAI/lintangsutawika-patch-4", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "b87ee15376e90b140e3dbf4d5621c8432110cf7c", "messageHeadline": "Update evaluator.py", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "9c0d4e930065559c368d09dd19f8a53324622dad", "messageHeadline": "test data parallel", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 52, "deletions": 27}}, {"node": {"oid": "0801d1ff678e6bc0bc900d5f0e22cb38d3ea5994", "messageHeadline": "add string to ignore", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "e6d4ec395f84d2757dd98592ed2881ee20618d98", "messageHeadline": "Merge pull request #1024 from EleutherAI/fix-mbart", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 9, "deletions": 6}}, {"node": {"oid": "52f75f0e1bb4db709cb92ac01b37d586583d796d", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 98, "deletions": 85}}, {"node": {"oid": "b072bb0da6547c7f34c4d779703225d51ae1740f", "messageHeadline": "Merge pull request #985 from EleutherAI/fewshot-in-table", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 98, "deletions": 85}}, {"node": {"oid": "331d7c51e46a0c2238307f578303999c399b08a8", "messageHeadline": "Delete lm_eval/tasks/siqa/bigbench.yaml", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 16}}, {"node": {"oid": "20463a3f418766cd8ce6918b3d48d14d9d0a0fca", "messageHeadline": "Update default.yml", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "6e5137c7acf34871a4dd92d5d29660d1f5569456", "messageHeadline": "Merge pull request #1032 from EleutherAI/big-refactor", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 14, "deletions": 0}}, {"node": {"oid": "5dcbcd63090136ceed0bbc52aaa7dcb1914621e2", "messageHeadline": "Rename bigbench.yml to default.yml", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "5ded4d5bdf2105e0aef2a8cb3109a1700508f65c", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "37a463518aebdf099d2b1f55db66df8792151681", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 124, "deletions": 122}}, {"node": {"oid": "525afc17beeaae17ddb1159b7ec7b1a51992833d", "messageHeadline": "add documentation", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "43f615c9f2a17f2c73bb3dd0d44deeba9fdf06b9", "messageHeadline": "update version", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "94cdface7ee744d385389554ceebc0249f05cdd6", "messageHeadline": "update version", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "0d03a9f3c1c9e0bdd6642d853485c9068e9446cb", "messageHeadline": "add versions", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 246, "deletions": 5}}, {"node": {"oid": "9e65d0cea8216e08707b87c5c3dcbd08581dbb87", "messageHeadline": "fixes how num_fewshot from config is read", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "9f518392b0e6ce41ecc6008af6eab507b3690c84", "messageHeadline": "resolved merge conflict", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2125, "deletions": 526}}, {"node": {"oid": "37ccb191501169f3284f55416743b8551652b4b4", "messageHeadline": "force configs with num_fewshot=0 to ignore manual num_fewshot setting", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "61d0f137170bba4233ba4828637bc5a78ad7d064", "messageHeadline": "use all_headers for headers", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 13}}, {"node": {"oid": "19738ed24160275e73febc5b4368dc2daa2c7a55", "messageHeadline": "renamed template file to make it easier to search", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 0}}, {"node": {"oid": "db8928b0839ef9fbf3724674eb0df02935bf826e", "messageHeadline": "renamed template file to make it easier to search", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 68, "deletions": 79}}, {"node": {"oid": "5b42436b7d8c2efccb2bf9fac6a27939625380ed", "messageHeadline": "resolved merge conflict", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8ea23e2dd03e74078734533c3edd72973b2f0b05", "messageHeadline": "updated oa_completion and fix import error", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 8}}, {"node": {"oid": "5d3bf2e73e9356127adcccd4976e86ca2cb825bc", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1163, "deletions": 336}}, {"node": {"oid": "bf26d97968e561c83a91a4fd0d8501efebbe344c", "messageHeadline": "Merge pull request #1029 from EleutherAI/bbh-fixup", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 235, "deletions": 225}}, {"node": {"oid": "3b9640b8c46359da3e3ee9af91550199f3b8caca", "messageHeadline": "removed flan from the naming", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 224, "deletions": 231}}, {"node": {"oid": "e7afee52a26f6c4e6ffa1bf832739646217e89f3", "messageHeadline": "Merge pull request #1030 from EleutherAI/social-iqa", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 51, "deletions": 0}}, {"node": {"oid": "00c05d631159d9f65fbb9e9e607b2cf603911275", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 20, "deletions": 41}}, {"node": {"oid": "5535fbd43c0177487883273e3e285312e02dff78", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4706fbb200f6117d6da460ce43d391334e2982e4", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 14, "deletions": 19}}, {"node": {"oid": "1112990696263ecbd39c39303fea3a3cad793fd6", "messageHeadline": "Create bigbench.yml", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 14, "deletions": 0}}, {"node": {"oid": "babdc8d682b4b1702b7cee2473fe7f706d7ba397", "messageHeadline": "Create README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 42, "deletions": 0}}, {"node": {"oid": "1e5ca7a403ea1d52791e93935141b6ae473e4e0d", "messageHeadline": "'flan' in groupnames optional for all variants", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 10, "deletions": 3}}, {"node": {"oid": "e408c2ce8ebf8a2ade0d7881df471020c2fae6c5", "messageHeadline": "make 'bbh' a group name for flan cot 3-shot, add stopseqs", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 11, "deletions": 1}}, {"node": {"oid": "b189066ddf970be1ec6d0deec31c121b9b83a4c9", "messageHeadline": "Merge pull request #897 from EleutherAI/orz", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 35, "deletions": 2}}, {"node": {"oid": "d665361002bba0a97a05df689480984452fccbb3", "messageHeadline": "add --gen_kwargs to interface docs", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "a3d082778e84f8f708cf849ece855d4a4dab5fc0", "messageHeadline": "Merge pull request #1011 from baberabb/big-refactor_vllm", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 526, "deletions": 29}}, {"node": {"oid": "4277dc35b375dd7e186162692be587ff4da7dd8f", "messageHeadline": "fixes for precommit", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "7fa496712037a36cb4afcb35120771dc987926b5", "messageHeadline": "fix precommint", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "05326b04d45feade6f4c4abb24ae017129e09cee", "messageHeadline": "skip vllm test (requires CUDA)", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "b228273919ecc0c0c9e3f6ab2e0e8e6b3ff3a77c", "messageHeadline": "Update unit_tests.yml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "08074cf92293c1cefa4d4e08005690f07896c3e0", "messageHeadline": "Update vllm_causallms.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 4}}, {"node": {"oid": "581dd9ff66ea1eb392c22eca5523438815abe012", "messageHeadline": "Add vLLM to README", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 24, "deletions": 3}}, {"node": {"oid": "b01c4893af48da9aec961762ffed12125b41e16f", "messageHeadline": "Fix gpu_memory_utilization kwarg typing, change to conditional import", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 16, "deletions": 4}}, {"node": {"oid": "d86400b5bcbf292a85ed266da41ab71e62e3135e", "messageHeadline": "always import vllm model", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 5}}, {"node": {"oid": "00b48ef87c206e840765c426235ac12e3161653f", "messageHeadline": "Update pyproject.toml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b03889e31a477eee9a51049e72dd081a24320fec", "messageHeadline": "fix version number", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "706268db0185cb812e53b086655527b6df9adc86", "messageHeadline": "add link to find the notebook", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 692, "deletions": 698}}, {"node": {"oid": "bbd6d051a11a10c83a74bc3c54daeaddb25b35fc", "messageHeadline": "add auto batching", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 0, "deletions": 5}}, {"node": {"oid": "37f10cadc26e8c6df92de7ed54223267948143ad", "messageHeadline": "add auto batching", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "28348b80a268eb3ac68a4dd5917f842685916f4c", "messageHeadline": "add notebook for overview", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1168, "deletions": 0}}, {"node": {"oid": "2c20df084c0a86e38064068601a4d557c1a4563d", "messageHeadline": "lazy load import in tests", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 7, "deletions": 3}}, {"node": {"oid": "f66730c438f126f1c682c5f1830bffad1ebf34a1", "messageHeadline": "fixed how messeges are sent to chatcompletions", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 58, "deletions": 51}}, {"node": {"oid": "026005ac78c6ffbb54d0fce02d57ad3f2da27a06", "messageHeadline": "Add tests for VLLM", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 42, "deletions": 0}}, {"node": {"oid": "a2fd682d578d1f79605ab2d1f6d068aa2ab215b4", "messageHeadline": "removed model tags", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "306e84fb74b5409fd3290fa2fc02c5be64c95efc", "messageHeadline": "readd old completion api", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 299, "deletions": 0}}, {"node": {"oid": "7ab782ec92305d0fd4f73b5e3c414949461ad2d1", "messageHeadline": "use Seq2Seq backend where either can be loaded from HF", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 9, "deletions": 6}}, {"node": {"oid": "30936bc7c6e03c41e1f0d45b68a02b0f39e399f2", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 7, "deletions": 2}}, {"node": {"oid": "8042479da8645e38804088696224abeb088fce73", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a5b65c9b4769ab4553b377c5a027f4b7410f6071", "messageHeadline": "udate", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "b64e592d3820bf3d9a804c5203319d8730280391", "messageHeadline": "fix missing lines", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 7}}, {"node": {"oid": "47933b98aa7734bda660d03c6fb51ab1b1c18c14", "messageHeadline": "gen_kwargs is parsed once", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "a2992d28cd9b95f9e9cb28791f6642f31b6635ef", "messageHeadline": "Merge pull request #1018 from EleutherAI/cleanup-examples", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 32}}, {"node": {"oid": "fde2a1d5fd9fb65c653dd9727a91cc191136e7f1", "messageHeadline": "update on args", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 3}}, {"node": {"oid": "470059f67ce1d96cd89d9e832f2811765faa2a42", "messageHeadline": "merge conflict", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 11296, "deletions": 735}}, {"node": {"oid": "3029679553f87433c96e947c686e58dafa1fa28a", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 60, "deletions": 33}}, {"node": {"oid": "9d030712672ad7893c3d112b91ee13aa94df320b", "messageHeadline": "Merge pull request #1020 from baberabb/big-refactor-readme", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 1}}, {"node": {"oid": "4fd6a731338fb233357cc54d77d1aba95f42ec52", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "d3e46bb3f27842a21c0fbdeb1b74cc2968d3846f", "messageHeadline": "Update README with additional usage instructions.", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 5, "deletions": 0}}, {"node": {"oid": "50107e10e7cc4d203cba756e6da7720d34f11d08", "messageHeadline": "Merge pull request #1009 from EleutherAI/fix-stopseqs", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 9, "deletions": 5}}, {"node": {"oid": "4f79fa7b7d8196cdeeabad0c91dedd702d8eb86d", "messageHeadline": "fix args", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 16, "deletions": 4}}, {"node": {"oid": "953bfdd2468530793583dc271a22f49349e18152", "messageHeadline": "Update vllm compatibility in README.", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 11, "deletions": 11}}, {"node": {"oid": "b570fce77945be83d9897e33a12132cc83b81875", "messageHeadline": "add dependency", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "af88402853e0a41e47cd804ffdc1fbb70b462655", "messageHeadline": "fix tensor parallel", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "5ba54539ee9817576196156892d73451bc75730d", "messageHeadline": "fix imports", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 5, "deletions": 4}}, {"node": {"oid": "ce164bb1c00671436484ad915b7edd3237b81044", "messageHeadline": "fix imports", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 26, "deletions": 19}}, {"node": {"oid": "667fc8372273c2d951e9c05fed489a9853cdf94a", "messageHeadline": "disable tqdm", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "ac664bce0b18dbd83cb401e3de7c611406840f5e", "messageHeadline": "fix logliklehood_rolling", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 5, "deletions": 3}}, {"node": {"oid": "9c7cfcee85e729fe7fbd209b44b172c0d91bcb7d", "messageHeadline": "remove examples/ subdir", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 32}}, {"node": {"oid": "2362ab414f3c8689b11b5c71922c70b92da0218e", "messageHeadline": "bugfix", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e54917095deb7e2b446ef2588f7a1ad2078cf4dc", "messageHeadline": "fix greedy_until", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "a1403c8f8c1bc7bfc8a854069413a37940de9c14", "messageHeadline": "Update model_guide.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "24f4e8d7be53afa0ec119b854a4f20b27bcc71fa", "messageHeadline": "(temp) fix import", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 6, "deletions": 1}}, {"node": {"oid": "f2890cc17f90d6dff69175fe20ee8e57de87e9f7", "messageHeadline": "bugfix", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "918689794882b2d13d50cac536f6ab4127840ce7", "messageHeadline": "Merge remote-tracking branch 'origin/big-refactor' into big-refactor_\u2026", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 296, "deletions": 1}}, {"node": {"oid": "0635af130d2489e1ffe2a093a0a262787aeccdce", "messageHeadline": "bugfix", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 20, "deletions": 12}}, {"node": {"oid": "103a10e32669190efe9090c5335ea45180e909eb", "messageHeadline": "bugfix", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "a2920e3d66477f5d7896bbe8c1215ba0ecda711d", "messageHeadline": "add vllm model", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "815cd29dcf4e51ed5fa18ad062574328a2118059", "messageHeadline": "Merge pull request #967 from EleutherAI/upstream-ggml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 278, "deletions": 1}}, {"node": {"oid": "3d8bc82f3c066555955a476e15bb5e5306152f14", "messageHeadline": "add vllm model", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 327, "deletions": 0}}, {"node": {"oid": "f7873a494520f0bd2251917ec6cdfebd524b65d6", "messageHeadline": "update multi-token stopsequence handling", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 9, "deletions": 5}}, {"node": {"oid": "12e92616db28b9a7324ca78012a84a222cb05fe6", "messageHeadline": "fixed parsing of API returns", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 9}}, {"node": {"oid": "bfe69313a81cc6a98b6c075db97d5c1f5c99c429", "messageHeadline": "API update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 39, "deletions": 318}}, {"node": {"oid": "6ac42518086c675705cf7581ce09d17a8886feb1", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 13455, "deletions": 879}}, {"node": {"oid": "9c3ba7d4e8c3022ea9cdf00193d70a51fd6a4e6f", "messageHeadline": "renamed greedy_until to generate_until", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "e3644fcc74efebef96069b7a655fb429cbec3772", "messageHeadline": "Merge pull request #979 from EleutherAI/benchmark-documentation", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 18, "deletions": 0}}, {"node": {"oid": "f87b864f27c982cab50a269f1f6db15fb8d4c8a8", "messageHeadline": "add ggml tests", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 151, "deletions": 0}}, {"node": {"oid": "1e9c8b59562ed574e3ddbe1b2b3a9a6997a4071e", "messageHeadline": "add typehints", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 17, "deletions": 9}}, {"node": {"oid": "b22f34401be695736695fda8a3aecbc177850d56", "messageHeadline": "add docs", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 47, "deletions": 4}}, {"node": {"oid": "afda6551e9e8d8021c1fdd35d2aad0fbe63f3919", "messageHeadline": "Merge pull request #958 from EleutherAI/verbosity-rework", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 83, "deletions": 42}}, {"node": {"oid": "9b596e8f9155064cdab7630633c92711372a39bb", "messageHeadline": "add initialize_tasks() to interface guide in docs", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "032820ddba95956f4b30ca98babe7bdc9bede523", "messageHeadline": "initialize tasks in test files", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "2c0c345a37c33585b6d7f814e208cc2601f616a6", "messageHeadline": "Merge pull request #1002 from EleutherAI/bugfixes-bigrefactor", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 9, "deletions": 5}}, {"node": {"oid": "487767ab0d0a5fc8d817dd114a5a385f70f50535", "messageHeadline": "update bigbench datasets to mirror, rename groupnames", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "2444b36436b13d3b82a818e7318253868acd93a5", "messageHeadline": "fix error when using wildcard task names", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "aa9e106235305ff54c43dd607dd04dc6174c371d", "messageHeadline": "fix error when using wildcard task names", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "c0000e036a1a881a7acbcb8f1312f2bb59663916", "messageHeadline": "Merge branch 'verbosity-rework' of https://github.com/EleutherAI/lm-e\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "66eed4bbaf950a1bcfa20a5d25a54639225f0e80", "messageHeadline": "update script to use initialize_tasks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 3}}, {"node": {"oid": "a58674aef370fef4d7bd632f0fa315b59c43da19", "messageHeadline": "merge cnflict resolved", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 843, "deletions": 126}}, {"node": {"oid": "115b2bd463c2516ee2bd911d49bf419f052b1400", "messageHeadline": "Merge branch 'big-refactor' into verbosity-rework", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 843, "deletions": 126}}, {"node": {"oid": "f76941ef15d0333443648940726428abf09272ce", "messageHeadline": "Merge pull request #1001 from EleutherAI/fix-alias", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "056569e5550a22fe874279cc8675281f6652b3bf", "messageHeadline": "Merge pull request #997 from EleutherAI/fix-tests-refactor", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "bf6cf1debe43c60012b448db006a58702b3d899e", "messageHeadline": "format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 3}}, {"node": {"oid": "b4c1ba30c608f28365955eb51df74cf8333c9477", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 55, "deletions": 15}}, {"node": {"oid": "2314ff4fdf6b9c6213154809f71b05881b70b0fe", "messageHeadline": "will check if group_name is None", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "84c2cda9b4c472c4e1048357f5542a249030b544", "messageHeadline": "Merge pull request #999 from EleutherAI/port_master_squadv2", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 55, "deletions": 15}}, {"node": {"oid": "f40b7d0efcc192ab54c78684b00ddf9e4ef37254", "messageHeadline": "precommit", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 55, "deletions": 15}}, {"node": {"oid": "f7ba0d63a97e358123cf7df6731f54e0a7e6d6c8", "messageHeadline": "Merge pull request #971 from EleutherAI/port_master_squadv2", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 849, "deletions": 175}}, {"node": {"oid": "0d209e2e8be992fe9aac8fd0555e21687dde9dbe", "messageHeadline": "removed provide_description arg", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 3}}, {"node": {"oid": "f56e2c472298e5f952520dc216f1965270db70ed", "messageHeadline": "Merge branch 'port_master_squadv2' of https://github.com/EleutherAI/l\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 11}}, {"node": {"oid": "acda980f4a3256591e39aee2bc170939bf2e2f25", "messageHeadline": "Merge pull request #996 from EleutherAI/update-pytask", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 11}}, {"node": {"oid": "10cc0a56886324e72310f50ee0114c021453c792", "messageHeadline": "edits and format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 5}}, {"node": {"oid": "cfbad4c4d0e4de9fe879f3a268d014f123188760", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 127, "deletions": 135}}, {"node": {"oid": "57ce579c2da7fa672146d18c5b67f2926c7f6f17", "messageHeadline": "widen HF model test tolerance", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1eed1cb5c73cbf8d2effd89be9b564e56bae7b8d", "messageHeadline": "update gsm8k name", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a745d589174ff436aa8d2a5dadd1cd49c4812988", "messageHeadline": "remove provide_description flag", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 11}}, {"node": {"oid": "c9a3fd3f677cf063deca35ca0222d3aa8d725e41", "messageHeadline": "update squad file", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 17, "deletions": 1}}, {"node": {"oid": "5a4fc8fb2de845e9110e2af6c0800347aca85e45", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 127, "deletions": 135}}, {"node": {"oid": "e35b06671831910a9d99827dfef62b4781089da4", "messageHeadline": "Merge pull request #987 from EleutherAI/alias-fix", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 8, "deletions": 12}}, {"node": {"oid": "d9230aa8868f71ad8f68cdce7463cece3be67ec7", "messageHeadline": "removed unused variables", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "7760573f7de555c6d8e8fba0a5be272f309d426f", "messageHeadline": "enables tasks of different group but same task_alis (for example, if \u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 11}}, {"node": {"oid": "81495b0ae9b8a75c71b57af283dcc5ee276428d3", "messageHeadline": "num fewshot is printed in table", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 18, "deletions": 2}}, {"node": {"oid": "8ffd263066509d2498d700c408a203e35572b67f", "messageHeadline": "Merge pull request #981 from EleutherAI/mmlu-fixes", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 119, "deletions": 123}}, {"node": {"oid": "69186ef227542f1926f673d4b7fa4f03637a567b", "messageHeadline": "fixes", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 119, "deletions": 123}}, {"node": {"oid": "056c9d8561ff2f2bacf7508c124443f17489008f", "messageHeadline": "added initialize_task and updated where eval_logger is imported from", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 25, "deletions": 21}}, {"node": {"oid": "574e565abce83e9712169fd0f4adc874221f54eb", "messageHeadline": "Merge branch 'big-refactor' into verbosity-rework", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 4304, "deletions": 3685}}, {"node": {"oid": "5175375056f3eebb199ed407fff4a5340f6349d3", "messageHeadline": "updates to scrolls", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 62, "deletions": 45}}, {"node": {"oid": "097c92537308bdc14fa86192e8959f5d89b02975", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 8}}, {"node": {"oid": "6c812bb54749f1627d4c681a5ff99040891ea090", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 1}}, {"node": {"oid": "9a64e6423d1bb4b10035a86cf0aa55749a7b5025", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 14}}, {"node": {"oid": "7513783645d1b3a3321b6f8eba43219aaa35b99a", "messageHeadline": "add scrolls", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 490, "deletions": 0}}, {"node": {"oid": "a35206191acac1776761e737b66e0d04975d21b9", "messageHeadline": "readded task descriptions", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 26, "deletions": 0}}, {"node": {"oid": "e376ad8a8a96d9bdaaed715612de24605a854714", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "23bb863605e311810fd3e2bbc77cf29589a24093", "messageHeadline": "add description on task/group alias", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 17, "deletions": 0}}, {"node": {"oid": "d3f429ac175b87c1f7a83908c3de8f39745c737c", "messageHeadline": "removed yaml version of squad", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 349}}, {"node": {"oid": "9817e7c2c925307d28af3c8fec9dea6116d81c87", "messageHeadline": "removed yaml version of squad", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 211, "deletions": 0}}, {"node": {"oid": "7b60ead42f00458cbcb59391088ebbf83d220ee1", "messageHeadline": "added comment line", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "e1d5c849740d3ac52e9a745ba2396306bf0145a0", "messageHeadline": "moved squad.py and format changes", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 9}}, {"node": {"oid": "8bf55a204d198f56228e572a74d9ef39a9fa8c74", "messageHeadline": "add squad from master", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 318, "deletions": 37}}, {"node": {"oid": "b7a4ea062159ea4b8e4e41cbe633e730b1925ce5", "messageHeadline": "Merge pull request #952 from EleutherAI/fixup-logging", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 7}}, {"node": {"oid": "01227a7e1dac750ad59cb37149faf914f15bbd29", "messageHeadline": "Merge pull request #965 from EleutherAI/remove-goldalias", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 14, "deletions": 45}}, {"node": {"oid": "815f59e69039b7d416e2bdf35164e36ab34a705e", "messageHeadline": "Merge pull request #922 from EleutherAI/mmlu_subgroups", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 3422, "deletions": 3638}}, {"node": {"oid": "44124d95a25195da0a3d129dddabb37c43ba5ce2", "messageHeadline": "add space after -", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "491d479930824577922741330197321fc88cdbb9", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1e1dbaf3545277a3ce0e890e36f4f528f6f3139d", "messageHeadline": "remove samples in the table", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 0}}, {"node": {"oid": "3533e4b9536c6dd8ee9903ad2e43d083814f231d", "messageHeadline": "Merge pull request #966 from EleutherAI/fix-verbose", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "b2b6a90bfef5cf3177ebb668c13c6f8f4bef9c39", "messageHeadline": "upstream GGUF/llama.cpp model to big-refactor", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 127, "deletions": 1}}, {"node": {"oid": "5f3b8bf668047ce6433c4f93c26a32499c78d992", "messageHeadline": "fix bug with args.verbose", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "6f700f98bc04457ddbe1c9263f598bb57088e08b", "messageHeadline": "Merge pull request #815 from EleutherAI/model-written-eval", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 864, "deletions": 0}}, {"node": {"oid": "93d088c8461f7a68d91f4985bdd935188c90e178", "messageHeadline": "fix gsm8k regexes", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 10, "deletions": 11}}, {"node": {"oid": "9f09016db25c09d3e5dff2a0e473f29f6108118e", "messageHeadline": "Update ignore.txt", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4d9b928ef283a9c83c40f0994d1fc4c4aa448811", "messageHeadline": "Update _generate_configs.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "b9f0d0d38656c5f4260d74585cf9f779d5026960", "messageHeadline": "make sure gold_alias not in docs", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "37ac5f468d2d67dddc2b68c4ea047a5e8bae20b5", "messageHeadline": "remove gold_alias from codebase", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 35}}, {"node": {"oid": "9e5648b87d10386fd0b01ac905c7aaafdeb35d5b", "messageHeadline": "Merge branch 'big-refactor' into fixup-logging", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 75, "deletions": 69}}, {"node": {"oid": "ad8eee89a917d60032c5484c04cbc32284f6670a", "messageHeadline": "cleanup hf tqdm", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 4}}, {"node": {"oid": "c7b3f53873c6c59d20eb50feaea1ea86b18d0205", "messageHeadline": "Merge pull request #957 from EleutherAI/Refactor]-Patch-for-Generate-\u2026", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "73f3029c1eaaaf67481b8650c5edf12baab3a0d8", "messageHeadline": "precommit format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 18, "deletions": 12}}, {"node": {"oid": "f701ba7ddc00db78e120cbcd2daf64a445e08259", "messageHeadline": "eval_logger is not imported from logger.py anymore", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 14, "deletions": 19}}, {"node": {"oid": "7ec8248520246012a02660eb1b98d135fc1c5d36", "messageHeadline": "changed the order of import to allow the verbosity to take effect on \u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 14, "deletions": 12}}, {"node": {"oid": "ca8b00ccee1cc0b891fc4af6e8b801e6c82d0e91", "messageHeadline": "very rough way of utilzing verbosity", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 33, "deletions": 18}}, {"node": {"oid": "0d1fb8a5e805bc7e3987ce301e1b835d1c551863", "messageHeadline": "Rename greedy_until_template_yaml to generate_until_template_yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "6eac00565e3fdc2741c1048960f2e16dd308222d", "messageHeadline": "removed print", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 3}}, {"node": {"oid": "e49f27069bd2aa91751e98859029cffe3e91909d", "messageHeadline": "merged conflicts", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 76, "deletions": 69}}, {"node": {"oid": "6dfe848e23c596cc968e254180a4ebb06fe26342", "messageHeadline": "added group and task alias", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 231, "deletions": 58}}, {"node": {"oid": "d1e7a30a13a709148f78df214b33a41c89a2f9db", "messageHeadline": "fixed stderr calculation", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 9}}, {"node": {"oid": "29ba8cb16e272084a34bce21a724ccfc2c100e8e", "messageHeadline": "fixing display name", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 20, "deletions": 11}}, {"node": {"oid": "2099099be63611ef8537863ed0517ce53f54c405", "messageHeadline": "Merge branch 'big-refactor' into model-written-eval", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 145, "deletions": 93}}, {"node": {"oid": "ae74b808e43cd1ee6d88a157777f27eacd6b12dc", "messageHeadline": "Merge pull request #953 from EleutherAI/mmlu-no-train", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "c4a3bbb2886ace82da2879f85028fa0c63828550", "messageHeadline": "Merge pull request #955 from EleutherAI/haileyschoelkopf-patch-2", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "575dd0de07cd2bb4f3407360891435e58094ba54", "messageHeadline": "Merge pull request #956 from EleutherAI/haileyschoelkopf-patch-3", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 10, "deletions": 0}}, {"node": {"oid": "aff4742024dbbd9edfb178ea6430ff7f2dc3d77f", "messageHeadline": "Merge pull request #934 from AndyWolfZwei/andy/big-refactor-bugfixed", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 15, "deletions": 7}}, {"node": {"oid": "4ac7f064d186bb8dfc72350a1a6973f20b0ed2e5", "messageHeadline": "remove `--huggingface_login`", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 15}}, {"node": {"oid": "56a2c8e3b72fbaa298bf46c8f17f1cdefb2a6c9b", "messageHeadline": "describe local dataset usage in docs", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 10, "deletions": 0}}, {"node": {"oid": "d50a2ad61d7d7da0694585ec50b9214fcd8e670e", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "06faed0cde9c9e2c48ab1531badbf722c8404c22", "messageHeadline": "Update CODEOWNERS", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "70b09fd9a148a05c865e429adc58d20a441837c8", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 7, "deletions": 6}}, {"node": {"oid": "2efac405e4db9271671b8ad9e78f5e29047fab86", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 16}}, {"node": {"oid": "7aad26fe2f3201eccc0a24cb5b0a05cd40d8fcc4", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 24}}, {"node": {"oid": "09a744b9ea1823a62107df87cd53e048f9a39c00", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c9b76e691135c594eab7f833e8030096221f415f", "messageHeadline": "Merge pull request #954 from EleutherAI/update-docs", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 32, "deletions": 7}}, {"node": {"oid": "7fa0761cb5514158c94ab33cb20161b50f03e3c2", "messageHeadline": "clean up documentation", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 32, "deletions": 7}}, {"node": {"oid": "f5cf38a9b730c81c39d35483de4a20c473d4ac3f", "messageHeadline": "replace with no_train mmlu dataset", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "a922db886c1fa0c0b8e8a5c40e6b8a12a44c1546", "messageHeadline": "rename template for generate_until bb tasks", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "2fda31ca0490a010131d8d676f4003f5132d6ae3", "messageHeadline": "remove nonstandard logger from evaluator.py", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 5}}, {"node": {"oid": "55407cd689fbd424ae3658bbe9ecebd2adedc5af", "messageHeadline": "fix tqdm total  for hf model", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 1}}, {"node": {"oid": "60dd33c80f61373455de89d1a54101144849ba19", "messageHeadline": "new way to display tasks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "f5bdefe84a07fb2db1fb3593d005c411eb21fcdf", "messageHeadline": "new way to display tasks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 47, "deletions": 7}}, {"node": {"oid": "a5d33ebe8df61d1e74ae1942d5456c506c70529b", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "dd7002d6c1324a79a7e97ec02211e1e3f808dbe4", "messageHeadline": "add task and group alias", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "12fe236de5871be01121a63e238aae46402f9c97", "messageHeadline": "Merge pull request #949 from EleutherAI/fix-whitespace-warning", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "888d30350ac7b20077bc7a540ee1e82a9c2b1220", "messageHeadline": "fix warning for delimiter having no whitespace", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "21cf65085860a7dcb39388c912ee6ab32a0f4c67", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 104, "deletions": 72}}, {"node": {"oid": "06e6914951c6f3e048093e4dc85041c8569fbcad", "messageHeadline": "Merge pull request #915 from EleutherAI/package-patch", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "84db6359e1079f9ba90dcd54abcff908ee481ec6", "messageHeadline": "Update pyproject.toml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a1b589df8e8a6eff383459bf01166a54d3decbe4", "messageHeadline": "Update pyproject.toml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 5}}, {"node": {"oid": "fa1f35718631060e6057b9f2cd3c82d77593c8ba", "messageHeadline": "Update pyproject.toml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "e7e3adeb3a0f047caba90fb3fbdb14e3a801ee2d", "messageHeadline": "Update pyproject.toml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 4}}, {"node": {"oid": "b3a2af333f9597ec74eab79b005c85d9695c6cb1", "messageHeadline": "Update pyproject.toml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "56b7021f7fbf86852676d8c9e14aa2440650f156", "messageHeadline": "Merge pull request #937 from EleutherAI/big-refactor_write_out", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 12, "deletions": 0}}, {"node": {"oid": "1a183d0374ca4df9d2715ad5f7cdb41a27264170", "messageHeadline": "fix: copy paste mistake", "author": {"name": "Michael Pieler", "email": "Michael.Pieler@Gmail.com", "user": {"login": "MicPie"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "1964ccd302215f59828a7b328ba78a61e577b138", "messageHeadline": "feat: add --include_path option to write out", "author": {"name": "Michael Pieler", "email": "Michael.Pieler@Gmail.com", "user": {"login": "MicPie"}}, "additions": 14, "deletions": 4}}, {"node": {"oid": "408115eaffc4eecc9584f543db573d708eef8ed6", "messageHeadline": "Merge pull request #935 from EleutherAI/fix-default-metric-call", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 55, "deletions": 20}}, {"node": {"oid": "305b546084a9ac7766565a8c7228f523c38efb90", "messageHeadline": "Force conversion to str if type exceeds expected", "author": {"name": "Zhiwei Zhuang", "email": "zhiwei.zhuang@shopee.com", "user": {"login": "AndyZwei"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "c891f4d0a88dce6bc0621992501f15e32fd36f88", "messageHeadline": "Merge branch 'big-refactor' of github.com:EleutherAI/lm-evaluation-ha\u2026", "author": {"name": "Zhiwei Zhuang", "email": "zhiwei.zhuang@shopee.com", "user": {"login": "AndyZwei"}}, "additions": 13, "deletions": 24}}, {"node": {"oid": "bff237e6badc292f8f87af11f679a822163d5b5f", "messageHeadline": "change huggingface_login from str to bool for privacy", "author": {"name": "Zhiwei Zhuang", "email": "zhiwei.zhuang@shopee.com", "user": {"login": "AndyZwei"}}, "additions": 9, "deletions": 5}}, {"node": {"oid": "1428ad575ffd701be935e3fc1833d525334091ff", "messageHeadline": "fixes to pythia benchmark, boolqseq2seq", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "d9b4ca7f93e1410e3910e254d94b797df62f0eff", "messageHeadline": "fix bug when target_delimiter is empty string", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "ff148cd862f51617724cf1f2c48dc83c47df980c", "messageHeadline": "don't call evaluate.load() every time", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "8d4d1fa9252539951e2b3b1a9d347d96484a5cd0", "messageHeadline": "fixed registered metric", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "9fbe6eef799213340d9871f23c853dcf8a154d43", "messageHeadline": "changed warning message", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "90d818daa915199d32e833975d60671ff4b5b451", "messageHeadline": "fix issue with default metrics and aggregation functions", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 50, "deletions": 12}}, {"node": {"oid": "6b4161c17bde2b063c59769b9f1378a305059044", "messageHeadline": "Changed `get_default_aggregation` to `get_aggregation`", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 4}}, {"node": {"oid": "a6179b494cb835e0dbdc4c19fa42055d652ef8fc", "messageHeadline": "Update registry.py", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 9}}, {"node": {"oid": "26bc3eab15ea4ee6caefc7c1eba6afc05d94c7d3", "messageHeadline": "Merge branch 'big-refactor' into model-written-eval", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1438, "deletions": 993}}, {"node": {"oid": "cf617ab1ee83e6ea24a33ba535fdfa8b25557267", "messageHeadline": "Merge pull request #916 from jasonkrone/big-refactor", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 13, "deletions": 24}}, {"node": {"oid": "0c5de3ac58e888be9d39e3ea45a55e6ca51a7b59", "messageHeadline": "fix :Object of type int32 is not JSON serializable", "author": {"name": "Zhiwei Zhuang", "email": "zhiwei.zhuang@shopee.com", "user": {"login": "AndyZwei"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a007bacb1941356233b0aba49dbcea9bec10429e", "messageHeadline": "fix two bugs when ran with qasper_bool and toxigen", "author": {"name": "Zhiwei Zhuang", "email": "zhiwei.zhuang@shopee.com", "user": {"login": "AndyZwei"}}, "additions": 25, "deletions": 7}}, {"node": {"oid": "ef33202653db7eb775c5ee942304cf3ca91e3f0d", "messageHeadline": "Merge pull request #931 from EleutherAI/fix-generate-until", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 43, "deletions": 46}}, {"node": {"oid": "a8d130abab6623fca6dc53b8d2af94491836d0db", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e66ba123cc9e16e50e90c6505737728e92f8db00", "messageHeadline": "don't raise error for unset higher_is_better", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "c28d100d00b8cfd07b809bafbcef0dc0d0452d83", "messageHeadline": "replace the rest of the 'greedy_until' occurrences", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 43, "deletions": 45}}, {"node": {"oid": "7d496612fd69029c6ea65bbdba815ddd0954f967", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4aafe026631468d357a3bd0b93de784134f5b6f2", "messageHeadline": "Merge pull request #929 from EleutherAI/fix-generate-until", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "785153f622e681ebfba07494afc768eafd669b7a", "messageHeadline": "fix greedy_until abstractmethod", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f77a3a271cf173cef9fbd98e15fe02c796960de5", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 822, "deletions": 873}}, {"node": {"oid": "f8342178d4da8bccc874df110850ff7b4eceb89b", "messageHeadline": "Merge pull request #905 from EleutherAI/fix-unittests", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6a49ed9b5ee7e032ce49536c5a1c43b6aa005f54", "messageHeadline": "Merge pull request #910 from EleutherAI/verbose", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 11, "deletions": 2}}, {"node": {"oid": "de71ad92b5f4c8b5e25e3f7ac521e487d6436020", "messageHeadline": "Merge branch 'big-refactor' into fix-unittests", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1375, "deletions": 916}}, {"node": {"oid": "73c80915efe8834d8a493f30832ceaa94b8357ad", "messageHeadline": "Merge pull request #923 from EleutherAI/fix_squadv2", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 37, "deletions": 13}}, {"node": {"oid": "a056eded1412675f659f2ed5b17ee651691bd52a", "messageHeadline": "Merge pull request #927 from EleutherAI/change_output_type", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 766, "deletions": 854}}, {"node": {"oid": "b3224ec0f40f2e0cfaf42c9265ba3545e7620ef9", "messageHeadline": "Merge pull request #926 from EleutherAI/patch_belebele", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a7ba3d76f26b3a581d0de0a19cb134bbd1debd0e", "messageHeadline": "changed file name", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c64bf9a98c7d2526674ff23bb59823217e2ead38", "messageHeadline": "change all mentions of `greedy_until` to `generate_until`", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 766, "deletions": 854}}, {"node": {"oid": "109ed1c7c0dc3f29e2ea206dba642e595f839ab6", "messageHeadline": "added subgroups for other mmlu variants", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2905, "deletions": 3478}}, {"node": {"oid": "a2c4139b537cdcdac48394b5b5270ad1bdc05d51", "messageHeadline": "precommit stuff", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "93a45962f73c8cb9277855d399afd6d90d0b0019", "messageHeadline": "print tasks in alphabetically", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 82, "deletions": 55}}, {"node": {"oid": "c4f0bf75a9c1dcf42f3564ca04cc0626ea850aec", "messageHeadline": "pre-commit reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 22, "deletions": 11}}, {"node": {"oid": "4ccd2ec68c4d206488c543b7ed945a57ac0531a5", "messageHeadline": "sqrt final variance calculation to get stderr", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 0}}, {"node": {"oid": "2ade969312b81d1bac5070793ecc6816c0b02604", "messageHeadline": "template yaml", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 0}}, {"node": {"oid": "04ca56711786951549aaa29e7c5036ff029ca7f6", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "11fdcb4944f791b13f45b6a6fd04c4ceef82ea8c", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 30, "deletions": 26}}, {"node": {"oid": "e97019c05b1eaf49d71028adb73eefa2226de769", "messageHeadline": "removed comments", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 3}}, {"node": {"oid": "00859825db00aee796be74292ea869978225f1e9", "messageHeadline": "added stderr reprocessing for groups", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 15, "deletions": 37}}, {"node": {"oid": "1dc8f96f56522625098cf07139acb32f151503eb", "messageHeadline": "default to weighted averaging", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 48, "deletions": 25}}, {"node": {"oid": "92f2546392e211423508bf90627a96a31567e911", "messageHeadline": "modfied to add subcategory", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 147, "deletions": 63}}, {"node": {"oid": "afda3d22fcc4f13627a12516149f490caa628e74", "messageHeadline": "Merge pull request #917 from EleutherAI/StellaAthena-patch-2", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ec581e8ef2f089686199780a0041c76eb8f0f078", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "10625bd808c7dbd26e048808ac8fce6156089af2", "messageHeadline": "Fix \"TypeError: 'tqdm' object is not subscriptable\" error that occurs", "author": {"name": "Jason Krone", "email": "jasonkrone@me.com", "user": {"login": "jasonkrone"}}, "additions": 13, "deletions": 24}}, {"node": {"oid": "17095c83c7e4130d523118b3895e206c244ff1e5", "messageHeadline": "Merge pull request #885 from ManuelFay/belebele_big_refactor", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 499, "deletions": 0}}, {"node": {"oid": "32a6362cff11118492ff6899180fcec581dd122d", "messageHeadline": "Update __init__.py", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "59204667be71c8b878a6a76c0f808924a8a19598", "messageHeadline": "Update pyproject.toml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "053e1f5f0ca4ebaab9a06bef5e67c67ebd4d189c", "messageHeadline": "Update pyproject.toml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "992f021bf6ef34953a836a51d771930dfaf73ecf", "messageHeadline": "Merge pull request #912 from AndyWolfZwei/andy/big-refactor-auto-batch", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 35, "deletions": 25}}, {"node": {"oid": "660dfb71d15ad0e4fe5ddd3f4a36373d49871da8", "messageHeadline": "check with pre-commit", "author": {"name": "Zhiwei Zhuang", "email": "zhiwei.zhuang@shopee.com", "user": {"login": "AndyZwei"}}, "additions": 4, "deletions": 7}}, {"node": {"oid": "2bd5dcb69d0eef6495005d63b7f126222bc2ffb8", "messageHeadline": "finished test code", "author": {"name": "Zhiwei Zhuang", "email": "zhiwei.zhuang@shopee.com", "user": {"login": "AndyZwei"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "1aa3bc1e3e9cf4ba7182512f3a40ab6b80decbdf", "messageHeadline": "add _batch_scheduler in greedy_until", "author": {"name": "Zhiwei Zhuang", "email": "zhiwei.zhuang@shopee.com", "user": {"login": "AndyZwei"}}, "additions": 37, "deletions": 23}}, {"node": {"oid": "0d701496f38dc16c3fa918a8241f008566577343", "messageHeadline": "fixes and adjustments", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 11, "deletions": 6}}, {"node": {"oid": "2afe7770b277f7478aa7b51022943eb967ad8f2e", "messageHeadline": "Merge pull request #908 from baberabb/big-refactor_logging", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 26, "deletions": 23}}, {"node": {"oid": "e33a7d92d68d33ed9fd6de4bcfae51742e7b68b4", "messageHeadline": "error set to DEBUG", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 17, "deletions": 17}}, {"node": {"oid": "9894597c6d39e23b7d39c3d64f509a88864ceb53", "messageHeadline": "add verbose arg", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 19, "deletions": 9}}, {"node": {"oid": "bc1b0eca0de8d02e380477523a089d0c1c88f935", "messageHeadline": "removed delimiter=\"\"", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 19, "deletions": 10}}, {"node": {"oid": "13f0d12619dead557ff53635f786d00c117310e4", "messageHeadline": "Merge pull request #907 from EleutherAI/StellaAthena-patch-2", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 0}}, {"node": {"oid": "ad7921405f2fecb44d59e5c5cfc098901b803682", "messageHeadline": "removed", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 43}}, {"node": {"oid": "0348ed975af5b6788c275445197287dd91acb221", "messageHeadline": "merged latest update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 13507, "deletions": 1059}}, {"node": {"oid": "e3400936b3f7438de8a6817510d30ab6aa778dd9", "messageHeadline": "fix indentation", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4d8094bb351682574eeb589a7f2155929e259c2a", "messageHeadline": "Improve error logging", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 25, "deletions": 22}}, {"node": {"oid": "e429b6d858e807abbc9eb6b14bf93544f0732bcf", "messageHeadline": "Added notable contributors to the citation block", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 7, "deletions": 0}}, {"node": {"oid": "09d20bfa721f0a61ef0dca0de1ea6504ecbe8f80", "messageHeadline": "fix YML error in workflow", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6769119f64c50d618fd39dae31d2da9be10f89d7", "messageHeadline": "Merge pull request #816 from EleutherAI/flan-benchmark", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 6822, "deletions": 251}}, {"node": {"oid": "7d5e511c2d56e04bf7c94e66ac56c9e160c5493d", "messageHeadline": "adjustments", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 3}}, {"node": {"oid": "2493c6ccc7605933275ed7cb34221b8d8086eb33", "messageHeadline": "changed default to 5-shot", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 4}}, {"node": {"oid": "f2c396ab044d3d5f5087757051ecb0534eebea6a", "messageHeadline": "pre-commit reformat", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 180, "deletions": 173}}, {"node": {"oid": "20a54b3af209ea191051cdb8a8abb3b6b1996048", "messageHeadline": "removed print messages, added cot extraction strings", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 4}}, {"node": {"oid": "337492adbf26ae65d16f9ad9b7efb3e0dd2d4116", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1972, "deletions": 160}}, {"node": {"oid": "24ecd3866a80eb5e70b49556445c676018c9e538", "messageHeadline": "change : to . in the prompt", "author": {"name": "ManuelFay", "email": "manuel.faysse@illuin.tech", "user": {"login": "ManuelFay"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "4824a8325ab75e2d0911d8e385fb55dd230c9ffe", "messageHeadline": "Merge pull request #883 from chrisociepa/transformation-filters", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 186, "deletions": 21}}, {"node": {"oid": "b8faaa93144f223914b028b6684c8845e563f49d", "messageHeadline": "Merge pull request #900 from EleutherAI/fix-wildcards", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "09d935eeb3e0c21c095d5ca1df5f711746196d46", "messageHeadline": "fix wildcards", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "7f2a9cbdd85041537d77ee6734b299080b78b1ab", "messageHeadline": "Merge pull request #852 from EleutherAI/bigbench", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1644, "deletions": 0}}, {"node": {"oid": "51a43a768ea527e9313fb28005c59bdd936e862c", "messageHeadline": "change back to original dataset path", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "54498fcc65167f2afb71a348a6deaf7bfd927bca", "messageHeadline": "change back to original dataset path", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "c4fa6fb555d82681df451b5442b9252c67ab63ac", "messageHeadline": "Merge pull request #899 from EleutherAI/fix-scripts", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 132, "deletions": 128}}, {"node": {"oid": "436b2697f70cef7b7cd836a2359b13918f2b6531", "messageHeadline": "also, reenable CPU tests", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 100, "deletions": 100}}, {"node": {"oid": "9b06de5498c37e76b82a75f9a775530b382848f9", "messageHeadline": "remove references to main.py", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 13, "deletions": 13}}, {"node": {"oid": "0856828fc3d8d0638570644f424156d68be119e1", "messageHeadline": "remove references to main.py", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "2e13caa64ce5a5195ceb81d7bb834df36f852807", "messageHeadline": "update commandline script", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "3b5e554ffdf51506820e73972ba4373d3348516c", "messageHeadline": "move to __main__.py", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 11, "deletions": 7}}, {"node": {"oid": "6f92c20da4b980230191bc0aea87dee07b640673", "messageHeadline": "moved main to lm_eval/ to fix import of main error", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "7e6e3c200a48fa0c586c7ad9006324b0e05267aa", "messageHeadline": "modify default yamls", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "b8d7d6c3ffbc49f0b14ee495d4a556f7174632b8", "messageHeadline": "fix indentation", "author": {"name": "USVSN Sai Prashanth", "email": "usaiprashanth2018@gmail.com", "user": {"login": "uSaiPrashanth"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "ec5846d7efbb59435225b4f46207969746f64517", "messageHeadline": "Allow Generation arguments on greedy_until reqs", "author": {"name": "USVSN Sai Prashanth", "email": "usaiprashanth2018@gmail.com", "user": {"login": "uSaiPrashanth"}}, "additions": 24, "deletions": 2}}, {"node": {"oid": "2041dc3435214effae7573caa61b57b17d957d31", "messageHeadline": "Merge branch 'big-refactor' into bigbench", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2627, "deletions": 374}}, {"node": {"oid": "67c0f73a9493282c8bda3f9b8a053bc775cd2b30", "messageHeadline": "add bigbench push_to_hub script", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "fa608798a8c5427a4d08aae03a7c8fa42ccba48f", "messageHeadline": "add bigbench push_to_hub script", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 32, "deletions": 0}}, {"node": {"oid": "8806944e8790170b1b5429af83ceeac63c961660", "messageHeadline": "push updated bigbench task yamls", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 334, "deletions": 334}}, {"node": {"oid": "1a77b4d54e84a7b9a49db2bbc7325ede487c9d00", "messageHeadline": "update generate_tasks.py and template yamls", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 9, "deletions": 7}}, {"node": {"oid": "15f4a3ef807c2d64bf88f20e96ae8e393dedb266", "messageHeadline": "Merge pull request #890 from tmabraham/big-refactor", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 8, "deletions": 8}}, {"node": {"oid": "3a7f52d033e72eb9bd34ce6aa8ff48090893d9e2", "messageHeadline": "Bring task name to `pubmedqa` in line with master branch", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "efa38caa9e810071f1186b7e128c8547ec653bd6", "messageHeadline": "Merge pull request #895 from chrisociepa/update-python-version-in-pre\u2026", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "bf275d00843369c22269a7890495bb9bb0e9f7a7", "messageHeadline": "Change Python Version to 3.8 in GitHub Actions", "author": {"name": "Chris", "email": "34248815+chrisociepa@users.noreply.github.com", "user": {"login": "chrisociepa"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "3f2dd9fbbd9f035870f6b7b224e9daae1757818c", "messageHeadline": "Change python version to 3.8 in .pre-commit-config.yaml", "author": {"name": "Chris", "email": "34248815+chrisociepa@users.noreply.github.com", "user": {"login": "chrisociepa"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "06ce7a62dece2427ede1ea34b6aad77b7ac5cbd4", "messageHeadline": "Fix formatting", "author": {"name": "Chris", "email": "chris@azurro.pl", "user": {"login": "chrisociepa"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e634f83fe931108d080936ee2b17f878fa3f1ba6", "messageHeadline": "Add PolEmo2 tasks", "author": {"name": "Chris", "email": "chris@azurro.pl", "user": {"login": "chrisociepa"}}, "additions": 104, "deletions": 0}}, {"node": {"oid": "6658f510321c97f33b0fb08e5ddaf136d12bfff2", "messageHeadline": "Update preprocess_pubmedqa.py", "author": {"name": "Tanishq Abraham", "email": "37097934+tmabraham@users.noreply.github.com", "user": {"login": "tmabraham"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "5c418e2e7d333b2c581dec88391e9d46312fb917", "messageHeadline": "Update pubmedqa.yaml", "author": {"name": "Tanishq Abraham", "email": "37097934+tmabraham@users.noreply.github.com", "user": {"login": "tmabraham"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "3d2ee4d43e25a069aec30ccbdff17861deed55e1", "messageHeadline": "Update _default_template_yaml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "6ae9e9e80c1e0049237d0179ac94fd57bc49029b", "messageHeadline": "remove comments", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 12}}, {"node": {"oid": "b2d16321724b572967bc7f1cbe64ae0de4eaa0dd", "messageHeadline": "update loading prompts", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 66, "deletions": 52}}, {"node": {"oid": "307118731f93443f14f490d058db7945492599b3", "messageHeadline": "modify to conform to pre-commit", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 24, "deletions": 15}}, {"node": {"oid": "3f090027112a6e9ea45d7ee2deb57edac5f03ea2", "messageHeadline": "moved files", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 48, "deletions": 16}}, {"node": {"oid": "47a7d41c04c359a160d9c0a50f57e1d446e978db", "messageHeadline": "match beleble paper prompts more closely", "author": {"name": "ManuelFay", "email": "manuel.faysse@illuin.tech", "user": {"login": "ManuelFay"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "2d6bc236290c529025e811da655d5052a5d39348", "messageHeadline": "remove description", "author": {"name": "ManuelFay", "email": "manuel.faysse@illuin.tech", "user": {"login": "ManuelFay"}}, "additions": 1, "deletions": 126}}, {"node": {"oid": "a5e93901646eb63e8d4e388d7daaeca03e49d7b5", "messageHeadline": "merged latest", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1734, "deletions": 30}}, {"node": {"oid": "c5ebdd0fac3ae074e7379e3d99d9744d8db53bcc", "messageHeadline": "add belebele", "author": {"name": "ManuelFay", "email": "manuel.faysse@illuin.tech", "user": {"login": "ManuelFay"}}, "additions": 622, "deletions": 0}}, {"node": {"oid": "694af7d656989d6a22623e5245db655569b40390", "messageHeadline": "remove templerature as do_sample is False", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "0dc556dcf6285fbb3f49fb726bf3209431d93d05", "messageHeadline": "add function to add both task and benchmark", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 46, "deletions": 144}}, {"node": {"oid": "8310e67e7de114ef2c41e249cdebda9f2779c1cb", "messageHeadline": "Fix formatting", "author": {"name": "Chris", "email": "chris@azurro.pl", "user": {"login": "chrisociepa"}}, "additions": 13, "deletions": 5}}, {"node": {"oid": "ad4ab52aa4f0952fb4958518d3942b32eb914d28", "messageHeadline": "Fix formatting", "author": {"name": "Chris", "email": "chris@azurro.pl", "user": {"login": "chrisociepa"}}, "additions": 4, "deletions": 3}}, {"node": {"oid": "34d1ac6bc35ca94808bac938d33e96b8577ac2dc", "messageHeadline": "Use 'is' for boolean comparison", "author": {"name": "Chris", "email": "chris@azurro.pl", "user": {"login": "chrisociepa"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b4c72cb9808bc23bb65b3eb56764361f6075da9f", "messageHeadline": "Update new_task_guide.md", "author": {"name": "Chris", "email": "chris@azurro.pl", "user": {"login": "chrisociepa"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "90b261e5786ec7d1dcc0f3ef26431d5142dfa7bd", "messageHeadline": "Add transformation filters: lowercase, uppercase, map", "author": {"name": "Chris", "email": "chris@azurro.pl", "user": {"login": "chrisociepa"}}, "additions": 51, "deletions": 0}}, {"node": {"oid": "5b26b3b0e01f5d2bf4969ce32dd0cb0b1d7e4549", "messageHeadline": "Cast gold to the same type as result", "author": {"name": "Chris", "email": "chris@azurro.pl", "user": {"login": "chrisociepa"}}, "additions": 4, "deletions": 3}}, {"node": {"oid": "94346b7ed8a699444d5f7b9defbb424134a45d7a", "messageHeadline": "Allow forced import of metrics from the HF Evaluate library", "author": {"name": "Chris", "email": "chris@azurro.pl", "user": {"login": "chrisociepa"}}, "additions": 17, "deletions": 17}}, {"node": {"oid": "0aa37743218177935bcdcbbf8576c98a6b4810ac", "messageHeadline": "Merge pull request #880 from EleutherAI/hotfixes", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 22, "deletions": 3}}, {"node": {"oid": "d98159b2dfe731736b1aea759282ff32f51eb0e7", "messageHeadline": "trigger precommit", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "e4aa240e1ef783ea0e8e0ec28ae0977329ded78f", "messageHeadline": "trigger precommit", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "43c91b5ca84df830176f2f63455a500f155c8c8d", "messageHeadline": "hotfix bugs", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 18, "deletions": 4}}, {"node": {"oid": "265a7cded3a1ee8247109616df33c956693bcc86", "messageHeadline": "Merge pull request #878 from EleutherAI/lintangsutawika-patch-5", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 56, "deletions": 0}}, {"node": {"oid": "bdddfec2b071e8dbd6e748148a998b9835b0a53a", "messageHeadline": "Merge pull request #864 from EleutherAI/add-fewshot-config", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 973, "deletions": 14}}, {"node": {"oid": "f88ffeee12e8971763fbccbbb4d4792e1bb77da2", "messageHeadline": "Merge branch 'big-refactor' into add-fewshot-config", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1256, "deletions": 203}}, {"node": {"oid": "0f6cd35806147691a86872dc16cf27c210fcd286", "messageHeadline": "Merge pull request #865 from EleutherAI/csatqa-refactor", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 104, "deletions": 0}}, {"node": {"oid": "c2848879ad07f1a8e285c6174a310e4e8cc153df", "messageHeadline": "Merge pull request #870 from EleutherAI/lintangsutawika-patch-4", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 40, "deletions": 11}}, {"node": {"oid": "d823336526ce8997c5aa92bf63dab4b541359413", "messageHeadline": "Merge pull request #861 from baberabb/big-refactor_math_sympy", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 429, "deletions": 6}}, {"node": {"oid": "2906a565108f642e4d5eea51269a62f12d8f7f6a", "messageHeadline": "Add [math] optional dependency, make antlr req more stringent", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "c8f56b1788ab5f08e7bf9da9cadb18d6711e3161", "messageHeadline": "Note [math] extra", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c640376551f198e598d930333517010322b87527", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 78, "deletions": 78}}, {"node": {"oid": "ea8b5beb04a705f63401969bcfd3ffa02f216645", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 17, "deletions": 12}}, {"node": {"oid": "f0d8b559a9fcca7f115bea8e5ed13996749aeba8", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 24, "deletions": 24}}, {"node": {"oid": "76227f0dbec309c7124de09a9b7ce4009f8bb8f1", "messageHeadline": "add codexglue", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 126, "deletions": 104}}, {"node": {"oid": "7418eddd9dca4bce8b4e553d6528ea0572777110", "messageHeadline": "Create README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 56, "deletions": 0}}, {"node": {"oid": "5bd274a8ac891482e7eb2832f474dd5c091c5b95", "messageHeadline": "Merge pull request #877 from chrisociepa/fix-hf-model-generate", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "877e9a6196be14148ab6195f00cb4094b901a3ec", "messageHeadline": "Fix positional arguments in HF model generate", "author": {"name": "Chris", "email": "34248815+chrisociepa@users.noreply.github.com", "user": {"login": "chrisociepa"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ac50adb555ea6ef8ba1e94586bf0682e567d4e97", "messageHeadline": "merged with latest big-refactor", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1208, "deletions": 433}}, {"node": {"oid": "a3252ed7f668acc976c8ce1f9d0e42cbc55d7483", "messageHeadline": "Merge pull request #862 from EleutherAI/bump-deps", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 13, "deletions": 14}}, {"node": {"oid": "fa2ae334dd3541e85269f940f9b253ef12f42c87", "messageHeadline": "Merge branch 'big-refactor' into bump-deps", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 684, "deletions": 185}}, {"node": {"oid": "54a53d6fbf574442e23ce3446c0479a3cd89b6d8", "messageHeadline": "Merge pull request #785 from EleutherAI/squadv2", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 166, "deletions": 69}}, {"node": {"oid": "d03974ad39a74ea375a074ce4cf65e0cba03c6f2", "messageHeadline": "Update main.py", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "8043cb1915966b64cdac78c2120cd2bd972cdea5", "messageHeadline": "Merge pull request #871 from EleutherAI/deactivate-ci-tests", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 99, "deletions": 99}}, {"node": {"oid": "a019eeae2ab337182f6959b2563d7a129336966c", "messageHeadline": "temporarily deactivate some Actions", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 99, "deletions": 99}}, {"node": {"oid": "d7ddf525cb15781f374d967870f05c18faef07d2", "messageHeadline": "Merge pull request #842 from Hojjat-Mokhtarabadi/big-refactor", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "f921fc19bf5029ba36732b9534ca8acf4b17fdc1", "messageHeadline": "fixed task and benchmark adding process", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 6}}, {"node": {"oid": "4a967f26511f150c3488d854f57b63b8e7e309dd", "messageHeadline": "Update mgsm_cot_native_zh.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0f2ff6afd6dcc6b15232e17b2917dcaba7f148e3", "messageHeadline": "Update mgsm_cot_native_th.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ae473f79e6a65f3b8d1bf2183c26f063cd5183fe", "messageHeadline": "Update mgsm_cot_native_te.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "d52d78aadc6ae23c9da35f79abecac9e354d7fae", "messageHeadline": "Update mgsm_cot_native_sw.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "5d5d53a7a8a597f51cef2952658b412b17196701", "messageHeadline": "Update mgsm_cot_native_ru.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "7700d78ac55323d0cb50df8889aa70fe374f2293", "messageHeadline": "Update mgsm_cot_native_ja.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f7cff1ec233be789eeb7366d3db606a1cf02bf1d", "messageHeadline": "Update mgsm_cot_native_fr.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "58bef0be2c5ed6ec827e3a9008eb8a6a99e6b4a2", "messageHeadline": "Update mgsm_cot_native_es.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b2eea396aacc86a7eb5a7c44cd6a162b4e6ac6e3", "messageHeadline": "Update mgsm_cot_native_de.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "17190841c147e9667d6a00596570aefeb02ab8be", "messageHeadline": "Update mgsm_cot_native_en.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "feedaf085ebbb451d0e48e661e4811a3e5bd1429", "messageHeadline": "Update mgsm_cot_native_bn.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "fa5ca8c361d37a5c67593cca9ebb879200fdb7d5", "messageHeadline": "Create cot_yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 29, "deletions": 0}}, {"node": {"oid": "a20094528dc4eb73454801a21afb074f1124f6d6", "messageHeadline": "format", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 20, "deletions": 12}}, {"node": {"oid": "4578ca146a4ff6cc6a8b71b244125741c0bfb308", "messageHeadline": "added additional squad tasks", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 13, "deletions": 0}}, {"node": {"oid": "1ed76cfa7213ebf7bd73a05baa2be089b7023581", "messageHeadline": "moved benchmarks back to tasks/", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 88, "deletions": 164}}, {"node": {"oid": "10d8ed6492c74b71b12734e3cd646347ee80acbd", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 410, "deletions": 6}}, {"node": {"oid": "a89b132dc4e096165ca9591e13bd4c63b48a9f82", "messageHeadline": "Update write_out.py", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "0f3f1267be0f80290d5fd7ef43237fc3c1ed734b", "messageHeadline": "Update write_out.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8daee3425f0a9e84917e32d0bfd92fd0759a34d6", "messageHeadline": "Update __init__.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0e75f58c98aa219cef860bb744c49b550a78d2b2", "messageHeadline": "raise exception if dependencies not met", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "78a33749ce9882ba4403efd66789e8669a16fd9f", "messageHeadline": "update readme", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "30cedf0cfe092bce64f49ec865842566feb3c3c3", "messageHeadline": "type annotations 3.8 compatible", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "f4c1b9d6bfb698473acb34dadc546f69697736ef", "messageHeadline": "sympy v. particular with dependency requirements", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c6dc1ca4068021c68052b5a6c694de10efde1562", "messageHeadline": "changes to promptsource output_type reverted", "author": {"name": "Hojjat Mokhtarabadi", "email": "hmokhtarabadi.1999@gmail.com", "user": {"login": "Hojjat-Mokhtarabadi"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "068155782c473e4017e69c42d687fa629a1bba6c", "messageHeadline": "added `sympy`, `antlr4` dependencies for `minerva_math`", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "e2ed49f81feba446d4f56eb568b494767f803b83", "messageHeadline": "Merge remote-tracking branch 'origin/big-refactor' into big-refactor_\u2026", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 475, "deletions": 33}}, {"node": {"oid": "91a37c90fe7e89b071fbe4e9f338d368ac8b72e8", "messageHeadline": "Merge pull request #840 from EleutherAI/qasper", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 410, "deletions": 6}}, {"node": {"oid": "3263c5729d11e45ddcd03e263bfe57637266dfed", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 2912, "deletions": 801}}, {"node": {"oid": "11be7c939f2d0d55df689d1d6197544820ae7529", "messageHeadline": "format", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 15, "deletions": 11}}, {"node": {"oid": "d1c189eaebad807e0d8a7b6ab1d8bbe15d55126c", "messageHeadline": "remove scrolls from this PR", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 361}}, {"node": {"oid": "1f3510673ce7c51a2ef1caed8edcfa8ccb00abd4", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 322, "deletions": 161}}, {"node": {"oid": "33d52483e4757330408e3c398f0e1e188ca3afb6", "messageHeadline": "Merge pull request #860 from EleutherAI/cleanup-deps", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 8}}, {"node": {"oid": "2d5d94da064a241a6c3813276b5faabaa73126fb", "messageHeadline": "add ceval", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 470, "deletions": 0}}, {"node": {"oid": "020c41156dce45f27017cc9cb6b0226e1373b055", "messageHeadline": "port csatqa to yaml", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 104, "deletions": 0}}, {"node": {"oid": "4ff8260dfbbe77d887ceeb11fde45bb9b141022c", "messageHeadline": "allow for trailing comma in model_args", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "7c2687cb31abb2ac1ea7f81673eb8e8fba4d30a9", "messageHeadline": "update typehints in decontam tool", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 9, "deletions": 9}}, {"node": {"oid": "b29ef52157d2d6a92fb69b86dfd02be3cf895a65", "messageHeadline": "update cmmlu readme", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 21, "deletions": 146}}, {"node": {"oid": "09ce0c02c61581f38345c63b5e9008385b4c5af9", "messageHeadline": "fix crows-pairs readme", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "b089e5f8a03d33423b02eb6565b04c2d2c4813ea", "messageHeadline": "fix storycloze task name", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "13c6f5e74ed51ca301b4dac82de0422779751439", "messageHeadline": "add draft cmmlu port", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 590, "deletions": 1}}, {"node": {"oid": "9d0df41bd02d8d2252300646de86ffd232621c69", "messageHeadline": "add fewshot_config field", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 3}}, {"node": {"oid": "624fb20752c9d51de9762715377d6c7d3b8b9281", "messageHeadline": "reenable bootstrapping", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "e4bdffd024ad38577a60c9acc62d0ba20a2314ab", "messageHeadline": "add first_n sampler", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 30, "deletions": 3}}, {"node": {"oid": "4c139701cc47222a0120952296fde62a469b9b47", "messageHeadline": "Merge branch 'big-refactor' into bump-deps", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 60, "deletions": 19}}, {"node": {"oid": "cc547c7b1306512d5201f4f8e083ea85eeebdd81", "messageHeadline": "Merge pull request #863 from baberabb/big-refactor_types", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 4}}, {"node": {"oid": "894370df6d7203445c5a952acca0de68d3258325", "messageHeadline": "make typing 3.8 compatible", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 6, "deletions": 4}}, {"node": {"oid": "ce1a8cd080fb0e573f8da2bbb7841e6fdba082df", "messageHeadline": "Merge pull request #856 from chrisociepa/bugfix-write_out", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "a554e41be7791a3545d9d4b746177ecb6d976ba9", "messageHeadline": "refactored hendrycks_math to minerva_math", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 45, "deletions": 38}}, {"node": {"oid": "fa8f2381024a01be4ba5e85baad10e56f4983de6", "messageHeadline": "Merge pull request #859 from baberabb/nqopen_baber_fi", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 51, "deletions": 14}}, {"node": {"oid": "5f205bd557a38c49bcf5c6b6476677ce89afd610", "messageHeadline": "list(gold) if `multiple_target`", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "5794ec3cc41e423a33a1ee31d27c18207ab95510", "messageHeadline": "update python ver. in mypy.ini?", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f6c81476cee118f11e572d610a96db1cd62a5218", "messageHeadline": "add python 3.8 to supported versions in pyproject.toml and to CPU uni\u2026", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "1a34d08498737312a0a8211a32b020b097bb1813", "messageHeadline": "add hendryck_math tasks; bugfixes", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 418, "deletions": 6}}, {"node": {"oid": "da43b3a239854e6b86250ab0fb28c25f276246b4", "messageHeadline": "remove stale deps + fix [sentencepiece] dep", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 4}}, {"node": {"oid": "b1dd5853022bbd668cdddacf998fcd384417c107", "messageHeadline": "fix typo", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "987476389a34d9fbbc6631de8b7fd9c3cfa2e756", "messageHeadline": "remove omegaconf usage", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "57b20eef06e2790d071eb0a58c3af56c68894bd3", "messageHeadline": "Merge pull request #857 from chrisociepa/python_3_8_compatible", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 6}}, {"node": {"oid": "5d4ac13453667e9520ab70880acf0124e1f06aa1", "messageHeadline": "Merge pull request #854 from ethanhs/pyproject_toml_v2", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 85, "deletions": 74}}, {"node": {"oid": "38c9689ae38cfa503355693f211e0a6ddf88f83e", "messageHeadline": "Use Dict to make the code python 3.8 compatible", "author": {"name": "Chris", "email": "chris@azurro.pl", "user": {"login": "chrisociepa"}}, "additions": 7, "deletions": 6}}, {"node": {"oid": "0ed5527a376009eb80a5eb249a347d8d01f0d6ec", "messageHeadline": "Fix \"local variable 'docs' referenced before assignment\" error in wri\u2026", "author": {"name": "Chris", "email": "chris@azurro.pl", "user": {"login": "chrisociepa"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "680c568689389b302da7f32b8cbaeb4996ef4984", "messageHeadline": "change prompt formatting", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "bf038375bcd845c6d366db2d3de42e15ebab4b41", "messageHeadline": "Refactor whitespace check and add handling for multiple_targets", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 18, "deletions": 14}}, {"node": {"oid": "a07d05f7ca188199cf5e6e5e827beb2e3780fa4c", "messageHeadline": "Merge remote-tracking branch 'origin/big-refactor' into nqopen_baber", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2912, "deletions": 744}}, {"node": {"oid": "6ba2a2b03a8487525f26480d4c4d47f5abfc7aab", "messageHeadline": "Merge pull request #850 from EleutherAI/better-benchmark", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 140, "deletions": 52}}, {"node": {"oid": "0594fe2b528f7b23a1c60721b7a4eb311257848d", "messageHeadline": "Merge branch 'big-refactor' into pyproject_toml_v2", "author": {"name": "Ethan Smith", "email": "ethan@ethanhs.me", "user": {"login": "ethanhs"}}, "additions": 2671, "deletions": 676}}, {"node": {"oid": "40c9f9cb71926f5802943d007c93e4a52b0a5240", "messageHeadline": "Switch to pyproject.toml based project metadata", "author": {"name": "Ethan Smith", "email": "ethan@ethanhs.me", "user": {"login": "ethanhs"}}, "additions": 88, "deletions": 75}}, {"node": {"oid": "b65b9ca3a09244f00e182f9a9e8ea723c1435b9b", "messageHeadline": "Merge pull request #853 from baberabb/big-refactor_mps", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 29, "deletions": 6}}, {"node": {"oid": "bd81b8c0f54e79bdf82887a7083d056dd399d95f", "messageHeadline": "Update device list and dtype detection for MPS", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 29, "deletions": 6}}, {"node": {"oid": "50f4428bf0ad0da0135cbe73bb9d1c73d8f67f00", "messageHeadline": "merge update", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 284, "deletions": 89}}, {"node": {"oid": "020abc86b29f2fb772602ed9a2c018c667301fc3", "messageHeadline": "add qasper in scrolls", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 63, "deletions": 0}}, {"node": {"oid": "b155946ec376c76efef657e9255f979b6ffbedc5", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 217, "deletions": 176}}, {"node": {"oid": "892f40a98e1bdc6ee7221b7900d78abd837e3186", "messageHeadline": "add comments", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 88, "deletions": 85}}, {"node": {"oid": "605787a9658e1acb5ff3f01900723c5bd360ca7b", "messageHeadline": "add bigbench", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1607, "deletions": 0}}, {"node": {"oid": "d2804132e2d73855ca139b3ade44bc7819664c19", "messageHeadline": "aggregates across tasks within the same group in addition to average \u2026", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 53, "deletions": 25}}, {"node": {"oid": "d4f62844742262ac1b4470d433b9915b32ff88d1", "messageHeadline": "better presentation", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 18, "deletions": 42}}, {"node": {"oid": "e1e05b19ed6eab82fea943e5090ccc6b22377ec3", "messageHeadline": "cleaner process", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 0}}, {"node": {"oid": "91d8484012b7ef1bbfd55ab9907717c42e0d444d", "messageHeadline": "cleaner process", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 80, "deletions": 61}}, {"node": {"oid": "6414d55b9d8f04c1c78e2d6f3f115d58f1de28b3", "messageHeadline": "fixed warning", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "993baaa6e6ff26a99c231a5c3aa6bf675befb758", "messageHeadline": "remove comments", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 17}}, {"node": {"oid": "973d563ae65b4c472f7a04af2c4e5ad6bd7ce133", "messageHeadline": "add TODO", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b11f7f3750a5efd3433ffe19c5d435a5cd77d3a4", "messageHeadline": "add TODO", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 1}}, {"node": {"oid": "d29c0940c0ca6bbe0df5562a55ee302c952bf8db", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "17ec49064a2eeba77c438b345fbc7dadabc436fc", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "aed4e6943dd45c460bbb984a1f9ec9c722d66934", "messageHeadline": "presented with task-group relation", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 88, "deletions": 27}}, {"node": {"oid": "6bba33dbf20a3e2cc0444c5bfffa2c1a93d29c27", "messageHeadline": "changed term to groups", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "4e4f0de222e9dfc05d52bf92557e3d6ba43c65d9", "messageHeadline": "changed term to groups", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 5}}, {"node": {"oid": "f7d9850072ae032d0eb49bc576ef4584a31d01a8", "messageHeadline": "allow nested groups", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 15, "deletions": 4}}, {"node": {"oid": "b8d1cef915ff82ec5d5c27a39c6c2b58a9171fac", "messageHeadline": "Merge pull request #845 from EleutherAI/fix-mgsm", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 116, "deletions": 175}}, {"node": {"oid": "5ebe28ebbc88fc443a928d10a08d082cf68ae0c3", "messageHeadline": "fix typo", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "f052d0597f51b6b61872f0c1498d7c69c22e2881", "messageHeadline": "Merge pull request #799 from EleutherAI/haileyschoelkopf-patch-2", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 170, "deletions": 68}}, {"node": {"oid": "edd7dde333fd1b7fe81500bd7ddc746d14a8afca", "messageHeadline": "Merge branch 'big-refactor' into haileyschoelkopf-patch-2", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2638, "deletions": 623}}, {"node": {"oid": "fb436108c1f35c68bdbec8a37e78a44365a91c86", "messageHeadline": "make Task._config a public property", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 72, "deletions": 67}}, {"node": {"oid": "0b99c7d2d65664567afb9d3c7f605144513c7f9e", "messageHeadline": "update evaluator docstring", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "095d12f744a3329055f5a82369ea79f8f298025e", "messageHeadline": "fill out interface.md", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 95, "deletions": 9}}, {"node": {"oid": "365fcda9b85bbb6e0572d91976b8daf409164500", "messageHeadline": "Merge pull request #810 from ethanhs/pyproject-toml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "e6939479c17f5bc3901f653cbeaf8de66bbe96c0", "messageHeadline": "english cot, and added readable text in utils.py", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 116, "deletions": 175}}, {"node": {"oid": "c87703f314d0b411344fafa26653cca482c46316", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 21, "deletions": 26}}, {"node": {"oid": "7a46245317ecf99e8085943e0908862b377e97f9", "messageHeadline": "undefined type and output_type when using promptsource fixed", "author": {"name": "Hojjat Mokhtarabadi", "email": "hojjatmokhtari.1999@gmail.com", "user": {"login": "Hojjat-Mokhtarabadi"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "0e1538e9b0e3e68d802b78a95a842d0ea29671cc", "messageHeadline": "Fix OpenaiChatCompletionsLM and remove is_async flag", "author": {"name": "DaveOkpare", "email": "okparedave@gmail.com", "user": {"login": "DaveOkpare"}}, "additions": 81, "deletions": 22}}, {"node": {"oid": "6355d06f22a3d30af924b0c77e303937cb20e8ea", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 573, "deletions": 1}}, {"node": {"oid": "a916afd7b5c94b397e6b5c8380e65dfe7a666e4b", "messageHeadline": "Merge pull request #830 from EleutherAI/add-imports", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "d1f502709b9720f11222d09c04b7c86758115e75", "messageHeadline": "add scrolls", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 302, "deletions": 0}}, {"node": {"oid": "4b711aa661d2cd2e15f8495e4133b971dd195cc2", "messageHeadline": "normalize path", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "d3d1b7f1c46a6d173dd484a1dda7bf9c4703f894", "messageHeadline": "fixed import", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 6}}, {"node": {"oid": "e0e0746d22f4c7b939393ad9954a831f2b1dd2d0", "messageHeadline": "checked mutual and qasper", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "46bd5e46de42e7f758974f81541e7bf449f5d0f7", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "2d5306648370be0c96aa41ada03e486dcd6fa3ac", "messageHeadline": "add qasper", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 199, "deletions": 0}}, {"node": {"oid": "af4b012e82414a65885ff22e17219489d86cf397", "messageHeadline": "add mutual", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 110, "deletions": 0}}, {"node": {"oid": "57c29709def352afa0ecc599dfd8756010c19dbc", "messageHeadline": "adjust", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 66}}, {"node": {"oid": "5ebc7c85e7c38a534139a05c271e692427fea454", "messageHeadline": "Added support for ChatCompletions", "author": {"name": "David Okpare", "email": "okparedave@gmail.com", "user": {"login": "DaveOkpare"}}, "additions": 109, "deletions": 0}}, {"node": {"oid": "3839125a405e3f0aaeb828c68074e2990b68a195", "messageHeadline": "Merge pull request #784 from EleutherAI/mgsm", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 572, "deletions": 1}}, {"node": {"oid": "78522c9448fd39ae42d10d79fcf7ca89e266633b", "messageHeadline": "edit readme", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "681b2e2c76b57f546c50b522f5613f8dec1755be", "messageHeadline": "change name", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "fd2790894642292f7b480e10a66e16a33e22335c", "messageHeadline": "tidy up", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 38, "deletions": 18}}, {"node": {"oid": "2387f39daf0ac28f190a911c6bdc1bb629a1cb09", "messageHeadline": "Merge branch 'big-refactor' into flan-benchmark", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 913, "deletions": 531}}, {"node": {"oid": "784fe0375687b0e00fb78991102905db461370ad", "messageHeadline": "Merge pull request #838 from EleutherAI/lintangsutawika-patch-2", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "00c4ffff9ada116ed58b46c137a204fbcad62e36", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e9d463800821722a28173dfa1d1c7c212b653012", "messageHeadline": "Merge pull request #831 from baberabb/big-refactor_python_final", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 161, "deletions": 280}}, {"node": {"oid": "dfc47bd515d6875b756b92931c02661c698e1f45", "messageHeadline": "changed atol", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 5, "deletions": 3}}, {"node": {"oid": "e0311cd5c8e603d7ac7835c29dce1aa310d9bf04", "messageHeadline": "Merge remote-tracking branch 'origin/big-refactor' into big-refactor_\u2026", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 627, "deletions": 237}}, {"node": {"oid": "96c60cf6ec2ff91310a9000002afd6574201dc43", "messageHeadline": "artifact loglikelihood results", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 15, "deletions": 1}}, {"node": {"oid": "f1e62d3650542c13b3e1fd9fbee4a4c0757c4f4d", "messageHeadline": "added torch deterministic", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "f86d687435c5a18c8bef9cc518fa3d7dddd69ee2", "messageHeadline": "Merge pull request #775 from EleutherAI/wmt", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 439, "deletions": 96}}, {"node": {"oid": "01ad787d3b9f26cc09e4abd01542fb2d8eb7c3c1", "messageHeadline": "format for pre-commit", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 4}}, {"node": {"oid": "c417292d31b957487ec853e7b71946b51633a75b", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 16, "deletions": 3}}, {"node": {"oid": "1a02d9dfaaf29155842b7b80b100d007bb5b177d", "messageHeadline": "Merge pull request #833 from EleutherAI/fix-ppl", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 16, "deletions": 3}}, {"node": {"oid": "9bca36a9b6334bb8891aa97b1709720199d52595", "messageHeadline": "pre-commit", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "a7286607ee29e2002382140a68dd3f4dba405b60", "messageHeadline": "change words/bytes calc for wikitext", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 13, "deletions": 0}}, {"node": {"oid": "7601d82827abfeb60288e1c561e8ac518a5b4d0d", "messageHeadline": "edit to fix cot filter", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "3b4fa26e0c57019589c273eeb4e0eab69ff300ba", "messageHeadline": "Merge branch 'big-refactor' into wmt", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1308, "deletions": 213}}, {"node": {"oid": "8f448eed3f9a0cf107ac36fc81d66007cf0a7d8b", "messageHeadline": "Merge pull request #809 from ethanhs/mypy", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 164, "deletions": 139}}, {"node": {"oid": "4721379e2bc7def98d742b83b94414af36b151e9", "messageHeadline": "Merge branch 'big-refactor' into mypy", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1302, "deletions": 124}}, {"node": {"oid": "6cc1ae7c0b0cb535af593fe1fad5d84912d0c59d", "messageHeadline": "consolidated tasks testing", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 129, "deletions": 271}}, {"node": {"oid": "cc7828dd738c83e5505818f5a1aa8a52a6472ed4", "messageHeadline": "Merge pull request #812 from fattorib/bump-accelerate", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 16, "deletions": 7}}, {"node": {"oid": "f96f330f6698e7062815811c6c01fc041a189380", "messageHeadline": "placate precommit", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 5}}, {"node": {"oid": "fdb601a547e45eef0ca10b5f2a8487eb5f5a0a89", "messageHeadline": "add top-level import", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "8c39bfc7038511001f26fa14261716ce988d8591", "messageHeadline": "add codexglue", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 341, "deletions": 0}}, {"node": {"oid": "ec03783fde292bd79c8f4df3c80e280756ee3994", "messageHeadline": "update for held-in tasks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 64, "deletions": 36}}, {"node": {"oid": "4f5b72bc21249c267e19667e057b48b6f34f82bc", "messageHeadline": "fixed template", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 110, "deletions": 110}}, {"node": {"oid": "4a7528679fcbc71e5ce51c2686908ed728e1a17a", "messageHeadline": "update flan mmlu", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 700, "deletions": 1}}, {"node": {"oid": "03be40e28b3ab9c0ea14ae454b883c08f5957ca0", "messageHeadline": "udpate", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 45, "deletions": 870}}, {"node": {"oid": "e795efcfacb1f16e0ba79dee221ff18e44906f52", "messageHeadline": "updates", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 75, "deletions": 38}}, {"node": {"oid": "c8b76a3dad9fb9c3c5d89db695b2b758e223f31c", "messageHeadline": "edit group name", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "96d9e38c5e302b5c4dcddae2542a9ec10333f057", "messageHeadline": "update benchmarks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 34, "deletions": 2}}, {"node": {"oid": "86e785899d703bd2edc37df2e7e872220f3fb2cf", "messageHeadline": "modified changes to fix loglikelihood prediction for seq2seq", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 7}}, {"node": {"oid": "0d195e90f2796069f6c1d92e7f19854a66b11245", "messageHeadline": "add flan_zeroshot", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 151, "deletions": 0}}, {"node": {"oid": "3531d9c162b58a92aa441111750b516d52b40a19", "messageHeadline": "add flan_fewshot", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 151, "deletions": 0}}, {"node": {"oid": "c06b0d6efb48abe7f171d283ffad765ee204f72a", "messageHeadline": "add flan_cot_zeroshot", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 157, "deletions": 0}}, {"node": {"oid": "13940f1ecd39377a25f86973c20e47c502271d31", "messageHeadline": "add bbh flan_cot_fewshot", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 157, "deletions": 0}}, {"node": {"oid": "06d3406e55d4fdc4d8f92061ff8143257aad87ed", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 224, "deletions": 432}}, {"node": {"oid": "2e747c5b233a962c83dd060b40218e13ec58ac70", "messageHeadline": "Merge remote-tracking branch 'origin/big-refactor' into big-refactor_\u2026", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1179, "deletions": 94}}, {"node": {"oid": "f23ae748125513cb6d23254b14c33fee23a567d6", "messageHeadline": "add mmlu variants", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5082, "deletions": 6}}, {"node": {"oid": "191458b88f370938550c30cc9216d3ea9fcf5f9f", "messageHeadline": "Merge branch 'mmlu-refactorbranch' of https://github.com/EleutherAI/l\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 128, "deletions": 1}}, {"node": {"oid": "a81ef1a798691b2e02129c7496f4c5883c951611", "messageHeadline": "edit stop token", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 22, "deletions": 3}}, {"node": {"oid": "d01cc479194551d0e4eb10c05a8f38eb44dbb550", "messageHeadline": "modifed error message", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "3c842795640f32cb6e01b58103d0e108f6a43d59", "messageHeadline": "modifed error message", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2e4b3c99c22d8ac1017db3fb872a839d2b853e2c", "messageHeadline": "modifed error message", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "6f42ab7c111946892de4d0402aa4f83895f3edfd", "messageHeadline": "detect if a task was not found", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 3}}, {"node": {"oid": "c3764d2c31bfa0e45c8f57fb200c7b4d2642e8c8", "messageHeadline": "update to generate bbh configs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 10}}, {"node": {"oid": "a346e6a06272093ff4997866e8fc6323422c1c32", "messageHeadline": "Merge pull request #807 from EleutherAI/wsc273", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 112, "deletions": 1}}, {"node": {"oid": "b6f773eb58ee8c6b8bf6f42f01d1ca0da817c8a7", "messageHeadline": "Merge pull request #822 from baberabb/big-refactor_toxic", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 13}}, {"node": {"oid": "788b6f9432240f5d5c260a6712b4b720ae6a23dc", "messageHeadline": "moved API inside function; API expects json", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 12, "deletions": 13}}, {"node": {"oid": "525f1d1593b754cca4c9282e4f484432708db035", "messageHeadline": "added more flan subtasks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 50, "deletions": 43}}, {"node": {"oid": "24754ee41de38d24b2a86228c6e10db03a9fbf48", "messageHeadline": "add bbh", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 199, "deletions": 0}}, {"node": {"oid": "88e1cdb0e162512db949b7f0f05e3771f98d17f1", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 466, "deletions": 9}}, {"node": {"oid": "6342636ecd28ab95de951b95cd1bb7eeaeed6fd8", "messageHeadline": "Merge pull request #819 from EleutherAI/811-fix-indexerror", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 22, "deletions": 4}}, {"node": {"oid": "ee59182d7fb59103f13a55500b3e1d439504e796", "messageHeadline": "formatting", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "c605a71009239288265d579f46a80146363f26dc", "messageHeadline": "add logging information", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 1}}, {"node": {"oid": "618c5936dd3a8661bbf527f7043a1315bd3c090f", "messageHeadline": "make visible available tasks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 4}}, {"node": {"oid": "dc544beb87f6bec0efa7c154f15058023fe41b55", "messageHeadline": "Merge pull request #813 from EleutherAI/asdiv", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 21, "deletions": 3}}, {"node": {"oid": "ebfff08f313a78c7500e324b43c791f5c1d44d74", "messageHeadline": "Update task.py", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 3}}, {"node": {"oid": "084a9119fc0cbdde3d32c3516a677d7a99a44302", "messageHeadline": "Merge pull request #821 from EleutherAI/drop", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 280, "deletions": 1}}, {"node": {"oid": "33137b360233fb94f5c20967d86460d924b413cf", "messageHeadline": "Merge branch 'big-refactor' into drop", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 144, "deletions": 1}}, {"node": {"oid": "8d1d003db5772224d0348c81260101c963bba236", "messageHeadline": "Update task.py", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 7}}, {"node": {"oid": "b5ed3a9b45d41569c4135f6d36a7687d2fdf4554", "messageHeadline": "Merge pull request #820 from EleutherAI/coqa", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 143, "deletions": 1}}, {"node": {"oid": "8287fe7cf2bb6d4dfae919c7a7ffee9c322df59f", "messageHeadline": "checked coqa on readme", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f555a583b2617ea88b21c55d0e4cadb8530c62bd", "messageHeadline": "fix formatting", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 6}}, {"node": {"oid": "79aa53b1ec0e6bf2cf161444c509170bf3662d2b", "messageHeadline": "running process for drop", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 182, "deletions": 13}}, {"node": {"oid": "64d4600c4717a578e76cbf5236d03e43f17937d4", "messageHeadline": "add coqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 143, "deletions": 0}}, {"node": {"oid": "f9558ce5ddea0dee5ad0e6e8fc5bc5676765aed3", "messageHeadline": "add drop", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 110, "deletions": 0}}, {"node": {"oid": "cddbf9f6db94a4a4600c0eaf733c39024a4f8148", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 6}}, {"node": {"oid": "65e5bc69de7b091a11a29603cb2053a570d53094", "messageHeadline": "added checks for if gold is outside choice options", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 22, "deletions": 4}}, {"node": {"oid": "c0cb0be7b764e03fc07dbb36345c78381a02c8b0", "messageHeadline": "moved benchmark up", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 37, "deletions": 0}}, {"node": {"oid": "0a5bd86978c6c66b03b87f85b872d7e17232a7f9", "messageHeadline": "updated yaml templates", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 20, "deletions": 19}}, {"node": {"oid": "c052637ff2dd1312efa5f2a57d88a567a80fd077", "messageHeadline": "Merge pull request #817 from EleutherAI/lintangsutawika-patch-2", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "7e875dcf81e051b00500efa12ef93e8800376eef", "messageHeadline": "moved templates, edit templates", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 48, "deletions": 85}}, {"node": {"oid": "db79d92743784911f18829aae9adaeab186d222d", "messageHeadline": "added subtasks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 73, "deletions": 15}}, {"node": {"oid": "f1620c2d9451c8095067eca24f8f4723d7df9cdd", "messageHeadline": "Update main.py", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "4bdf11e1785bc6b2005153449975d05eaf7fd8b1", "messageHeadline": "udpated to work", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 17, "deletions": 17}}, {"node": {"oid": "d1c3cb3d0c6530d6a4a4eea3e5ddbe470557796e", "messageHeadline": "expanded benchmark to allow new source of prompt templates", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 236, "deletions": 41}}, {"node": {"oid": "21e1ed1741b953c38cf91fa82edc42badd332bf8", "messageHeadline": "Merge pull request #769 from EleutherAI/superglue", "author": {"name": "Lintang Sutawika", "email": "lintang@eleuther.ai", "user": {"login": "lintangsutawika"}}, "additions": 544, "deletions": 51}}, {"node": {"oid": "a27e8ed11fbc47048cac952f132eaa95c04f9bf1", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2064, "deletions": 295}}, {"node": {"oid": "451a1873e42619ce629a441ac9f88541faed33a5", "messageHeadline": "adjustments", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 549, "deletions": 305}}, {"node": {"oid": "a9757f023356c7c8f7946a367e6076a1a1e17cf5", "messageHeadline": "use uploaded datasets", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 24, "deletions": 618}}, {"node": {"oid": "5172d176a26524201126a8ef38dbc459471fc743", "messageHeadline": "update to format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 29}}, {"node": {"oid": "d627333a6088bf8dee6d5cd51e9bff40f3170c85", "messageHeadline": "merged with latest", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1829, "deletions": 283}}, {"node": {"oid": "e4b33c33283b9fe81d069eaf2323e04901647a13", "messageHeadline": "removed unused task", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 5}}, {"node": {"oid": "25a667f5330247fec17c5c19bc0d76eee3b20f5f", "messageHeadline": "Delete dataset_infos.json", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "1b3330f3e0a4d1f5edde31114b59c9e4418f8bd2", "messageHeadline": "Delete asdiv.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 111}}, {"node": {"oid": "79b6434b920620ca8e2e42e74550ef3e8d40096e", "messageHeadline": "Delete __init__.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "ddfcb1fd47daf45038df0d3c75962b78737fb3ff", "messageHeadline": "Delete __init__.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "94011b10b3ffba46b1d9aa5004e69c08f83ad736", "messageHeadline": "Delete README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 8}}, {"node": {"oid": "8817b561d8a1dc0dec65b7183380e25ec4c3a412", "messageHeadline": "Update default.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e75fd29852a7420b2294f03ed2e5e168b1e24ead", "messageHeadline": "fix typos", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 45, "deletions": 1}}, {"node": {"oid": "d1ce515a80ee007a82c8ea9e6d2882589d555fac", "messageHeadline": "Merge branch 'asdiv' into model-written-eval", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 141, "deletions": 3}}, {"node": {"oid": "153c335125af30285cfaab02377a63c18bd22c00", "messageHeadline": "add to prevent task name change", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "32efba1705b53d5da4baf13878e11b5c24fe3f3a", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 25, "deletions": 22}}, {"node": {"oid": "f4b1eb0f277b69d0c972fe6d12b7399d6f493587", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "d97e7155f14b7cf6b8116bfe03a8173ea0b532b8", "messageHeadline": "added model written eval configs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 594, "deletions": 0}}, {"node": {"oid": "10c377ea33ef1c219a574e9f9ff7b5e5600e1da0", "messageHeadline": "added anthropics' evals", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 611, "deletions": 0}}, {"node": {"oid": "dd5b526b59b438980ba348b7f1330883b64bae37", "messageHeadline": "format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "3768b1c4e37a74eca2ffd70c76841d655334631b", "messageHeadline": "format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "e55da20fc235306395f3962f5a2ca23a8c909ed3", "messageHeadline": "add mention of interface.md here", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "352e83161d36e501d64abb0b893ec6452f11497c", "messageHeadline": "add mention of interface.md here", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "648dcef104e8bf894704d8d45664c925408a7566", "messageHeadline": "add yaml", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 0}}, {"node": {"oid": "69a0822226b524887954929c6a11c6ab6ac6d586", "messageHeadline": "add asdiv task", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 128, "deletions": 3}}, {"node": {"oid": "573c2bf91b6e9ab18574f4c4eb3d521c246c317b", "messageHeadline": "update README with accelerate FSDP fixes", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "7c30839b12d4efeae2520a98cf7eb5da4571f627", "messageHeadline": "bump minimum accelerate version", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "92561822662e383b41a8400e8439687dc208de3e", "messageHeadline": "fix FSDP error with .prepare_model()", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 10, "deletions": 4}}, {"node": {"oid": "04f5697d647e193cb13cfe6b9f07f5661dd89ea8", "messageHeadline": "add to readme", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4cda3a1c476fce0d721d12da049ab9758b780650", "messageHeadline": "Merge pull request #744 from EleutherAI/update_docs", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 44, "deletions": 20}}, {"node": {"oid": "a551c7895867a45c1429f26f31370bb54c859e24", "messageHeadline": "Fix lint failure", "author": {"name": "Ethan Smith", "email": "ethan@ethanhs.me", "user": {"login": "ethanhs"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "cd460fc5ab1b23c7bfd39b3aff397668b1585b02", "messageHeadline": "Add pyproject.toml", "author": {"name": "Ethan Smith", "email": "ethan@ethanhs.me", "user": {"login": "ethanhs"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "8ba30fa090d6b8ff3f1c854e9048dca7a6f6501f", "messageHeadline": "Add trailing whitespace to mypy.ini", "author": {"name": "Ethan Smith", "email": "ethan@ethanhs.me", "user": {"login": "ethanhs"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "fc69d84f0fdeed782fa4971b08f096d80fa9b8bc", "messageHeadline": "Add suggestions from autotyping", "author": {"name": "Ethan Smith", "email": "ethan@ethanhs.me", "user": {"login": "ethanhs"}}, "additions": 122, "deletions": 136}}, {"node": {"oid": "da85f290dca3deedde2e88cb870afadf7aab9ad3", "messageHeadline": "First clean mypy run, disable all errors", "author": {"name": "Ethan Smith", "email": "ethan@ethanhs.me", "user": {"login": "ethanhs"}}, "additions": 35, "deletions": 0}}, {"node": {"oid": "20f0f313c2ec5b7493b0441c0fb683749de7b349", "messageHeadline": "add wsc273", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 111, "deletions": 0}}, {"node": {"oid": "3add967a5fc52ada88556c2781ffcdbe645be555", "messageHeadline": "Update new_task_guide.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b7082722ccc69fc6bb80459f4b5e461f2bcb6282", "messageHeadline": "removed unused line", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 3}}, {"node": {"oid": "5aec6f26851a14720ae1dee815aae71b494df2d5", "messageHeadline": "add benchmark with t5x", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 0}}, {"node": {"oid": "1d77a7b816f7803b7de066e34c485e529176d4b8", "messageHeadline": "added info", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 69, "deletions": 0}}, {"node": {"oid": "ec4bc4930f30ec370a0c8a9a35f1ad26f1e322cc", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a3eb639302959fabf04d734d4187d3f03b0d57bb", "messageHeadline": "fixed group to int", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "caac08430e6e610466f7cf40eaf03f3c2bd23ad2", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 127, "deletions": 110}}, {"node": {"oid": "adbbfb44d33a363a2a2a09d37744c646c4ba5c0e", "messageHeadline": "adding record processing", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 165, "deletions": 6}}, {"node": {"oid": "599205c478357e9ef919d8aa85a7799a2df2a581", "messageHeadline": "edit to accept docs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "faa2e43d7a16d2de5b321855f00ba741d69ee417", "messageHeadline": "add wsc utils", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 96, "deletions": 6}}, {"node": {"oid": "007d485b6c84b0bb9635ba81e29b3d7c42a9ab55", "messageHeadline": "modified apply method to accept docs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 7}}, {"node": {"oid": "cb4858837505dfead1e096cc899afabe08722114", "messageHeadline": "filter takes docs as argument in case filtering requires it", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 17, "deletions": 8}}, {"node": {"oid": "71ab0f2cbed0663e850a961a4352f5a58c7188d3", "messageHeadline": "added setup caching", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 3}}, {"node": {"oid": "8b1e652f3d38d580d0961dbd9b47c71a539b826f", "messageHeadline": "added multiple python check", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 9, "deletions": 4}}, {"node": {"oid": "7d16a7cd8af33c63da1fd2e36f8a1957788e33a6", "messageHeadline": "merged updated big-refactor", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 9}}, {"node": {"oid": "a005aebaf51e731038905102f8fc57e42806b261", "messageHeadline": "Merge pull request #802 from EleutherAI/fix-metrics", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 9}}, {"node": {"oid": "f5d0f8e63e2963985b1d60fca2d8bb3045a19653", "messageHeadline": "minor fix", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4fe3409e8307668f1e091c3813e1dcd520fb8a66", "messageHeadline": "readded missing chunk", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 20, "deletions": 0}}, {"node": {"oid": "c01d5bac1115670de26ca545a114a18891977521", "messageHeadline": "Merge pull request #803 from Wehzie/big-refactor", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f3a82004c4cab322348c26a9f1526ec2dee0ba54", "messageHeadline": "Merge branch 'fix-metrics' of https://github.com/EleutherAI/lm-evalua\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "2f0ef6ca03f26353ba92d3a4417accee181fddde", "messageHeadline": "fix condition where a benchmark and a regular task is called together", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "91d3f6c47491f7d87a898c57f815e2b42a8c6666", "messageHeadline": "Update README.md", "author": {"name": "Rafael Tappe Maestro", "email": "r.tm@outlook.de", "user": {"login": "Wehzie"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "45cc68ee965d3bb383b9c1465eece6b31264c00a", "messageHeadline": "Merge branch 'big-refactor' into fix-metrics", "author": {"name": "USVSN SAI PRASHANTH", "email": "55424359+uSaiPrashanth@users.noreply.github.com", "user": {"login": "uSaiPrashanth"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "691b4ca979a3bc7581d27c52650116258138e7b4", "messageHeadline": "temp", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "32294ff1768723666857b97fdb53702a3aa2366f", "messageHeadline": "fix for choices", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 0}}, {"node": {"oid": "18f7dfb3c5548c2b65124e8095802b9ecc890113", "messageHeadline": "temp", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "8c93320ae03518b74730417b6d895e7ccb916143", "messageHeadline": "add multirc metrics", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 51, "deletions": 1}}, {"node": {"oid": "4a768feb23e8534896fe291d2f8f0e81b7a32ad0", "messageHeadline": "add cb metrics", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 31, "deletions": 0}}, {"node": {"oid": "80d88899f9b06e7d6108900ea40833598f239189", "messageHeadline": "fix test_choice test", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f7a0235d8467bcf6fe1402f2b64f0c499dea2f28", "messageHeadline": "update yaml", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 4}}, {"node": {"oid": "d4e075e3f672b89b83fc39bb51507795ff02b7f4", "messageHeadline": "added truncation option", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "f96f5fad4ced4b23afa7a5071946ad9637147412", "messageHeadline": "add endspace", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "bf4eb3e8e344dee0b9a620f5b294be569bbb4417", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 96, "deletions": 13}}, {"node": {"oid": "cc6662e10d066fa515b80a038fefa8084feb4d7a", "messageHeadline": "add endspace", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b5b01a497e698c78cc8bc1eb575b2c25bca64e35", "messageHeadline": "Merge pull request #780 from EleutherAI/fix-metrics", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 96, "deletions": 13}}, {"node": {"oid": "48e3254051253d58925bbb73cb7fe3bb247f717f", "messageHeadline": "add to ignore.txt", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "e3c70cad06c405cbd152028bb7d6d0dff757a249", "messageHeadline": "fix bleu", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 13, "deletions": 6}}, {"node": {"oid": "26196917d25454ee3118430ee9c3a97247bdaed5", "messageHeadline": "result is just 1", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "eaa7afe2597a1fd0c7f07fceb049475fc9dd6f0d", "messageHeadline": "Create user_guide.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 9, "deletions": 0}}, {"node": {"oid": "b1d468f25f0041dca2664345ac82fc0d56c518d1", "messageHeadline": "fixes to multiple_target", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "8436e11f5dcebbd17f96abfe41d575bb838ceee4", "messageHeadline": "initial commit for nq_open", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 29, "deletions": 0}}, {"node": {"oid": "4bba7abbe6fd15abb9ca002e3e23b227a8ad9a31", "messageHeadline": "no need to iterate over results for greedy_until", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 26, "deletions": 26}}, {"node": {"oid": "f71d56eb6caad4fc3ec234a677d5835722cc8216", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3155, "deletions": 438}}, {"node": {"oid": "a38a04504ee26cf924d290f39b497a48c3ebeb19", "messageHeadline": "fix naming", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 4}}, {"node": {"oid": "2f8702659dce7826b6dbb0daab7b3ef31e0c0798", "messageHeadline": "Merge pull request #781 from EleutherAI/process-whitespace", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 29, "deletions": 2}}, {"node": {"oid": "b2aee121ab72eee8b2330c8aba8777aff9598b3f", "messageHeadline": "add checks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1cc2a764c6012e5a1d7f87344d6c383c3cbe0cd5", "messageHeadline": "Merge branch 'big-refactor' into fix-metrics", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2361, "deletions": 251}}, {"node": {"oid": "3d732e68237720905ff97ef41e36a1373c7d8a57", "messageHeadline": "Merge pull request #793 from baberabb/big-refactor_test1", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "21706139103a8bf47bb06ae4f047749db7aecce2", "messageHeadline": "increase atol", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a093ad5f9b8a61f3e3047348520ae3dd622c39f4", "messageHeadline": "increase verbosity", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0cdd730e547d9185bc4b21b071f8c757729cfc4c", "messageHeadline": "Merge pull request #791 from baberabb/big-refactor_hgtest", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 133, "deletions": 16}}, {"node": {"oid": "51882c1eee28562ee52c07a6d25359c88c42b854", "messageHeadline": "atol=1e-8", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8d2852444b6cd5ed16889924c379a57b7861229e", "messageHeadline": "Merge branch 'big-refactor_hgtest' of https://github.com/baberabb/lm-\u2026", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "31ed1c2a62655d1df44055d12a0faac50de1c498", "messageHeadline": "add branch wildcard; pytest verbosity++ in case failure", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "ac290ce8b2bbfb380a1b54fe2ca6598f10339a3d", "messageHeadline": "get _RES from CI and change atol to 0.0001", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 43, "deletions": 42}}, {"node": {"oid": "e1c46787f403b730ef199a71e6f425826ee5472e", "messageHeadline": "add print to view result on CI", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "a004fdc03f71b53363eec8358b0000722e553003", "messageHeadline": "delete pull_request.yml", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 0, "deletions": 13}}, {"node": {"oid": "9fc9fd021c12bd7d27118685032f30acf264f680", "messageHeadline": "consolidated precommit to linter", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 15, "deletions": 13}}, {"node": {"oid": "da77b2168c228bacc1cbde68e5199a7707775d7d", "messageHeadline": "added indices equality check", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 5, "deletions": 0}}, {"node": {"oid": "72bb97ca9b8dc05de39ad09bdf42950893ed222c", "messageHeadline": "change logliklihood atol to 1e-2", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "91a060819ac52ebbb4239b23995b90b7c19a5d7a", "messageHeadline": "fixup! added tolerance for likelihood, rolling", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "dbbc613fd8840f8aa9b780505c29cdcc8a68a1f9", "messageHeadline": "added tolerance for likelihood, rolling", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "b9df9fb4cb2e89380a7b2fe385e688b3e2e1f2f7", "messageHeadline": "added pip cache to CI", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "c2e76e7917c211c8f52fb68d8c05bb1e39edeccf", "messageHeadline": "Merge remote-tracking branch 'origin/big-refactor' into big-refactor_\u2026", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "1c5a73c98f3ef9f7bde1d650b6ad17936aa4d444", "messageHeadline": "Merge pull request #788 from EleutherAI/fix-perspective-apikey", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "f0aa2c7b411a62bd69bba4d2de25b344aae4347c", "messageHeadline": "added hf model test", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 117, "deletions": 0}}, {"node": {"oid": "4156a0051e8a7fdb6ad7b8fb043a15ca8353ae98", "messageHeadline": "improve --write_out", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "b5ce475d23a8ed56db8810806c4a760eddcf229a", "messageHeadline": "re-enable MultiChoice on --tasks", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2e2f28a5f3031753f3b5ed6e3fd51165b83ec02a", "messageHeadline": "add XX->en direction to translation tasks", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 86, "deletions": 41}}, {"node": {"oid": "197940f7ba1cf6e1e01bdc676210143b2b8db207", "messageHeadline": "don't exit if RTP api key not set", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "129762c2496014e283554429e60431597df26f50", "messageHeadline": "Merge branch 'wmt' of https://github.com/EleutherAI/lm-evaluation-har\u2026", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 759, "deletions": 31}}, {"node": {"oid": "603eced6302a5ac1803a0abf76ede2ed7e3f2851", "messageHeadline": "update readme on translation tasks", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2d5d3cd28edd73f5e60f6ed8ee9df5825f387677", "messageHeadline": "format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "767c58b9f809b2af83c898e42205f8705c3fa40d", "messageHeadline": "Merge branch 'big-refactor' into update_docs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4077, "deletions": 528}}, {"node": {"oid": "759da8d53bae2d1d1f7796aa9dddcf8c84aaa161", "messageHeadline": "Merge pull request #757 from EleutherAI/add-readme", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1129, "deletions": 208}}, {"node": {"oid": "c05a5ad4f93f45e815339e87b28b239860ef60b7", "messageHeadline": "Merge branch 'big-refactor' into add-readme", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 626, "deletions": 29}}, {"node": {"oid": "9f4682a33c330a8d1e53e6efdfd55bc4bfd0c39e", "messageHeadline": "fixed typos", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "3317c627346e144c8912f852281d30f71769020b", "messageHeadline": "Merge branch 'big-refactor' into process-whitespace", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 540, "deletions": 91}}, {"node": {"oid": "5a973941924304d1ce4c549ae8a46cae72b876c7", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 33, "deletions": 29}}, {"node": {"oid": "9e9327bc36aec344fc827cd62982a812a5017a76", "messageHeadline": "Merge branch 'big-refactor' into mgsm", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 367, "deletions": 28}}, {"node": {"oid": "73912efba86938e0dd3fcf9dfcf3ed8230f3cae1", "messageHeadline": "Merge pull request #786 from EleutherAI/fix-doc_to_target", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 42, "deletions": 24}}, {"node": {"oid": "13f84b834d3ca2e69d6540405099829284d16d90", "messageHeadline": "raise SystemExit 0", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "2fcd429827054719156a358e460bf2333805a5f5", "messageHeadline": "changed raise variable", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "e8825ef6c5f9d7fbdef8c583ede7bcfba4ce03f9", "messageHeadline": "Merge pull request #782 from EleutherAI/truthfulqa", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 263, "deletions": 4}}, {"node": {"oid": "5b4f175f1a677530650637a2c20034798120fce7", "messageHeadline": "Update utils.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 46}}, {"node": {"oid": "d53a81b02361738868d6cef61ac41d634d756601", "messageHeadline": "Merge pull request #783 from EleutherAI/mctaco", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 67, "deletions": 1}}, {"node": {"oid": "579c9db9387704f1cc14a76b118fba8876751363", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "60007dcff07547d63da4b33f7d8b8e1279700105", "messageHeadline": "change process to not require googleapi library", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 25, "deletions": 18}}, {"node": {"oid": "d6030e129c568f0401c77e83c874613a4410eb8b", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "db37024b3c22e99443c19d3224ebe90d5cfc6de9", "messageHeadline": "fix doc_to_target processing", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 5}}, {"node": {"oid": "83f95961e4991493b83727581a6306b9a804bfa7", "messageHeadline": "add task variants", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 269, "deletions": 29}}, {"node": {"oid": "d99e6cf4f9e6734932b5f13c501bfc6fb9c566e1", "messageHeadline": "fix doc_to_choice being called when target is digit", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "3e9c9a950cb8dec463d000249be11ff5478bf680", "messageHeadline": "add mgsm yamls", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 105, "deletions": 17}}, {"node": {"oid": "416a3ca1cfb6f0287ceb080c492f5956217119cf", "messageHeadline": "add mgsm", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 219, "deletions": 0}}, {"node": {"oid": "da8af9716b501b61dd20990d8b1a08760c7f188e", "messageHeadline": "update readme docs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 27, "deletions": 10}}, {"node": {"oid": "cfbc5a8ac2bf501c54f12e1b33475551519905d4", "messageHeadline": "add mc taco", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 49, "deletions": 0}}, {"node": {"oid": "135b41ce5c18f0ff8ff3bd9b06d83600d8cd0e0e", "messageHeadline": "format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 23, "deletions": 21}}, {"node": {"oid": "0436b5d6558a495f93b442223370e6febf1b9d5b", "messageHeadline": "edit readme", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "b990b75673efbbce15f41c086dff531eac1502c8", "messageHeadline": "add all truthfulqa tasks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 304, "deletions": 1}}, {"node": {"oid": "e3077dcf4b0008c84dcc6b1b45cdb30dead4b4ff", "messageHeadline": "Merge branch 'big-refactor' into wmt", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 759, "deletions": 31}}, {"node": {"oid": "8eab2a586ac06c6d683acff29ba0395e9e7a2713", "messageHeadline": "Merge pull request #779 from EleutherAI/pawsx", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 253, "deletions": 1}}, {"node": {"oid": "21aa92d26ffaf028976e25a7593abaee8f414d2e", "messageHeadline": "rename var", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 14, "deletions": 12}}, {"node": {"oid": "fc329d319afc263fb6d600663944d3b24944f0de", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 42, "deletions": 74}}, {"node": {"oid": "40f81c42e5eb82adbfb389cd7d35f7e7031856d4", "messageHeadline": "add translation tasks", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 239, "deletions": 0}}, {"node": {"oid": "39bd5caf02cb0cd94c1c090baada2a7fcb924289", "messageHeadline": "make pycountry optional dep", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "60ca1a27c6bac1d787079c61316243335e37abad", "messageHeadline": "update squadv2", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 183, "deletions": 0}}, {"node": {"oid": "523ba9541173a317e7340bc55a69a27ce9fd607e", "messageHeadline": "edit xstorycloze", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 84, "deletions": 0}}, {"node": {"oid": "5be5cfcf4a1335f3a07a4d5ebbd50b073079bbfa", "messageHeadline": "update triviaqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 2}}, {"node": {"oid": "14ad1317796367c17c927694c01e7948cc99e67b", "messageHeadline": "udpate storycloze", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 4}}, {"node": {"oid": "95b2a7d29bde44095ead1ea26cad54309014618e", "messageHeadline": "update glue", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 15, "deletions": 4}}, {"node": {"oid": "95360bc2be8ec300e766c6919fb989c8e0fbc56b", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1703, "deletions": 223}}, {"node": {"oid": "30aa9c337a03268d3fcc0f190abc0c253e11ad41", "messageHeadline": "Merge pull request #758 from EleutherAI/add_triviaqa", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 114, "deletions": 9}}, {"node": {"oid": "4b0ab122a699141f402190aab65dd170d552bdb0", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0629a8bff61bf4cec10b0396b3d11aa4a15f1813", "messageHeadline": "Merge pull request #761 from EleutherAI/glue", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 173, "deletions": 81}}, {"node": {"oid": "b78a0045e0cf35bf6a2d7f5facd72a1556220eb9", "messageHeadline": "merge big-refactor", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1416, "deletions": 201}}, {"node": {"oid": "0a694910c5cc15274702f81b52dd174312feb32b", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 11}}, {"node": {"oid": "61c494499382a851db9980de780bc76e67127d8d", "messageHeadline": "Update seq2seq.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "64e50a497d2ec8aa27e0a0201efdb6b6a0f3c8b0", "messageHeadline": "Update seq2seq.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0458e152aef701dd83c2e2857e737362732c8a99", "messageHeadline": "go through choices and targets", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 22, "deletions": 8}}, {"node": {"oid": "ffb56822d32c2b3e5c28c00685b38cc31c5eb7dd", "messageHeadline": "removed comment", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "db97d7f8a406186d257955049046e92ee9a426cd", "messageHeadline": "added warning regarding use of whitespace for target_delimiter and ta\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 27, "deletions": 8}}, {"node": {"oid": "e85ca1a9cb1ed04a995a20ecfb99dd15170822eb", "messageHeadline": "Merge pull request #759 from EleutherAI/xstorycloze", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 217, "deletions": 3}}, {"node": {"oid": "55749b9b4535ec1bf9d325f29cddd1fbc1d718ed", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a68a3092a4a649506dc68329638bdb55afe16734", "messageHeadline": "Merge pull request #725 from aflah02/RealToxicityPrompts", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 66, "deletions": 1}}, {"node": {"oid": "3e636f5bdcfca5bff25715df85be95fdb759d076", "messageHeadline": "minor formating", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4b87456d5ed7b2dea17e7d4c4d25a347254726ef", "messageHeadline": "made fixes", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 102, "deletions": 25}}, {"node": {"oid": "35a2465256c4a6ad85076aeba69fc316e05acff1", "messageHeadline": "Merge pull request #1 from EleutherAI/toxicity-test", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 2240, "deletions": 391}}, {"node": {"oid": "4a9b37867439e35813d982783b89fa759d94e4e5", "messageHeadline": "format readme", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6c587ba989ef8ba77c89eae3ad6b0d5a7e451826", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 23, "deletions": 12}}, {"node": {"oid": "e9377f7271f6cb0c4e8ba68518d0e0857de33c62", "messageHeadline": "add paws-x", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 241, "deletions": 0}}, {"node": {"oid": "adde09b30d1dfa016677bd56a9f473439f9079d5", "messageHeadline": "Merge branch 'big-refactor' into wmt", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 763, "deletions": 186}}, {"node": {"oid": "0021de216bfe77ffedf6b7c35ab2daa47030fe3c", "messageHeadline": "Merge branch 'RealToxicityPrompts' into toxicity-test", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "4c22a521f0ae648e695f9847fbc1e2a8a3a18897", "messageHeadline": "adjustments", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 34, "deletions": 47}}, {"node": {"oid": "c478e20a9caab0ab43a40af1a7e975c3ce29d46d", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 763, "deletions": 186}}, {"node": {"oid": "88511112e8f5ae94c0a04db9991472a70ae2b4b3", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 28, "deletions": 12}}, {"node": {"oid": "33273d6ef7d5fbdd7299f9e2eb5eb8d11fdfd2ff", "messageHeadline": "pull and modified input args to metric", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "607634089b27722e7461d0f4a3b8baddabb5df90", "messageHeadline": "Removed Implementation from metrics.py", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 0, "deletions": 44}}, {"node": {"oid": "bdc23326a5ff2a986eecf927380d769d58ce9b7d", "messageHeadline": "Delete realtoxicityprompts_yaml", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 0, "deletions": 4}}, {"node": {"oid": "ce7eb7b6249017c6641f93f4fcfb8404f5058938", "messageHeadline": "Addressed Review Comments", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 46, "deletions": 1}}, {"node": {"oid": "67bbc5b42b56d7a7e3c123f501095451555738f8", "messageHeadline": "Fixed Wrong File Push", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "57bcb4d66a9b6a5a920ca8c37b400a7e5008aced", "messageHeadline": "Improved DocString", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "b221531afbfedae242151d784727f32485548e5d", "messageHeadline": "CMD Line Works", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 11, "deletions": 4}}, {"node": {"oid": "d857e5397e2044eab07270ccac7a391a179a0e5f", "messageHeadline": "Added Metric and Seems to Run but some Errors", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 15, "deletions": 5}}, {"node": {"oid": "94ccc42939eb2e8a88ed44784e3a914003821e07", "messageHeadline": "Base Template", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 48, "deletions": 0}}, {"node": {"oid": "64c76fc3f41d85424339259dec22329e2a564bb3", "messageHeadline": "Merge pull request #747 from EleutherAI/haileyschoelkopf-patch-2", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 76, "deletions": 31}}, {"node": {"oid": "725d7e3ec09a2a9c3ee4c7962b9485d80028ed10", "messageHeadline": "Added info on `lm-eval` command.", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 0}}, {"node": {"oid": "be79a281c17fbef6884704a8ca86026e987fe9f4", "messageHeadline": "Merge pull request #777 from EleutherAI/benchmark", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 289, "deletions": 149}}, {"node": {"oid": "416b198e6222c5f476bd55a98c1e04811299527b", "messageHeadline": "Update default.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "f918c8fdc2705a5ac26732a91ea053b4873bd8d3", "messageHeadline": "Merge pull request #776 from EleutherAI/xnli", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 368, "deletions": 1}}, {"node": {"oid": "1768f118107c9a6b1bf3a10801d78b832e22185d", "messageHeadline": "Update with task list.", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 61, "deletions": 0}}, {"node": {"oid": "545fb8fc1fc98e3c7cf9e91229cf6dd471bb2798", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "f2343efb9e83c67fcdcb47064b499dbf832bb295", "messageHeadline": "edit xwinograd", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 4}}, {"node": {"oid": "dfa8c5c056148e030c2074d5ef17efd7d8db47e4", "messageHeadline": "edit xcopa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 43, "deletions": 2}}, {"node": {"oid": "1abe9c51b16da4b8c6fb351d567633851f12fd10", "messageHeadline": "edit winogrande", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 54, "deletions": 0}}, {"node": {"oid": "7984983fc8df46edd3ae674d704dcbf45d0361d7", "messageHeadline": "edit wikitext", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 4}}, {"node": {"oid": "bb869c27cd3992f479979872a8f7c1bf0025a8a1", "messageHeadline": "edit webqs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 4}}, {"node": {"oid": "05a8191af6237341c685616d8ef372f0a7bff344", "messageHeadline": "edit unscramble", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 6}}, {"node": {"oid": "4c93551951bf3e6e9fe3b3d173b2708a512be77d", "messageHeadline": "edit thruthfulqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 22, "deletions": 5}}, {"node": {"oid": "ed3b0444a842942f002bccb904b6451782edc590", "messageHeadline": "edit toxigen", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 25, "deletions": 32}}, {"node": {"oid": "ef429f65de7bd46707ed2324f99bb20eee6b416b", "messageHeadline": "edit swag", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 52, "deletions": 2}}, {"node": {"oid": "2ee7d5ba1d28b9e5cca6ee821801ca66d46d07f4", "messageHeadline": "edit sciq", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 49, "deletions": 2}}, {"node": {"oid": "8a22e9e354e261c836db81c2081b6f8c2d22fea3", "messageHeadline": "edit race", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 44, "deletions": 2}}, {"node": {"oid": "3f1c75e130d1b361991025b62cf6e61c8caa7bbf", "messageHeadline": "edit qa4mre", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 56, "deletions": 1}}, {"node": {"oid": "e74e3a2e976001c985945d9311c3f08a4c6e56e1", "messageHeadline": "edit pubmedqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 56, "deletions": 2}}, {"node": {"oid": "9b571101a7bb7991c13f57c321b258b496b65fe4", "messageHeadline": "edit piqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 52, "deletions": 0}}, {"node": {"oid": "b7b58bc46ea49673cd07faab8945180fd756900e", "messageHeadline": "edit prost", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 62, "deletions": 2}}, {"node": {"oid": "32ba8c58d3e4de6c40ea89eeaaab249bf88d5eeb", "messageHeadline": "edit piqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "e240d2f2644d6ed65722d8748dc992512048d459", "messageHeadline": "edit openbookqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 1}}, {"node": {"oid": "d19e90de33d82f370ee6926dc64fc7a2e3aeaf2b", "messageHeadline": "edit pile", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 47, "deletions": 4}}, {"node": {"oid": "7714e30bc8560545ca8c4b09664d31577edb7e65", "messageHeadline": "edit openbookqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 49, "deletions": 2}}, {"node": {"oid": "e0024475337fab7f6197cb560a310f088d72d13a", "messageHeadline": "edit mathqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 2}}, {"node": {"oid": "687587617507200f103e41a8c7966f32df05e76e", "messageHeadline": "edit logiqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 61, "deletions": 11}}, {"node": {"oid": "223a63c4bcd835de562359e8b7d30272f1eced3b", "messageHeadline": "edit lambada_multilingual", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 19}}, {"node": {"oid": "66e05496d2842cc7363f54368701798c2c85a15d", "messageHeadline": "edit lambada cloze", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 56, "deletions": 2}}, {"node": {"oid": "219c81bd0fa91e206b38de5c90d9a251b89255a6", "messageHeadline": "add lambada", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 15, "deletions": 6}}, {"node": {"oid": "986943e50f7c35b0df6671d8e212cc76cc4d508f", "messageHeadline": "edit hendrycks_ethics", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 5}}, {"node": {"oid": "2a9dc8f407ebf54a1f8bff3299ea2d9b569f30bd", "messageHeadline": "edit hellaswag", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 1}}, {"node": {"oid": "c4c30c6aa3f6b3942ecece4d3a5ed246bbb250aa", "messageHeadline": "edit headqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 2}}, {"node": {"oid": "459a534779b64bf45fda76fa594cf872f010a594", "messageHeadline": "edit gsm8k", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 8}}, {"node": {"oid": "e414727a0da8364680c7d8735ea52ee3258a2df2", "messageHeadline": "edit crows_pairs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 3}}, {"node": {"oid": "fe6a8658d4cb0e3893776163391d348dc939d29a", "messageHeadline": "edit babi", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "414f6596efce34033d4b1f7f36b7ebfc9a378f65", "messageHeadline": "edit arithmetic", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 7}}, {"node": {"oid": "e2529a6fa881a0b63a03921bdb68ca599bbeb7d4", "messageHeadline": "edit arc", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 10}}, {"node": {"oid": "0456d54385de36a8ec7245e43d290bc085acf5d1", "messageHeadline": "edit anli", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 58}}, {"node": {"oid": "ae55476a1825c8935a0e1103ca6248d611d1153b", "messageHeadline": "adjust format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 3}}, {"node": {"oid": "3c261c3e5488636beb15f11bef5e8d6b6f9563b8", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 4}}, {"node": {"oid": "cf4519f28954670c24953dca42831918b726d16f", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 0}}, {"node": {"oid": "bff066c6161234b1c846c172ed35fa7dffc46462", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 29, "deletions": 4}}, {"node": {"oid": "1eefac8845111f58453c966c6e44fcb1358f7e82", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 41, "deletions": 6}}, {"node": {"oid": "8dbb795827330ce27259aebb445d4ec0bbecaece", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 0}}, {"node": {"oid": "be2ebb3b793c5bc543c1197723ab1709ec8caa98", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "3de2c0e7e73d5ea1e18327f4ab76cdd93675e14b", "messageHeadline": "adjustments to have `task` to modify a task name in benchmark", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 54}}, {"node": {"oid": "cccb294f3ded35666ed96e9e7eadf92b6fbfbfe5", "messageHeadline": "explain how to use benchmarks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 78, "deletions": 1}}, {"node": {"oid": "7758d16a7f644fc655ab8f86856ccb2d2d281425", "messageHeadline": "return fewshot_delimiter if we get \"\"", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "1701c4e23defaa7fb054b60c49fe238edd92fb35", "messageHeadline": "add warning log if prompt returns empty string", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "722b8974a31e23c3322e30184426164834287361", "messageHeadline": "add warning log if prompt returns empty string", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "bc951ac161b5e68b2264e9097196f853fea5051f", "messageHeadline": "add warning log if prompt returns empty string", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 2}}, {"node": {"oid": "ce8c2b14c92a3f9ec38eccd1e36155d7bcfa8577", "messageHeadline": "formate", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "5ddabc29f444962b008cde8c2d4f289ccdfc3724", "messageHeadline": "update and fixes", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 105, "deletions": 77}}, {"node": {"oid": "c523063d7bc8b0d49fb8e9d475bdeafbdf71884e", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 3}}, {"node": {"oid": "81a6cbfa51d89d3cd55836d30c792abbb55ccbfd", "messageHeadline": "check readme", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2d54f2c92d3be89db8114dd7af80b0ea0a7f6ccd", "messageHeadline": "xnli reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ac98d9e1702883cec314fb6c4b1a0b54cc68f4fe", "messageHeadline": "xnli reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 38, "deletions": 36}}, {"node": {"oid": "054ed37f3395c7b1f70d22532c2244a6154e5e9a", "messageHeadline": "Merge pull request #766 from EleutherAI/patch-crowspairs", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "4f27f0b994f954a626af9076649cfa32e4b7a0ff", "messageHeadline": "Merge pull request #774 from EleutherAI/speedup-hellaswag", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 27, "deletions": 2}}, {"node": {"oid": "db0c46b0933c1f160dec8278c2206d57ff4d6429", "messageHeadline": "update in task guide", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 0}}, {"node": {"oid": "22ece14a27e7724abe0403deb88a9cc47f624d6f", "messageHeadline": "uncommented tasks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "f4813cfc26cf2306d9382d6a9278baf93ff2dc68", "messageHeadline": "moved to a new dir", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 112, "deletions": 48}}, {"node": {"oid": "83683ddd04c92dcaa6009d8e01f1b1d73f3ceb66", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 51, "deletions": 33}}, {"node": {"oid": "18e2d093e38243735d26368309246e463700fa82", "messageHeadline": "added xnli", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 286, "deletions": 0}}, {"node": {"oid": "df6c5dcbe6127e39974d98bb20e55557fdac2d92", "messageHeadline": "fix multi-metric greedy_until", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "116c540ad3162d6a701fa567ae16f153e89f7587", "messageHeadline": "make chrf and ter aggregations", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 51, "deletions": 31}}, {"node": {"oid": "8189eb188d3f659b9e345ec2e77cdcd01b557356", "messageHeadline": "switch to process_docs", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 27, "deletions": 2}}, {"node": {"oid": "8806eff5796ece5453b6525d874b247f4de660b4", "messageHeadline": "support bleu score as a metric", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 71, "deletions": 49}}, {"node": {"oid": "dbf2c083defa56779b6c3f3e72d07cbf711a9b9c", "messageHeadline": "Merge pull request #768 from EleutherAI/lintangsutawika-patch-2", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c2f76ec795e0f1ad634a7e3e990bd329bacb634e", "messageHeadline": "Merge pull request #770 from tju01/big-refactor", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "d69a962ddf7c69031bea8ec5930487dc734e4f8f", "messageHeadline": "placate pre-commit", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "c7ef3da7bafbfa9cfdfb67cf93e02412f4e7df9a", "messageHeadline": "Use evaluation_mode=True for accelerate to prevent OOM", "author": {"name": "Til Jasper Ullrich", "email": "git@tju.me", "user": {"login": "tju01"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "33f2f9bfe3f06ace71a87d04b39922efff51fcc2", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1436, "deletions": 155}}, {"node": {"oid": "e332a1ec6b7e686f0439f0f392532c57aee18f2a", "messageHeadline": "add readme for storycloze", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 68, "deletions": 0}}, {"node": {"oid": "5c7445a4d1902061ee7afaca93b57523b766fb0a", "messageHeadline": "add storycloze", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 33, "deletions": 1}}, {"node": {"oid": "506a32acf128e125c1c206f880b738fd140ed228", "messageHeadline": "Patch Blimp", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "bede79c29042206c098d17435fa0c6284fbe45b8", "messageHeadline": "Update utils.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "83ed584c06a1a5f6c221afa66b6624318723e922", "messageHeadline": "Update crows_pairs_english.yaml", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "75a6f020c08b6c32810b72a89f47830093a957fe", "messageHeadline": "update readme", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "a5a9017ba29909bf409449f8482f558da62085a5", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 437, "deletions": 12}}, {"node": {"oid": "7634a6ecd3449dcc898129a0bf9363ecae35473f", "messageHeadline": "Merge pull request #763 from EleutherAI/blimp", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 425, "deletions": 0}}, {"node": {"oid": "367f96937e2fc62521649cbb789d06860c1f2777", "messageHeadline": "fix higher_is_better", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "94dce75342395f904630aaf0089986c45a244dd7", "messageHeadline": "add .[all] install option", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 31, "deletions": 19}}, {"node": {"oid": "4a7bbcb484bc7461ce3e7c58f391c8d87308a036", "messageHeadline": "fill out links", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "4907defdcc91d4bd091618ccd3cc806304b7048e", "messageHeadline": "Merge pull request #764 from EleutherAI/lintangsutawika-patch-2", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 12, "deletions": 12}}, {"node": {"oid": "f0cc1507b6169477243d356a2eddd38321115bc3", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 12}}, {"node": {"oid": "b22ccf927cca633a1cabb74e7bd767055e13073d", "messageHeadline": "format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 68, "deletions": 69}}, {"node": {"oid": "b20bb95fce5f78f3ff1f257caeb7e6e325881772", "messageHeadline": "add blimp", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 426, "deletions": 0}}, {"node": {"oid": "e4362b2d3dab821dbeec74e9844892460119783c", "messageHeadline": "removed irrelevant files", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 113}}, {"node": {"oid": "f931ecfb34af2d0f46557a1a8ca66df92fa859b1", "messageHeadline": "fix include", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c04a66e4fccf697dfa9df20b9c11cb003dce5743", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0fa178a52f1fbe510842ed1d281fb6ae0610dd8d", "messageHeadline": "add readme", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 60, "deletions": 0}}, {"node": {"oid": "30bf8801f914200e3ebf7c8fe9a5311ec633a87b", "messageHeadline": "add more subtasks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 27, "deletions": 0}}, {"node": {"oid": "fb7d233610c4032f703ace5f0e262e9ad743aa4e", "messageHeadline": "add glue, renamed sglue's rte to sglue_rte", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 85, "deletions": 12}}, {"node": {"oid": "a04a600a7c59d9a398c601ba3bf4c2eae9551870", "messageHeadline": "forgot \"", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "af3fbe032b37443f04b91e185cb94bfafa655882", "messageHeadline": "forgot to add doc_to_choice", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "cf96455a8f46c375e0638e1c6f6de1553f163db2", "messageHeadline": "fix import", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 10}}, {"node": {"oid": "43bb41454542aba6f722253d954dbd28924c329f", "messageHeadline": "added xstory cloze", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 113, "deletions": 0}}, {"node": {"oid": "1d3599a3eb3c168a57425cfd0bcd8243bccaa0eb", "messageHeadline": "add sampler condition for if target is multiple, and target_delimiter\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "87e147e171350d2d700d10890aa54864eecd3fb6", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 4}}, {"node": {"oid": "6accb48fabc44c617740ca05d1384d1e4bbd7023", "messageHeadline": "add trivieqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 1}}, {"node": {"oid": "898cdd8b778ec3df36ca9cf98f85323b3ceb026b", "messageHeadline": "removed line", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "489edf2c2ee1dbbccd7e827260dfa7484831f89a", "messageHeadline": "added new filter to remove whitespace in the front of a prediction", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 25, "deletions": 0}}, {"node": {"oid": "c2990e29928b39131888215e8ad77a5669b4ebc6", "messageHeadline": "allow doc_to_target to be list so that multiple_target works", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 7}}, {"node": {"oid": "c86fd1a79d1085af7272357d953e26fe9bdfa644", "messageHeadline": "added triviaqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 70, "deletions": 0}}, {"node": {"oid": "dbf03c054ab6d98a6c34b879f7cdf47b0874f749", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 28, "deletions": 5}}, {"node": {"oid": "375346e6f208b21466e50a004c6c76fbb72484e9", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 27, "deletions": 24}}, {"node": {"oid": "decf7d875744bff578e54417fa3bad086df7eb92", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "600073574e0cb728b39fb617bc008d6448d822ec", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "5e2d29584dc234515cacd03493934e1127089aa9", "messageHeadline": "Create README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 18, "deletions": 0}}, {"node": {"oid": "8ef93f15962c2caecba7f027988b7feb46057103", "messageHeadline": "Create README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 24, "deletions": 0}}, {"node": {"oid": "3a67a226e288d5f5fa66065f428c5842d8ba3ac7", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "d8f9640482912d6c5bba596bc940253e99c9a620", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 23, "deletions": 26}}, {"node": {"oid": "d1a44c852f5973943d25638f0d267c1a5616d999", "messageHeadline": "Merge pull request #749 from EleutherAI/xcopa", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 186, "deletions": 1}}, {"node": {"oid": "47ceb0a07ec512f4be58532a87842abb5343903a", "messageHeadline": "Merge branch 'big-refactor' into xcopa", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 299, "deletions": 12}}, {"node": {"oid": "15821d2029c547fcbd048b36331a7357bc985f0d", "messageHeadline": "Merge pull request #745 from EleutherAI/haileyschoelkopf-patch-1", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 9}}, {"node": {"oid": "7483a7ea5068a7205e5e2d9769f39f1da2207d81", "messageHeadline": "Merge branch 'big-refactor' into haileyschoelkopf-patch-1", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 292, "deletions": 5}}, {"node": {"oid": "2f53b1904b18f8fb2c6e307b276d8a3ac27d15e0", "messageHeadline": "Merge pull request #751 from EleutherAI/crows-pair", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 270, "deletions": 3}}, {"node": {"oid": "53487cf67feba6a20b17d4224e3fd6c17a16e8ef", "messageHeadline": "checklist README.md", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "5d55b685afe76f1bea10b741e297633fc755f481", "messageHeadline": "Merge pull request #752 from EleutherAI/babi-refactor", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 21, "deletions": 1}}, {"node": {"oid": "ce1389c90e309ec144a927ca13e419461a22cfc8", "messageHeadline": "doc_to_target set to just label", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 7}}, {"node": {"oid": "dd8bbe2c050e7e6e07f1d2f88b21964098a72b6f", "messageHeadline": "pre-commit format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 35, "deletions": 34}}, {"node": {"oid": "9b00813fc0cf289feaea8f8bbc5e49c71bd947e1", "messageHeadline": "fix choices", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "620d8a362291a24244d38962c5c9805e3439af8c", "messageHeadline": "add draft abstract algebra task, fix genning script", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 6}}, {"node": {"oid": "25a4e2c61ff6b29a5466daf3d987e038168de2db", "messageHeadline": "add yaml-genning script + WIP base yaml", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 126, "deletions": 0}}, {"node": {"oid": "f33d37582af1ab68c1eb32a6788c6897b4fb418e", "messageHeadline": "fix error in a logging msg", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "87323176b00a4077919071e50d1b649cedb5761b", "messageHeadline": "add babi task", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 20, "deletions": 0}}, {"node": {"oid": "3c8610c5899c6e7d82a6683117a59cd8d15ea243", "messageHeadline": "add documentation on tasks", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 99, "deletions": 1}}, {"node": {"oid": "d0d86b1ca6b828c6f35be842443e6f9e61ba979e", "messageHeadline": "add all crows pairs variants", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 127, "deletions": 0}}, {"node": {"oid": "0247833d86808edf6807a7fb285eccb2ef954fb2", "messageHeadline": "add french dataset task", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "9f67a436080fce8b9d07a27fd88dadd28452539d", "messageHeadline": "cleanup yaml", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "b39ff1065c989faf0948cee1dd9b89f1c153a617", "messageHeadline": "crows pairs english works", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 39, "deletions": 0}}, {"node": {"oid": "cf1370d127812eace470ceb871a48741a795b15a", "messageHeadline": "ensure multiple_choice task type respects target_delimiter", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "1a2cf3b2dc39664c7076895f72788d4be56fabed", "messageHeadline": "add xcopa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 190, "deletions": 0}}, {"node": {"oid": "ea41c5bf811c3fac4eb6d12aac79679d2fae77e3", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 35, "deletions": 25}}, {"node": {"oid": "4c1c32e349844ecfecd7b93f125c5e6704112593", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 16, "deletions": 0}}, {"node": {"oid": "3bfbddc41dc64c15a6084945a69bf5cba8d76d76", "messageHeadline": "renamed", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "7f69c48b146fc56615e4982eb4953dbc3c5fa41a", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "e4ceb227b872dc28b3ff22567a45c4c04a4c953e", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "07bd8c032422e62086581d43cadafeb7239e0e4b", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a9de5550a7331f67cfcea11b9991cd720b729c59", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c01463020dd15b52cf8aaef3c122a8adce347fe4", "messageHeadline": "update on jinja explanation", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 7}}, {"node": {"oid": "005edfbf327116555aa4ff0d3e26c18e201f428f", "messageHeadline": "added explanation for basic prompts", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 25, "deletions": 7}}, {"node": {"oid": "1710b42d52d0f327cb0eb3cb1bfbbeca992836ca", "messageHeadline": "Merge pull request #742 from baberabb/big-refactor_typehints", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 29, "deletions": 30}}, {"node": {"oid": "e1fdf2a88cec2ec37539e52d4d6abc46a7938554", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 17}}, {"node": {"oid": "0e26c0bd9c152cf51aa9a89812f584e781d23244", "messageHeadline": "update to match t5 prompt template", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 10}}, {"node": {"oid": "97f5c0206ee61d4367662e4dc88fe0f6f5758548", "messageHeadline": "added typehints", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 29, "deletions": 30}}, {"node": {"oid": "b0f67f2c67f60f445efd731e32275202e25e7fb7", "messageHeadline": "Merge pull request #736 from baberabb/big-refactor_opai", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 158, "deletions": 76}}, {"node": {"oid": "ca3863928750493e4923705159740349433e96c9", "messageHeadline": "added typehints to readme", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 7, "deletions": 3}}, {"node": {"oid": "a5d57f15577b357459eb98ac2be485e17daa3f2f", "messageHeadline": "added tiktoken and changed openai dependency to optional", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 22, "deletions": 25}}, {"node": {"oid": "8ad386eb67310ee3b5ef7ad0eea147e9781b1d4a", "messageHeadline": "added logliklihood_rolling and fixed greedy_until", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 37, "deletions": 9}}, {"node": {"oid": "d46d792de661fdbff0f221ba6322cd8e698694a0", "messageHeadline": "add :", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "bee52bf2578a5cad020642917f688cd1cf7f05d3", "messageHeadline": "add :", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "81775d0ea621b4970f1612fa17ca9da15c904dfa", "messageHeadline": "1st pass through", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 14, "deletions": 6}}, {"node": {"oid": "9c74820471735651e07704c4ca89be33f5603507", "messageHeadline": "making t5 version of superglue prompt", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 82, "deletions": 6}}, {"node": {"oid": "6c753760dc5e550f5fc250205794105420cf6a97", "messageHeadline": "Merge pull request #739 from yeoedward/fix_packaging", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f70cbed929bd177d90054c0b1ca7e72fab3d0647", "messageHeadline": "Fix task packaging", "author": {"name": "Eddy Yeo", "email": "eddy.yeo@gmail.com", "user": {"login": "yeoedward"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "331340ad9d28ac21d674a526296ec090d80b4d17", "messageHeadline": "added typehints and updated docs", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 20, "deletions": 19}}, {"node": {"oid": "7aff9b75c48b1e8d9623e03f0f80602049c9de11", "messageHeadline": "Update error message if task name misspelt", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "718eaec838a3630e5cc0a567a110f53ad15a7654", "messageHeadline": "added methods to openai", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 46, "deletions": 11}}, {"node": {"oid": "eabadf46bf3064abcc286dcb78ee97f281f0049d", "messageHeadline": "added type hints", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 31, "deletions": 16}}, {"node": {"oid": "e9b938f2e009a420ee739cac67d7c2f6dc657453", "messageHeadline": "Merge pull request #711 from baberabb/big-refactor-logiqav2", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 178, "deletions": 8}}, {"node": {"oid": "321aef35ef23dbd6dfe847ca17234964103f7020", "messageHeadline": "Merge pull request #734 from EleutherAI/fix-table", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 5}}, {"node": {"oid": "c098188d1a25b657287ee62d5b774a36a183672c", "messageHeadline": "Merge branch 'big-refactor' into fix-table", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 58, "deletions": 23}}, {"node": {"oid": "f55250abda17defa8029983f7315d7a6542728e0", "messageHeadline": "Merge branch 'big-refactor' into fix-table", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 116, "deletions": 13}}, {"node": {"oid": "0a0f879c12dec5f3a3968111f042a8e7020c3c00", "messageHeadline": "Merge pull request #724 from EleutherAI/Refactor]-fix-anthropic-impor\u2026", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 58, "deletions": 23}}, {"node": {"oid": "e49972b0dd0aee4edfb6a5ce41ab21671c18e162", "messageHeadline": "make AnthropicLM class still registered when missing anthropic extra", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 21, "deletions": 9}}, {"node": {"oid": "07f94446beb1d9e36f3b0c0e54ddcbd85119c66b", "messageHeadline": "remove num_fewshot in table that was supposed to be removed in benchm\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 5}}, {"node": {"oid": "fa2d592af7551e8a081d032844502a5b7dd34a08", "messageHeadline": "Update evaluator.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "d0add6e5c441eef3e5287edb8a39acb3f7199d75", "messageHeadline": "Merge branch 'Refactor]-fix-anthropic-import-bug' of https://github.c\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2f4124fae397104476b0902538f238e4a38aa53a", "messageHeadline": "pre-commit format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 29, "deletions": 16}}, {"node": {"oid": "804c6ffefb49340fcd662e37bb60907ca62cf876", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 123, "deletions": 15}}, {"node": {"oid": "aa1cf230b800e250b55d738030befa7ba3e7e8b8", "messageHeadline": "Update setup.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b3a8c985cd32cb594ffba6ce8f4b3097162b5b3d", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 3}}, {"node": {"oid": "881fcf15a2528a5293cb7e373c5901007674317c", "messageHeadline": "Update __init__.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ca36c41cd2f7f8df8045efe3d27a53ea5d2abd6d", "messageHeadline": "Merge pull request #673 from fattorib/big-refactor-autobatching", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 111, "deletions": 11}}, {"node": {"oid": "1ef21c7bc4d1e94f672ee967f70676a1c5308352", "messageHeadline": "Merge pull request #732 from EleutherAI/StellaAthena-patch-1-1", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "752b08f93c044faeeb295167c1a59a0e2464a0eb", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "52213e29b0e7c48a89df5779f648838a6fae7ec2", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 28, "deletions": 12}}, {"node": {"oid": "5cf2a063957dcf7f7a48058697fa4c419fdf9386", "messageHeadline": "pull and modified input args to metric", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "920e1e732971183db2d312cb36808d9a37dafd0a", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "7010b2aae69256f8da113b404127496cc43e2a07", "messageHeadline": "Merge pull request #730 from EleutherAI/lintangsutawika-patch-1", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "16c4afc655d481af6e6a1079b3b5308e6f3bbce8", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 692, "deletions": 21351}}, {"node": {"oid": "7b376ae1e32c3a0261a58c548d8fd072e3b89b18", "messageHeadline": "Removed Implementation from metrics.py", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 0, "deletions": 44}}, {"node": {"oid": "939404a772a6bbb19f082125602b8ce9560fbc08", "messageHeadline": "Merge branch 'RealToxicityPrompts' of https://github.com/aflah02/lm-e\u2026", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 0, "deletions": 4}}, {"node": {"oid": "917b07d118df1a4f6340891eb1405a9852c84690", "messageHeadline": "Addressed Review Comments", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 46, "deletions": 1}}, {"node": {"oid": "45dd6275b3bfa0e4922548120d15e3e4e2758e89", "messageHeadline": "Delete realtoxicityprompts_yaml", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 0, "deletions": 4}}, {"node": {"oid": "39c5059e36c9f3408065a28d2e794ae200bde656", "messageHeadline": "Fixed Wrong File Push", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "f40c434e72fd0aa7dd4b2ad62271f3bbac215f17", "messageHeadline": "Update huggingface.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "176d5a2664b07136d26f114ec457a25a21cc1dee", "messageHeadline": "Merge pull request #728 from EleutherAI/StellaAthena-patch-1", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "54758343e8584451785da41a3e74141eba817e3a", "messageHeadline": "Add note about MPS", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "c42cb5624250b6f987e0a8d513fdfb5e9fa228ab", "messageHeadline": "Improved DocString", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "eaa1c76696505306157e4a4ff98fdb2ce1716714", "messageHeadline": "CMD Line Works", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 11, "deletions": 4}}, {"node": {"oid": "62b8a6ce98eac8fb43e7e2453ed9d2245c61c291", "messageHeadline": "Added Metric and Seems to Run but some Errors", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 15, "deletions": 5}}, {"node": {"oid": "6cb8169c222ba7d99a13a4dd4c5ce54217967cab", "messageHeadline": "Base Template", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 48, "deletions": 0}}, {"node": {"oid": "deecaa1a8ae2c2afddc168e24ae02ff3dbebb95b", "messageHeadline": "Merge pull request #723 from EleutherAI/fix_max_length", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "53105bba84e5a47e933a650553da3673d6f0b0b7", "messageHeadline": "Update task.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "5decd2cb90c1d378cf8f5a8e51cc55fb9ca5d9c8", "messageHeadline": "Update __init__.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e5bb2d57b11c9be0353bfb6c2753820e48fbd8bd", "messageHeadline": "Update __init__.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 1}}, {"node": {"oid": "ba864e0999620f00d35cc0f128c4af733a96c97a", "messageHeadline": "fix to add a case for if a user add `max_length` to generation_kwargs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "8591f0895e83d447288ed498c714eed3fe0653e9", "messageHeadline": "added logiqa, logiqa2 and loqieval tasks; fixed tests for greedy_until", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 57, "deletions": 241}}, {"node": {"oid": "d88a566ce154064fa4301961ed751eb37f687bb9", "messageHeadline": "Merge pull request #612 from EleutherAI/benchmark-scripts", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 487, "deletions": 142}}, {"node": {"oid": "29f12dd9e7923e3be3eada61d035e1a6799a2c12", "messageHeadline": "Merge branch 'big-refactor' into benchmark-scripts", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 220, "deletions": 21216}}, {"node": {"oid": "4168c05f5ef0a437d8d88696846a42f28c27fdea", "messageHeadline": "Merge pull request #720 from EleutherAI/lintangsutawika-patch-1", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 11, "deletions": 12}}, {"node": {"oid": "aa8482275e356ff4a42c9a5fbbd18a234d278116", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "8ebc85e8d8aa155b87bd7f31b7c6e74faa616b51", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 10}}, {"node": {"oid": "546fd5cd5b2bff5a61bdebba2502a0bd7a1c7028", "messageHeadline": "Merge pull request #686 from EleutherAI/cleanup", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 107, "deletions": 21156}}, {"node": {"oid": "e37698df0459be1e9ef1dad5e02a2ae0f4b34ce8", "messageHeadline": "update on metrics and delet files", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 33, "deletions": 197916}}, {"node": {"oid": "1bc408ff97ca7ed526af2accf9e7ddc9e3674f46", "messageHeadline": "added rte", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 13, "deletions": 0}}, {"node": {"oid": "223da3917906f871587c68ba966ff8bb8296a62b", "messageHeadline": "fix configs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 44, "deletions": 4}}, {"node": {"oid": "1813bf0466df998280084c649d5113aed4479b1d", "messageHeadline": "skips metrics prep if process_result is not None", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 16, "deletions": 5}}, {"node": {"oid": "0eb94c8b99ab563cc74184353657d630b830e4b4", "messageHeadline": "process aggregate fix", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 4}}, {"node": {"oid": "b2598de8b7b7abd72761397ec0ef9802ac8f578c", "messageHeadline": "process int label to string for greedy_until", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "8da6033fb233914a1e700c7d06cb1238ca30eac4", "messageHeadline": "remove comments", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "540d468e9cc7bcee79b00e6a1c05c86b7903c0f2", "messageHeadline": "fix Pile dataset name", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "16bc6bc0baa9d3827f6020e48ab7df09141d7e26", "messageHeadline": "merge conflicts", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 656, "deletions": 138}}, {"node": {"oid": "c8b434dba43cc6e6d20da6569139d63e3cfa0f55", "messageHeadline": "changed dataset_path", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "465c695b9cc7f6731187e925f8f04b3939263b18", "messageHeadline": "Merge pull request #710 from baberabb/big-refactor_claude", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 81, "deletions": 40}}, {"node": {"oid": "471297ba6424a19718aefced70f87217653c0505", "messageHeadline": "fixed generation_kwargs; added dependency groups to testing on CI", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 11, "deletions": 8}}, {"node": {"oid": "6efc8d5ec1784ebc1cc8c4558ad61184c1799645", "messageHeadline": "Merge pull request #702 from EleutherAI/num_fewshot-bug", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 31, "deletions": 9}}, {"node": {"oid": "c79d71fd3b55bc2ff1850d505a94844f26124265", "messageHeadline": "remove temperature", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b0cbf5d7bd7470d8de701073e9944db9c50659e7", "messageHeadline": "added regex", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 12, "deletions": 0}}, {"node": {"oid": "19f6d1bdb4ebaaf19d73b82bc6f7b47ee949a22d", "messageHeadline": "add logieval", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 52, "deletions": 12}}, {"node": {"oid": "66777b63fc4e78f80df15fe4487ebf57c03dda75", "messageHeadline": "added logiqa2 and loading script", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 302, "deletions": 0}}, {"node": {"oid": "b8510001964b18116e43f15bb595fc81f4ba8c3a", "messageHeadline": "fixed error handling", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 22, "deletions": 21}}, {"node": {"oid": "3a3655d6435a766888e3fd1fbcd1dc3260fb3d37", "messageHeadline": "passed kwargs to client", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 13, "deletions": 3}}, {"node": {"oid": "fe358061a870d99814b7757a3429e3d6f2742dc5", "messageHeadline": "added kwargs to client", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "8ffa0e67e1308f3c5710590e7d0770e444c378d5", "messageHeadline": "updated anthropic to new API", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 51, "deletions": 26}}, {"node": {"oid": "e02811269cbf1b8cd8ae63b912438d0e9db2542e", "messageHeadline": "Merge remote-tracking branch 'upstream/big-refactor' into big-refacto\u2026", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 443, "deletions": 111}}, {"node": {"oid": "96ea9f54058550e019057d50d870fd704454f24a", "messageHeadline": "skip recomputing batch size if it is maximal", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "7d8f54694d8bd5e7b4f64efebe0871c59a3dcbed", "messageHeadline": "fix on output_path_file", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "3168fc0035f4f0e6eff9f235d3dc0b29690033d8", "messageHeadline": "autobatching support for enc-dec", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 15, "deletions": 3}}, {"node": {"oid": "b7cd829b3c542707768e79c231a0368222e28289", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 859, "deletions": 193}}, {"node": {"oid": "81a11d6dbe3f86c6215eb58cc1474b22c955f262", "messageHeadline": "materialze \"out\" tensor during batch calculation", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4e44f0aac16c56adb127620da6b961c15e2a0ed0", "messageHeadline": "Merge pull request #703 from EleutherAI/use_console_script", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "6b1897d6752f3926a91d0af07deb84258030b26d", "messageHeadline": "Merge pull request #690 from EleutherAI/lintangsutawika-patch-1", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 4}}, {"node": {"oid": "05c879ccb01ea33a03e8099d3a9d04d8791bdf05", "messageHeadline": "Merge pull request #700 from ZZR0/greedy_until--patch-1", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8e8212fc8316e9f924ba5fdc334ac1d947e27f92", "messageHeadline": "Merge pull request #695 from yeoedward/xwinograd", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 194, "deletions": 0}}, {"node": {"oid": "e0b3cbf51ddba8bf42b89af1eb4b8e1bf222f3e8", "messageHeadline": "__setitem__ allows setting key-value like dict", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f148c2e2adfc75c9fb956be239ab515517b41389", "messageHeadline": "reorder", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0b8446a4417132d2973fd180c9c4a5061ea23c1a", "messageHeadline": "Reorder", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "d553e0609040e31ea60c592b9c65d2f9bc61fa8e", "messageHeadline": "Merge pull request #693 from baberabb/big-refactor_fixfin", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 244, "deletions": 106}}, {"node": {"oid": "20d7006787d1a05fd8c0826596f562322cd3c2a5", "messageHeadline": "added `lm-eval` and `lm_eval` as command to use instead of `python ma\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "0cd992b11c9d984a910ab049928630d8bc69d39d", "messageHeadline": "Update evaluator.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "8add5ed60e8b9496b4dacef0f5cdc30a9e63ff74", "messageHeadline": "Use num_fewshot set in yaml and show warning if it's being overwritte\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 32, "deletions": 9}}, {"node": {"oid": "2d96a8c85dcc4009155a350b3a1393c60a8f5714", "messageHeadline": "add condition if --task is not a benchmark", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 10}}, {"node": {"oid": "ed304c1d9747251efda562c564b2fc2c0907e96a", "messageHeadline": "removed promptsource yaml file", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 63, "deletions": 151}}, {"node": {"oid": "984f8793282577c7e6429edc34fbff26ef2c79fd", "messageHeadline": "Early stop bug of greedy_until (primary_until should be a list of str)", "author": {"name": "ZZR0", "email": "38292503+ZZR0@users.noreply.github.com", "user": {"login": "ZZR0"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0a7720e9f6d36f1636909336c45aa160dce43848", "messageHeadline": "multi-device: take minimum computed autobatch over all ranks", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 12, "deletions": 1}}, {"node": {"oid": "f4df3e48527d5a787a8db2809cc2d84e12d58cbc", "messageHeadline": "batch_size arg accepts str", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6a6a0ebbced10f690927dad0dbd8462991a62860", "messageHeadline": "Merge remote-tracking branch 'upstream/big-refactor' into big-refacto\u2026", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1246, "deletions": 576}}, {"node": {"oid": "4d8cee80fb18a4cf04428036634612018e143183", "messageHeadline": "added conditon to test_create_choices and minor chnages.", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 10, "deletions": 8}}, {"node": {"oid": "48d861ce71797f1dbba69de05e2e131b377f446c", "messageHeadline": "fixed fixture", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ba75c838f07b3cd109c5d9c232d2cc062b2b0c0c", "messageHeadline": "changed task_class fixture", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 4}}, {"node": {"oid": "3864b560f39b323790361e3fe094f63be4ee7983", "messageHeadline": "workflows on just big-refactor", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 9, "deletions": 1}}, {"node": {"oid": "3d0282cf7bc460b941fc1cb85d6abf9bb3c2c885", "messageHeadline": "new_tasks have own tests now", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 153, "deletions": 154}}, {"node": {"oid": "94563a365fbb7f9cb74e2ce0f30ad3897027a93f", "messageHeadline": "[Refactor] Migrate xwinograd tasks to yaml", "author": {"name": "Eddy Yeo", "email": "eddy.yeo@gmail.com", "user": {"login": "yeoedward"}}, "additions": 194, "deletions": 0}}, {"node": {"oid": "a199314a677c7d7a1fb31bd94b6828d45ad30782", "messageHeadline": "seperated tests for new tasks into own file.", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 185, "deletions": 55}}, {"node": {"oid": "f4cb06b1f963bfe85aacff2e62e8ce868e720389", "messageHeadline": "try to placate precommit", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 3}}, {"node": {"oid": "3d7f777df71d2198d8c54781759001e50f96d78e", "messageHeadline": "make process_docs boolean more clear", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "380c62f951da5a892b36de1b99d49a99bac80d89", "messageHeadline": "handle trust_remote_code models better", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 16, "deletions": 1}}, {"node": {"oid": "310250d385f44c8928b270aa208154c46081ad1c", "messageHeadline": "Update evaluator.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "73c04968373c4c5ad1353795c6965f45ea4c1fc2", "messageHeadline": "changed nameing", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "748a98989de4e61cbfd56eaed84920f33e8ce80e", "messageHeadline": "can now process a benchmark that uses promptsource", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 160, "deletions": 32}}, {"node": {"oid": "7411466c1f892557916bdbba77bf4cec33ba3782", "messageHeadline": "added way to search for all prompts", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 15, "deletions": 0}}, {"node": {"oid": "ae386303582e010e7c98665c225c542c72a03bce", "messageHeadline": "don't know why  is missing in the args", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 0}}, {"node": {"oid": "2820042d05e91c87852c82293f8973dc841c1a25", "messageHeadline": "Merge pull request #689 from yeoedward/webqs", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 102, "deletions": 5}}, {"node": {"oid": "1e2fbe6b5e2ca86f165b2f092e68c4d542b51e75", "messageHeadline": "attempt to fix multiple tasks modified case", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "d23321491b5ccbdf500d445f9824837b5a6d50af", "messageHeadline": "[Refactor] Migrate webqs task to yaml", "author": {"name": "Eddy Yeo", "email": "eddy.yeo@gmail.com", "user": {"login": "yeoedward"}}, "additions": 102, "deletions": 5}}, {"node": {"oid": "020ff063916d059c7c85d7aeb952afb4365fbc6b", "messageHeadline": "Merge branch 'big-refactor' into cleanup", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 421, "deletions": 184}}, {"node": {"oid": "dad7e9773322399a2e31a7b8ee25ea3e5086d7ac", "messageHeadline": "improve docs on doc_to_choice", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 36, "deletions": 9}}, {"node": {"oid": "fbc26541a00dada0d550361c165cf4b06937f7b2", "messageHeadline": "remove template_aliases fully from taskconfig", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 16}}, {"node": {"oid": "98c5411cd0dafc45f1e7576bb6e95c1661315ca2", "messageHeadline": "remove template_aliases from existing tasks", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 36, "deletions": 149}}, {"node": {"oid": "2bd45fd92b4baa0af928262f95981004c27f02db", "messageHeadline": "remove winograd_schema task type fully", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 5}}, {"node": {"oid": "80358e5a5e1f317dd9eaf2b15882fe7cc8cbb066", "messageHeadline": "add assert for passing generation_kwargs w/ output_type neq greedy_until", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "21882abe0d9fcfa2c9a62249a31fd1c247c20f58", "messageHeadline": "Merge pull request #688 from yeoedward/fix_dummy", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "9bd6b0be5c265be5d9aa7facf9de56cf2900b8ac", "messageHeadline": "Fix dummy model", "author": {"name": "Eddy Yeo", "email": "eddy.yeo@gmail.com", "user": {"login": "yeoedward"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a8bdf20612cdbec50d33f0267f53652884169a96", "messageHeadline": "Merge pull request #675 from baberabb/big-refactor_testeval", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 419, "deletions": 182}}, {"node": {"oid": "50e1ed408e1fdf5d0cf02849ccdcbb569e774644", "messageHeadline": "Merge pull request #687 from baberabb/big-refactor_output", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "9e209949fa771e3f8fe4ac11ba8ac65f31680e6b", "messageHeadline": "Update main.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1d7de830ce0bad3deae91e1b5196f8b4be6be2d1", "messageHeadline": "typo: results is .json", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "62bac284181cae3ca891a5aee02a1b0d4c44f044", "messageHeadline": "update format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 16, "deletions": 3}}, {"node": {"oid": "5418acaf880475b586b34f720247bfe08cbcc44d", "messageHeadline": "Merge pull request #685 from EleutherAI/add-comments", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 1}}, {"node": {"oid": "7520ea80fc7ff24eb9b33562caa1bb45ba7ecef3", "messageHeadline": "Merge branch 'benchmark-scripts' into add-comments", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 82, "deletions": 26}}, {"node": {"oid": "1f21b8ef5de18df0de2368a3e3b81c0d611d1825", "messageHeadline": "Update evaluator.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b85ec188a34ba02356cc181d76355e7d61d048c7", "messageHeadline": "Update evaluator.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "3ab0aa5ce7a97d95c923cf0c09717868c3f6eb5d", "messageHeadline": "Update evaluator.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "7f6544399c024afdb59a7cf8a368aa944dfb1629", "messageHeadline": "Merge branch 'big-refactor' into benchmark-scripts", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 78, "deletions": 25}}, {"node": {"oid": "db9231e242f9daf2aa42ae41c8c90c2b98c7051f", "messageHeadline": "Update evaluator.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 5}}, {"node": {"oid": "705e57c76ebc12c2a4933b876df9470228df62df", "messageHeadline": "fix new_tasks function", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8c0db2ac8f3e3427fc607d503db478721f01cc73", "messageHeadline": "fix typos", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "80b84321aed97c950d0af5061ec706f0d41653f4", "messageHeadline": "add CI test case if api/* files changed then pytest more tasks", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 16, "deletions": 8}}, {"node": {"oid": "fc63c7afb85db9c66a5050cc0afee8cdee89dd83", "messageHeadline": "edge case for P-Tuning in causal LM", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "3362727de7218345bf05023c96098ad919766e4d", "messageHeadline": "remove results folder for now", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 20987}}, {"node": {"oid": "8e3f1a4990df6e8666736a2884eeb6485059a264", "messageHeadline": "Update and rename __innit__.py to __init__.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "777f57ce9cd61efec6da01deac95835c5278fd17", "messageHeadline": "add comments to all vars defined in evaluate()", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 10, "deletions": 5}}, {"node": {"oid": "3cd3921f7dfc74eaaaf93f846642438d5dbcd34b", "messageHeadline": "Merge pull request #684 from baberabb/big-refactor_output", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 72, "deletions": 23}}, {"node": {"oid": "decb55cd5d6f52bcfa2d6e3a70d7bef199e1f490", "messageHeadline": "removed stale args. added assertion if log_samples true and no output\u2026", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "9a4e7e7f09dfd2ca6538d6e2af6d7310e521ff9f", "messageHeadline": "Merge branch 'benchmark-scripts' of https://github.com/EleutherAI/lm-\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 121, "deletions": 8}}, {"node": {"oid": "18a7c8b18569d9cf686f31cfff3ccbb23c759238", "messageHeadline": "attach group identifier to task for aggregation", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 30, "deletions": 7}}, {"node": {"oid": "ef7588b617df3b861b687ab6aefc95fb4e0c5e1e", "messageHeadline": "add column as option", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "1f6d3cd6dfd1c8473018fdf496bface746f497d4", "messageHeadline": "print two tables", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "74ba283d939513e1b50521fd6818e40004da4655", "messageHeadline": "edited output_path and added help to args", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 72, "deletions": 23}}, {"node": {"oid": "027fe14cd4a94537fb682a893d6bcbc9aeee7807", "messageHeadline": "Merge branch 'big-refactor' into benchmark-scripts", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 121, "deletions": 8}}, {"node": {"oid": "32a70d893692ca24b5cd4bef0206889be0572711", "messageHeadline": "aggregate is shown in the table", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 11, "deletions": 1}}, {"node": {"oid": "1d995b6d944a2f8afb685b2ded9fbae1746febfa", "messageHeadline": "modified to arc_*", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "5e2f0ffe66e15e23b85cd4670f146b6678ccaa4c", "messageHeadline": "printing all the configs are made optional with --show_config", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "4dfa8abab4fde429bb1189ade7870a7c14b9ba1a", "messageHeadline": "Merge pull request #682 from yeoedward/anli", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 133, "deletions": 0}}, {"node": {"oid": "97a172aea41d1eca45e8dc6e24e629c6b0ccb53e", "messageHeadline": "Update anli_r3.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "cd7f642dd01e2d49537aad69f5e2f5056c3e0d57", "messageHeadline": "Update anli_r2.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a7c1a639639215b4477e67b53bad9bb4ba167b3b", "messageHeadline": "Update anli_r1.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b783c68e8fc7172a97193782acdb4263a98c4e4b", "messageHeadline": "[Refactor] Migrate anli tasks to yaml", "author": {"name": "Eddy Yeo", "email": "eddy.yeo@gmail.com", "user": {"login": "yeoedward"}}, "additions": 133, "deletions": 0}}, {"node": {"oid": "7aa13c479109c4d5de3191e378dad5b6cef74c04", "messageHeadline": "Merge pull request #680 from baberabb/big-refactor_fixup", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 2}}, {"node": {"oid": "3001569be2fbc7d3f0da3eae9356ee893b029f7c", "messageHeadline": "prototype for aggregate", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 4}}, {"node": {"oid": "53754d41a13390d474f2b996fef38c6c130238ad", "messageHeadline": "Merge pull request #679 from EleutherAI/fix-padding-ranks", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 4}}, {"node": {"oid": "fa660ad8cabd35effafa121639982f40664cc71e", "messageHeadline": "Delete test file", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "4c25086097ba24a23a1639364d5d5f6db9b6e1fe", "messageHeadline": "edited test_tasks.py and workflow files", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 198, "deletions": 118}}, {"node": {"oid": "43407f36ba7eb89abd235660291a3d80919ae667", "messageHeadline": "set fp32 if device=mps", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ce214979a5729aa39f2e6e757656bbdbaea3b5b3", "messageHeadline": "set fp32 if device=mps", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "2bb7ce3bbc144d3572f6094c559d0fd0ecff5d98", "messageHeadline": "add mps", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "73b2149fa1d1d5d77a76ec554307b090cb613bad", "messageHeadline": "removed rnd from task.fewshot_context in write_out.py; add mps in hug\u2026", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "31995161845e6352d2ef903fdcfe8fef076759ab", "messageHeadline": "Merge remote-tracking branch 'origin/big-refactor' into big-refactor_\u2026", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 360, "deletions": 302}}, {"node": {"oid": "1c27e3713eb2f039a4f12f2f551eb7c9a12aa015", "messageHeadline": "typo", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "58aa729f9756adf01447534e476e4b69339d806f", "messageHeadline": "Merge remote-tracking branch 'origin/big-refactor_testtasks' into big\u2026", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 827, "deletions": 402}}, {"node": {"oid": "fae09c2c4da810ab5bd5cdeb88b95ea2f819b0c7", "messageHeadline": "updated test_tasks", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 161, "deletions": 77}}, {"node": {"oid": "abccc7568ad2d18e4a7153b19fcb932b8abd0c99", "messageHeadline": "padding now handled correctly per-reqtype", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 4}}, {"node": {"oid": "e5161a6d43c4de0dc24e195e3a60bd744a5477d1", "messageHeadline": "changes", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 33, "deletions": 53}}, {"node": {"oid": "1ec0a12909bc3c1894ee296d26e61f1cae174689", "messageHeadline": "fix", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "57f08e40a046ed293c1b50c0d2a81d1bce20a04b", "messageHeadline": "removed json files", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 9205}}, {"node": {"oid": "708bc65f81606ae2625b560a613ee5d2d07f1614", "messageHeadline": "fix", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "34fc9e68c993452869e2c450d2b434e66f00c087", "messageHeadline": "add winogrande", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 31, "deletions": 0}}, {"node": {"oid": "241aa8715135aa32a3c58e5cfb6affe788ad61fb", "messageHeadline": "add where to put aggregation", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "89c4ee5c620b263070a0a03d9ac672bcaf089ec4", "messageHeadline": "added include_benchmarks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 25, "deletions": 3}}, {"node": {"oid": "1d3a16018e7f367dc151815665a5530bfac17535", "messageHeadline": "added benchmark configs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 38, "deletions": 16}}, {"node": {"oid": "c70383d5462ca8061883ef742e147f11eb306417", "messageHeadline": "simplify", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 13}}, {"node": {"oid": "4a1da52b98c6e8b59c351c3a4806a030b11be357", "messageHeadline": "add ouput/ to ignore", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "15c186b7ef9e3617dfcc1b8adf0b55a00c907a05", "messageHeadline": "change variable name", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 7}}, {"node": {"oid": "e5306ea6981acf285ec9e734ae3e3de208a8b017", "messageHeadline": "added script to build benchmarks from promptsouce", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 88, "deletions": 40}}, {"node": {"oid": "3713ec5291caae101be27eb3f8f6c20a56f07cb2", "messageHeadline": "update on how yaml file is made", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 40, "deletions": 0}}, {"node": {"oid": "ee098a47dfbc1558e92cd090ef2fe53997602133", "messageHeadline": "more tasks", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 207123, "deletions": 0}}, {"node": {"oid": "068f8ab28384fc47788636f257369a0966fd7711", "messageHeadline": "can include paths in main", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 9, "deletions": 2}}, {"node": {"oid": "7e565f33c87ef72971977e3cb25ec2f54f24a0f1", "messageHeadline": "changed name of boolq and cb config task", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "d4ef6e5b8345b6ff120a87b6cbe312672e96cc3b", "messageHeadline": "fix", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "82c12e9bbfb08cc356fc6ce393c501bcd0326a53", "messageHeadline": "fix", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "b182d1b4fcffe59dc0db670da220a799a7da33d6", "messageHeadline": "fix", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "10103d5de3177b6a59917c73916a47a793fcb28b", "messageHeadline": "unstop flake8", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "455c52c922a978619ea91e675999380e5bc1ff27", "messageHeadline": "fix typo", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "94cba2874bb35ea65a5e58ba21e5eeb7b918f910", "messageHeadline": "Add CI setup", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 82, "deletions": 60}}, {"node": {"oid": "283cad707d98fa605d3c2f58f03e152a9dac7f1c", "messageHeadline": "Update main.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2c20cd1f358158ef3b0379f3efd2be5fc723bf1c", "messageHeadline": "Merge pull request #671 from EleutherAI/revamp-process", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 359, "deletions": 301}}, {"node": {"oid": "0dadc92ab1695ce907f4575d99338fd30e1a93ec", "messageHeadline": "Merge branch 'big-refactor' into revamp-process", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 68, "deletions": 51}}, {"node": {"oid": "bfa57719d8fc6ae29cf4a175a974fc36ab3131e4", "messageHeadline": "Merge pull request #678 from EleutherAI/fix-generative-multitarget", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 29, "deletions": 6}}, {"node": {"oid": "d70f713430ad1f843aa83b7121096fd832e49ee8", "messageHeadline": "make sure boolq works", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 29, "deletions": 6}}, {"node": {"oid": "a36348faf5ac0a04f5f3a0ecd0e2c00414c50d0e", "messageHeadline": "override target_delimiter", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "addada53d742997c27b4b7182fb1c6461071c34e", "messageHeadline": "fixed whitespace for target", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "60acbde21059ce4294e17240c3b15e3e838ee7fa", "messageHeadline": "fix error", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1ca2c0ba78af9da4975ac05ead7a28cded84bde0", "messageHeadline": "remove info", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 3}}, {"node": {"oid": "b0597a94ec5373d7563ffdbc0aba6a0e3ee4de8b", "messageHeadline": "adjustments for super_glue", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 4}}, {"node": {"oid": "0a48d7c5597bb2b3716d750b09fa3dff61dcc374", "messageHeadline": "Update python-package.yml", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 6, "deletions": 3}}, {"node": {"oid": "6862fa7d5dfdbea0b363e3b368b4c6bf9101fd23", "messageHeadline": "Merge pull request #676 from EleutherAI/new-flags", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 90, "deletions": 65}}, {"node": {"oid": "f7dde0c3ad239b3d4538aa92000686c2ec4dfaf2", "messageHeadline": "formatted", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 18, "deletions": 13}}, {"node": {"oid": "7684bfa96a09c0311e203fcd45a0b3b217a9491d", "messageHeadline": "fix typehints in Instance()", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 7}}, {"node": {"oid": "04c81e7d26cd486b1190e6b8cfe8a940a5feb433", "messageHeadline": "don't log samples if not requested", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "5fbc3f86c1768d8b9d849a2aa914d2be453208fe", "messageHeadline": "fix random seed issue, log_samples optional", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 55, "deletions": 43}}, {"node": {"oid": "64c15e41ef6718ef65f886b6a9aa537f74a0e21b", "messageHeadline": "fix", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "9c26a957633410f4215fbede2806ddd2de09d479", "messageHeadline": "Create python-package.yml", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 40, "deletions": 0}}, {"node": {"oid": "e7047626626aa087da51d6b017ad98a117531131", "messageHeadline": "fix", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "2f386a9d4ecd21455b73091eb5f25b6ca908d9fd", "messageHeadline": "fix", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 6, "deletions": 13}}, {"node": {"oid": "a8b32e864421f79ecf061a9edd303053c4f3b70f", "messageHeadline": "Merge remote-tracking branch 'origin/big-refactor_testeval' into big-\u2026", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "196fbbb884d1c3a58621ee0f4f7724e46a25f029", "messageHeadline": "test pythia-70m and pytest only test_evaluator.py", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 61, "deletions": 49}}, {"node": {"oid": "4225df50bb45d187269f3e00eedecab36365cebe", "messageHeadline": "test pythia-70m and pytest only test_evaluator.py", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 59, "deletions": 47}}, {"node": {"oid": "ee633332cddc2069b06892b15948503c11cb8dd2", "messageHeadline": "updates, corrections, and fixes to match big-refactor", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 29, "deletions": 94}}, {"node": {"oid": "94a49f70b2b5d8fbbf7663b12e636f6b4ef8f086", "messageHeadline": "doc_to_target and doc_to_text should also accept int", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 39}}, {"node": {"oid": "248b45dace4eae0d521315e2fbcfcebf278f778a", "messageHeadline": "config fixes", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 4}}, {"node": {"oid": "70e3ed25f5033e4c97742154ada20d1ad09c3f17", "messageHeadline": "adjustments", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 11, "deletions": 3}}, {"node": {"oid": "457c4262185e5cdd8b7dca51f29458b9d8e3ed94", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 74, "deletions": 8}}, {"node": {"oid": "f4efb8e59e97a5493c472b1d42690eefc13cc52b", "messageHeadline": "fixes to boolq", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 4}}, {"node": {"oid": "ae41f67cc87633520ebb467202a33e951fdc0130", "messageHeadline": "fixes to bollq", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "09a71562848d7051937ffc1acca0e76de5b3b994", "messageHeadline": "log all arguments per doc", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "165f8493010ad928e4c8414a54abe6f796f37ac2", "messageHeadline": "add fast tokenizer flag", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "98c85d73100bce8d950708af0b638c133bc64162", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 7}}, {"node": {"oid": "ce5ec854bef0bef921dd11739b38f72801475fa7", "messageHeadline": "doc_to_choice can run when a promptsource template is defined", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 33, "deletions": 14}}, {"node": {"oid": "5288813a152bffb542f1a40a9e2a3932da9a48ec", "messageHeadline": "adjust wsc", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 19, "deletions": 2}}, {"node": {"oid": "82d57f655caa94d7175fa6ebda2fa6ddd8368bfa", "messageHeadline": "process condition for doc_to_text", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 1}}, {"node": {"oid": "24864d24586838597d3a972461eb7f33e7471e55", "messageHeadline": "Merge pull request #666 from nopperl/truthfulqa", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 72, "deletions": 1}}, {"node": {"oid": "a410f2bd214f8e3ddee9d498c62559205727421e", "messageHeadline": "Merge branch 'big-refactor' into truthfulqa", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 579, "deletions": 265}}, {"node": {"oid": "e9432fad9495e33e4a3dfae122f4b2ca5a5e7a44", "messageHeadline": "fixed doc_to_choice in race", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "9218bcff0a0dc94ec328e3476a3592042c33b545", "messageHeadline": "adjust headqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 3}}, {"node": {"oid": "f7a3cf20a24d053d198cd8e07e7a0db61f48fe20", "messageHeadline": "adjust mathqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "a83874e594a148f2ceb0c5fe9b3342985fff8064", "messageHeadline": "adjust mathqa", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "5dd41ebe7633ef65f76f8bde80999b7875c14ab2", "messageHeadline": "fixes and added decontaminate args", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "f2b52d2638c85ccceab5c693e6edc370bec59012", "messageHeadline": "doc_to_target should also be able to output str instead of solely int", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 1}}, {"node": {"oid": "e454df7fd3e242ec9ca44d174615b1ec3c287100", "messageHeadline": "Merge branch 'revamp-process' of https://github.com/EleutherAI/lm-eva\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 568, "deletions": 250}}, {"node": {"oid": "c827e4ce23e84c162019652cb6a7fa89815c7725", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 16, "deletions": 11}}, {"node": {"oid": "d1bb50e023c71d4aa7008e39d922df27807acb2c", "messageHeadline": "removed gold_alias for greedy_until", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 4}}, {"node": {"oid": "ca821452e989c20df9e72f6143fc4604b04bfb36", "messageHeadline": "Merge branch 'big-refactor' into revamp-process", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 568, "deletions": 250}}, {"node": {"oid": "9657b04c5c1c753aa6009534d553f85d14e07215", "messageHeadline": "minor change", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "5699c7abbdd6f4e2e8dad8420d98878ed88dc17e", "messageHeadline": "update winogrande format", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 8}}, {"node": {"oid": "b7449cd96d2678e9dafa133e26f670684afb2e80", "messageHeadline": "process focuses on doc_to_text, doc_to_target, and doc_to_choice", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 42, "deletions": 51}}, {"node": {"oid": "d9b547b77afe5ac1dda4c0af5d8fb89187250025", "messageHeadline": "fix test_evaluator.py", "author": {"name": "baberabb", "email": "92168766+baberabb@users.noreply.github.com", "user": {"login": "baberabb"}}, "additions": 15, "deletions": 17}}, {"node": {"oid": "e4acfcaa9645b1e82e2cb9221912c5ea2bdc3661", "messageHeadline": "remove TODO comment", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "74076cf9138c63cbebbb647ce8c4a184aedeaa2a", "messageHeadline": "Merge remote-tracking branch 'upstream/big-refactor' into big-refacto\u2026", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 502, "deletions": 252}}, {"node": {"oid": "5978dc4e88ce50faf4323c3c2826371c66ead57e", "messageHeadline": "superglue add yamls", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 62, "deletions": 23}}, {"node": {"oid": "027efa000209b18c2268ab6fa9f4019f73e3b5b5", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1610d548b9ec03162de384bf35cacf2a9e165523", "messageHeadline": "Merge pull request #660 from EleutherAI/unscramble+toxigen", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 501, "deletions": 251}}, {"node": {"oid": "e0417b3bea73c2fabb3cdc8cb1029ffd79be3512", "messageHeadline": "Merge branch 'big-refactor' into unscramble+toxigen", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 173, "deletions": 27}}, {"node": {"oid": "907c99c0abb682e3b0a19788602d8967c6240175", "messageHeadline": "Update evaluator.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "622bdda1f07118cf297d7370572e7ee4ddf5e71e", "messageHeadline": "updated yamls", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 17, "deletions": 24}}, {"node": {"oid": "499a4468eb57d15921213c02dab1e328c1f812d9", "messageHeadline": "added doc_to_choice", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "31c52f5528289e8c5f743c51afd1858bca1faf7e", "messageHeadline": "fixed arc", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 24}}, {"node": {"oid": "155160a1ca3c18c94e9b0cc76851cd54a7bccc36", "messageHeadline": "add ast.literal_eval to doc_to_choice", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 3}}, {"node": {"oid": "c37ad6ece0088f010ad1e46a98b7d14f02b74439", "messageHeadline": "process multiple_output", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 19, "deletions": 11}}, {"node": {"oid": "4c363ef833dbfa627df37446f0ed89e5307efd55", "messageHeadline": "multiple_output option for greedy_until", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "f09f3ab80a81744f180feda7d9defe8e23897ca4", "messageHeadline": "added error message", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "4054ab84662bbef87495874b99382d7c0716c101", "messageHeadline": "typo", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "fc5a114fd7dcc3f251a5847fbf5af8029ef78e2f", "messageHeadline": "modified yamls", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 8}}, {"node": {"oid": "9b20e2d0e48e1fb58857203f1c4c4a5c23087940", "messageHeadline": "update choice process", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 7}}, {"node": {"oid": "656c310e4b8545f0c4785dd8ee68288d2bb978b6", "messageHeadline": "process update to process doc_to_target variety", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 35, "deletions": 19}}, {"node": {"oid": "3b50b94158be6d6a23132946981e8addd3afd29e", "messageHeadline": "updated winogrande", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 28, "deletions": 45}}, {"node": {"oid": "36a14985b27286a1702809a1529a50e8df10d35d", "messageHeadline": "add section for autobatching to README", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 11, "deletions": 0}}, {"node": {"oid": "0fa9f03f4a1bb9914a9cc486680f0056c5300958", "messageHeadline": "pass override bs to _loglikelihood_tokens", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "d42e1706c5271087f380de6f46981caadf8baed6", "messageHeadline": "initial autobatching commit", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 73, "deletions": 10}}, {"node": {"oid": "7d4e92fa92c93bb5c31099ae7b5b10fa34a79f17", "messageHeadline": "Merge pull request #659 from EleutherAI/headqa", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 76, "deletions": 11}}, {"node": {"oid": "287f7efc15c1c2dcf6d560ef52b9b37c6ec0acf1", "messageHeadline": "Merge branch 'big-refactor' into headqa", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 545, "deletions": 38}}, {"node": {"oid": "f33c6c9bc4e1492c770166165f99e8096966546b", "messageHeadline": "Testing for multiple inputs from doc_to_text", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 28, "deletions": 19}}, {"node": {"oid": "552b09e028417494d8a55e7d998f900dc9e33d27", "messageHeadline": "template aliase process update.", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 3}}, {"node": {"oid": "9383f49e2704be9774d34dce486648f7ead16865", "messageHeadline": "Changes in winogrande yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "49a116eec389655f55781d5c16eaf4b1ab8d0bcd", "messageHeadline": "post_init to test if multple_choice has doc_to_text return list or str", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 23, "deletions": 7}}, {"node": {"oid": "003c2581b10fa34d3a0665dfa0a030ed137bf482", "messageHeadline": "multple_choice handles 2 cases", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 4}}, {"node": {"oid": "87d93e997ca231f34035812626c137a3393e1ac1", "messageHeadline": "use prediction and prediction index", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 6}}, {"node": {"oid": "3aeb95b609fff77d49f9100c2d4257e816e7d6f7", "messageHeadline": "Very rough draft and scribbles", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 32, "deletions": 12}}, {"node": {"oid": "a09e04bcd1a38f23156a75be54a97aabcc65cad9", "messageHeadline": "Update task checklist", "author": {"name": "nopperl", "email": "54780682+nopperl@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "5af4a8678cdcb5644105bf20bf0f5b64cc689d86", "messageHeadline": "Port truthfulqa mc1 task", "author": {"name": "nopperl", "email": "54780682+nopperl@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 69, "deletions": 0}}, {"node": {"oid": "76bf11c51697b5abe1a1e8c5d1736d77897095b2", "messageHeadline": "add MWP tag to mathQA", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "255f4c6ea6aa9df4784e5b260977c0d6ec295c70", "messageHeadline": "add mathqa", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 77, "deletions": 0}}, {"node": {"oid": "884ba785a4080d5f376e6be5159eef53eeaf4fab", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 4}}, {"node": {"oid": "aa60d2b6a32c2e88c538b7295fa4d5397a48404c", "messageHeadline": "add hendrycks ethics", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 165, "deletions": 0}}, {"node": {"oid": "6fc2e148b9db2131c0d0b205c3fd94353e450ff9", "messageHeadline": "revert change to template_aliases", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 10, "deletions": 10}}, {"node": {"oid": "71305a78caf12bd20ddc8163e43fde634663023d", "messageHeadline": "Merge pull request #661 from fattorib/race-update-hf", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "31250b07f7659d154da94a2cc3f31aa1bfc140cf", "messageHeadline": "move race on HF to EleutherAI group", "author": {"name": "ben", "email": "46356259+fattorib@users.noreply.github.com", "user": {"login": "fattorib"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "fdcdc494512cfe767bdc8f21f8c848c1b1e16130", "messageHeadline": "Update task statuses", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "94c465fde1e2def0882c2b4f7ee2e1c435e25388", "messageHeadline": "Merge pull request #657 from EleutherAI/Processing-`generation_kwargs`", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 23, "deletions": 7}}, {"node": {"oid": "557e6779aba3f1c1d466d06268a840e5e4fded77", "messageHeadline": "Merge pull request #651 from farzanehnakhaee70/add-qa4mre-config", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 66, "deletions": 0}}, {"node": {"oid": "80b429aacbe71ae375b477e2f6d0c4073308e49a", "messageHeadline": "Merge pull request #1 from EleutherAI/qa4mre", "author": {"name": "farzanehnakhaee70", "email": "30573681+farzanehnakhaee70@users.noreply.github.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 13, "deletions": 6}}, {"node": {"oid": "e8702f15f8fb1b60dcc9e821291efe47005f5807", "messageHeadline": "Merge branch 'big-refactor' into unscramble+toxigen", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 449, "deletions": 23}}, {"node": {"oid": "dcb162634c68ea186c035a15a6cf947ae569d105", "messageHeadline": "change Prost task name to match old ver.", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a4b0eafe38f310a26e62b5eb1aca8538dc4f635c", "messageHeadline": "fix bug in gsm8k", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "4e5a328e09609612dcee1696e78d0e4883a3ff9e", "messageHeadline": "bugfixes + add write_out", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 24, "deletions": 18}}, {"node": {"oid": "45737a385c5945aa7182266e1ccf9877d39b6db3", "messageHeadline": "Merge pull request #627 from fattorib/refactor-more-tasks", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 353, "deletions": 16}}, {"node": {"oid": "02362e6a2912e4083069df8078277ef91c24ada2", "messageHeadline": "Merge branch 'big-refactor' into refactor-more-tasks", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 218, "deletions": 134}}, {"node": {"oid": "7ffa1d1bf1ac8fdd4aea3566d79bba3643f8d8d3", "messageHeadline": "Change `target_delimiter` to `fewshot_delimiter`", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "0ba4ae157df91744e4bf10fe31d31200b37b706a", "messageHeadline": "Merge pull request #658 from EleutherAI/edge-case-lowbits", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 95, "deletions": 6}}, {"node": {"oid": "ff8903f25f56a0d96ea5c50bd68f01066b200fca", "messageHeadline": "remove stale templates", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 219}}, {"node": {"oid": "e071dd5eb318cd6a14f13065744cbb38b8ae4d88", "messageHeadline": "update task statuses", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "7a76696e3aad5bcbf51657fa4ece8c1315179dc6", "messageHeadline": "push unscramble", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 141, "deletions": 0}}, {"node": {"oid": "d93a96b4ddee38d8c79896189028f16ad644e122", "messageHeadline": "add toxigen task (confirmed working)", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 81, "deletions": 0}}, {"node": {"oid": "29940d164acfa9cf4674ecdfc13db0763dab8225", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "33add99973a94e5d62174a9fd803f8a3e4e9268c", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "3ab3c3881c283a85ad51bb37d9c8975928f994a1", "messageHeadline": "add headqa", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 75, "deletions": 1}}, {"node": {"oid": "80a0302a4b862e8ee4040ed23fe77762529120ce", "messageHeadline": "remove acc_mutual_info", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 9}}, {"node": {"oid": "b296ff8fb61e58f1b176773e5bb06f922932ea3f", "messageHeadline": "Merge pull request #650 from EleutherAI/lintangsutawika-patch-1", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "b569720553c93b92adeefec18e1243646fd8f68f", "messageHeadline": "check off Lambada Multilingual from task list", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "38bd2cc86bd38534b34dcd7a27cdab81a3325ab5", "messageHeadline": "remove batch_size param from config", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "797f47c78504d7a20ef70617e3f2890faba5b186", "messageHeadline": "Update __init__.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "23318d0e050a2787be376afce2e33cf93d567579", "messageHeadline": "remove unused line in evaluator.py", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "0e5da3b0d710969f45108e9d823069afde8c1882", "messageHeadline": "fix typo in copa group name", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "9735ac344f04024ac3d8b24cef77697bc3b6a963", "messageHeadline": "add lambada_mt tasks", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 92, "deletions": 0}}, {"node": {"oid": "59b047fb386ec3e65b701830b0a9afcdb105be33", "messageHeadline": "Update task.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "633fb45847ab6d91ddac5722039e56ae9127e554", "messageHeadline": "Update task.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 21, "deletions": 7}}, {"node": {"oid": "4f0a7e573acdf2cf5f0b000a762d5166ca258a9d", "messageHeadline": "Merge pull request #648 from EleutherAI/edge-case-lowbits", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 95, "deletions": 126}}, {"node": {"oid": "42c10bd67af329f70d8dbfa8e09fa59729a55734", "messageHeadline": "Update qa4mre status", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "7f557daa85d634a2c965388936fc40fc60f28774", "messageHeadline": "raise ValueError when invalid task passed", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c750840132436f69fe1ff2b8ffb92e90097997eb", "messageHeadline": "list out answer choices", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 13, "deletions": 6}}, {"node": {"oid": "5a75f82aaa64f5d99c40a0e0df7d076eac7c5139", "messageHeadline": "revert utils black", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 7, "deletions": 4}}, {"node": {"oid": "d75128dc4168776a9290f9b6977f320507b6d35f", "messageHeadline": "black update", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 5, "deletions": 8}}, {"node": {"oid": "dd584086b231939f80f39cf58667bf1d7f12519e", "messageHeadline": "resolve newline issue", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e04a803644906edae04df89ea950566ed9ccc04b", "messageHeadline": "Merge branch 'add-qa4mre-config' of https://github.com/farzanehnakhae\u2026", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 21, "deletions": 1}}, {"node": {"oid": "ddc26f349be724eeaacb62b525784f98e9d6e444", "messageHeadline": "resolve new line error", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "fe9af3fd4f85d1f81fd19dea46f9ed30dff85546", "messageHeadline": "Merge branch 'big-refactor' into add-qa4mre-config", "author": {"name": "farzanehnakhaee70", "email": "30573681+farzanehnakhaee70@users.noreply.github.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 21, "deletions": 1}}, {"node": {"oid": "b1b5239d6703287396da94d12191f8740013b70c", "messageHeadline": "address PR comments", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 4, "deletions": 24}}, {"node": {"oid": "66b91e025a08393d3e3cc58ef8b4eb63ba26a76a", "messageHeadline": "Merge pull request #640 from farzanehnakhaee70/add-prost-config", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 21, "deletions": 0}}, {"node": {"oid": "e6a8846cee1ed3b3602a6fc066234c142001d411", "messageHeadline": "Upload qa4mre_2013", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 19, "deletions": 0}}, {"node": {"oid": "09459176d35b7d1548cc7bd5136957d49374c736", "messageHeadline": "Upload qa4mre_2012", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 19, "deletions": 0}}, {"node": {"oid": "9907e0a74622d119a9e9bb064f3d0a1e056765de", "messageHeadline": "Merge branch 'big-refactor' into add-qa4mre-config", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 192, "deletions": 148}}, {"node": {"oid": "d289277d9b20883bda1b250d63a1d9f89b402936", "messageHeadline": "Update __init__.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "66bb89e56b31030ad19a55506ea795cffdceb2b7", "messageHeadline": "Merge branch 'big-refactor' into add-prost-config", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 1555, "deletions": 748}}, {"node": {"oid": "e8bb77db72f5ff92135b0748ef3957b181848eef", "messageHeadline": "Update doc to target and gold_alias in the config", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "6e3ef5ff34fe99c3d334c216846b219966d1aaac", "messageHeadline": "Merge remote-tracking branch 'upstream/big-refactor' into refactor-mo\u2026", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 315, "deletions": 227}}, {"node": {"oid": "ccae768a2b3f1e224bcc9a421789a9c1714a659f", "messageHeadline": "edit doc to text in yaml file", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "026d2c212cbc26d2c5f744d3c70145e11f2cdad8", "messageHeadline": "update task readme with completed race", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "81bdd5bde004e6dcd9c9c3b51779303358825a97", "messageHeadline": "acc_norm not needed for pubmed_qa", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 0, "deletions": 3}}, {"node": {"oid": "328f0e85151922aa19d12320447fd3c7a1268c7e", "messageHeadline": "cleanup metric loading code", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 44, "deletions": 42}}, {"node": {"oid": "2411c9d36fbc645c3df75505a83b7a9d6b82b59e", "messageHeadline": "update race preprocess + rename dataset", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 8, "deletions": 5}}, {"node": {"oid": "7feab5e35ed89904ec8580fe88aff89e460de878", "messageHeadline": "add race task, with custom HF dataset", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 47, "deletions": 0}}, {"node": {"oid": "6a2620ade383b8d30592fc2342eb1d213ad4b4cb", "messageHeadline": "remove output_base_path", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 9}}, {"node": {"oid": "070b6b9c1286d5a62e6f53fab285de46feefd46e", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "93b7a418616906ed1a4687260b5eb8ccddc85c9d", "messageHeadline": "Merge branch 'edge-case-lowbits' of https://github.com/EleutherAI/lm-\u2026", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "899a24c7867377c703c20934297a7020af27bced", "messageHeadline": "remove out-of-date description guide", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 49}}, {"node": {"oid": "97220c9b47dbcafc4761775aeb48a792e2ed3beb", "messageHeadline": "add up-to-date config fields to docs", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 29, "deletions": 16}}, {"node": {"oid": "0b62e042ab73a03836d5e31b7021d4a235b8b465", "messageHeadline": "Update task.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "445beadef7ae3ec7332130ef127db333b60bf1fc", "messageHeadline": "bump torch ver. 1.7->1.8", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "46ca66de127aad089f70537251dbc4d4f150a021", "messageHeadline": "fix squeeze() in pad_and_concat", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "0b896bd25a26af5f97affa9443febecf402db952", "messageHeadline": "don't place quantized models .to(device)", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 16, "deletions": 5}}, {"node": {"oid": "0d8e206c4c5b5af496c259088f754e0028d90bb0", "messageHeadline": "Merge pull request #647 from EleutherAI/handle-multigpu-errors", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 14, "deletions": 6}}, {"node": {"oid": "b35980581d433d6e7351c36b5268f3f74c5fecdc", "messageHeadline": "revert typehint", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "5cb7af7e961f0c46803b49eb6ba4d8da99cff0af", "messageHeadline": "revert gpus > num processes case", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 20, "deletions": 25}}, {"node": {"oid": "62f4d6636cce72fc7db792f894d619da7bdb2fec", "messageHeadline": "add typehint", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "b8154374e1ceaed3d2bce7c08efcb42dca0a2a17", "messageHeadline": "handle device assignment better", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 34, "deletions": 25}}, {"node": {"oid": "59aef1891903c7a521e9bc8bfe98624bcc63a1a1", "messageHeadline": "Merge pull request #644 from gakada/bits", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 92, "deletions": 28}}, {"node": {"oid": "1d8f782f665017c5e1e3487e121446051d86a9bb", "messageHeadline": "auto-gptq -> gptq and an import warning", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 10, "deletions": 4}}, {"node": {"oid": "85ebe408a5f99d64edad8c9490e136471d87d04a", "messageHeadline": "Merge pull request #646 from EleutherAI/fix-sample-logging", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 1}}, {"node": {"oid": "aeebf6f2c05d2a7769db96ca1864e7bd35071f5b", "messageHeadline": "fix and gather sample logging", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 10, "deletions": 1}}, {"node": {"oid": "11121ead9b8ac57591f857774b9c34b1a5b15075", "messageHeadline": "Merge pull request #645 from EleutherAI/fix-super-glue", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 65, "deletions": 107}}, {"node": {"oid": "8737d53a5ada22e346f5014adf0ee4b93528aeaf", "messageHeadline": "fixed super_glue and removed unused yaml config", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 65, "deletions": 107}}, {"node": {"oid": "4b8e7ca73891d04bd2b4b40a145c0443511d9e83", "messageHeadline": "update task readme for winogrande", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a8396b2c7a51dc08c607eb022d8d91aa086d612d", "messageHeadline": "add new task type: winograd_schema", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 75, "deletions": 13}}, {"node": {"oid": "d674c7bd74079f576798a08d7149b1bda2f74b3f", "messageHeadline": "winogrande task implemented", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 35, "deletions": 0}}, {"node": {"oid": "18d842abd1f56fad17e1a19cade5fcfab62481c4", "messageHeadline": "Add PEFT, quantization, remote code, LLaMA fix", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 83, "deletions": 25}}, {"node": {"oid": "2ef2d35f70930d312c4f4849689e5aad3beec00d", "messageHeadline": "Merge pull request #638 from EleutherAI/haileyschoelkopf-patch-1", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "649a7f95693f7c36a2c3ecc04e5be30eaf6e0272", "messageHeadline": "Add space for doc to text", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "fb8324c748db22cce66359b068cf723244d6dd34", "messageHeadline": "Update gitignore to resolve precommit issue", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "28da615acce580597c0ef13a353e5897535bf61e", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6cc461f5e8041598c57a7f30ab3cbb51374509ec", "messageHeadline": "Edit yaml file", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "6c69e30ae19b7ddf6fa0f69af1a45aad2a53d639", "messageHeadline": "Add qa4mre", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b88e3618127f47f3720454095d51b5ccac1ee352", "messageHeadline": "Add newline", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "cc3bdad31774e096971fd5cb9bec0bdcd28641b8", "messageHeadline": "Add qa4mre", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 21, "deletions": 0}}, {"node": {"oid": "e4a281f7a7550f7800b9ca31f9d1e77511edcdf4", "messageHeadline": "Update gitignore", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "605964ad0fbae49586a99046ebe562624ccef043", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "0fe7e8ebb90a63c6cc69ce8a0aaafe1c6031b328", "messageHeadline": "Merge pull request #639 from nikuya3/hellaswag-fix", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8b9e5fa8fffbe5bcc32033ea4e1da294af627a16", "messageHeadline": "[hellaswag] cast gold label to int", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4b5cf54343ef5918d5dc7f9e667b82ead6cc27d7", "messageHeadline": "Add caveats to `parallelize=True`", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "d98023036b21144801f3a1daef0742477042c003", "messageHeadline": "task readme updated for pubmed_qa", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "eb29b97b4233f1d2096ac169389454ea539cebfb", "messageHeadline": "pubmed_qa implemented and tested", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 36, "deletions": 0}}, {"node": {"oid": "e50c8c70dafff024672b5017f107217468d4baa6", "messageHeadline": "update task readme for arithmetic", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "470e33b126ecdef421ce3ab6e887826906766dd3", "messageHeadline": "add arithmetic tasks", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 150, "deletions": 0}}, {"node": {"oid": "72c618667c58624e829070a4475c94d462ea6d60", "messageHeadline": "update task readme for swag", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a4b6f914c9a776002a83e0e4ee009e2d95bc9167", "messageHeadline": "SWAG completed", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 20, "deletions": 0}}, {"node": {"oid": "d15ee17aafd52ee9425988c1e17a89f0a3877dd7", "messageHeadline": "Fix typo", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0e56fccacf0bbb5c0ae1f2f72f9521fd3935af8d", "messageHeadline": "Merge pull request #624 from EleutherAI/log-args", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "bd5b29eb0c2344e51247d18557b3b9cd3a9af0c8", "messageHeadline": "Merge pull request #609 from EleutherAI/misc-cleanup-refactor", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 30, "deletions": 75}}, {"node": {"oid": "46d3bead921b606c4cfb87e6de9bcd1c491a4588", "messageHeadline": "Merge pull request #625 from EleutherAI/device-mapping", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 96, "deletions": 15}}, {"node": {"oid": "f2b3d7f64f1119bec64bcdb8f163348bc2d0fdb2", "messageHeadline": "Update gitignore", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "a2e1d0520a735596aaeb5d5a9d8a3732b3f97b9c", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "fbd712f723d39e60949abeabd588f1a6f7fb8dcb", "messageHeadline": "Merge branch 'big-refactor' into misc-cleanup-refactor", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 372, "deletions": 528}}, {"node": {"oid": "909f734665879c4eb67a30186a901a93d7026c57", "messageHeadline": "Remove \"select_continuation_from_batch_left_padding\"", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 20}}, {"node": {"oid": "ffff47c6c0f689caa5df0abc28e867353c2eae2c", "messageHeadline": "Merge pull request #629 from EleutherAI/haileyschoelkopf-patch-1", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 7}}, {"node": {"oid": "4d0ac35c00cfd0e3b7b04b050aa5d1960c196cff", "messageHeadline": "Update task statuses on tracking list", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 7}}, {"node": {"oid": "886190e99c2c8fa53a6505e3aa3cc2141cdf99f3", "messageHeadline": "commit blank task yamls", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "dbc4c7d10830f9173af351287154a50af50f860d", "messageHeadline": "Merge branch 'big-refactor' into device-mapping", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 365, "deletions": 522}}, {"node": {"oid": "cb5423d331ac6ac725ce51bc4d8cff1ed73d1ec2", "messageHeadline": "remove stray print", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "865c49954c8a2ae2e5051d5a01d51b27022e0a2c", "messageHeadline": "update readme with parallelize=True", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 17, "deletions": 1}}, {"node": {"oid": "1b4fc896b30c3cf5b3f7a641f1dd3a010afe0872", "messageHeadline": "cleanup + change to name=parallelize", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 27, "deletions": 24}}, {"node": {"oid": "23eaa2db46e46076357000505f573e43711a1d7b", "messageHeadline": "add device_map options for HFLM", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 65, "deletions": 6}}, {"node": {"oid": "8762b07c8a2dfab994d4fc14a941fb31de919c9d", "messageHeadline": "make self.rank and world_size defaults more robust", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "ef4a26f6e3bc9fe4979aa86c4fa1081e10e3d21e", "messageHeadline": "Merge pull request #613 from EleutherAI/add-back-cache", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 208, "deletions": 512}}, {"node": {"oid": "c5dbf289c69313b2275b42805c1aee83c3704103", "messageHeadline": "Merge branch 'big-refactor' into add-back-cache", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 36, "deletions": 8}}, {"node": {"oid": "4e0d0e3a15ab30c8a39bbd47cbc4335e5b0b1834", "messageHeadline": "Merge pull request #619 from EleutherAI/cachinglm-only", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 172, "deletions": 25}}, {"node": {"oid": "03b9db6b2ecfe58397628128e6e919372394a451", "messageHeadline": "revert mutual info whitespace change", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 4}}, {"node": {"oid": "b250b0018b28277a29d8ce517a82347b83775480", "messageHeadline": "clean up batched code and add comments", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 20, "deletions": 16}}, {"node": {"oid": "5a5442ffae2a5173ec06600479e62e14d8336d2e", "messageHeadline": "make OAI completions work with new generation kwargs format", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 16, "deletions": 4}}, {"node": {"oid": "d1c5abef67318afa5795c20a7ecc7464b802359e", "messageHeadline": "add cache hooks to API LMs", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 11, "deletions": 0}}, {"node": {"oid": "732f7ed2707c2d9d4d774515ec05553d0d025d29", "messageHeadline": "change generation_kwargs -> kwargs", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "7ffb74ce16a8a01d384013b602a12a2e1dc7d4c0", "messageHeadline": "make cache-loading msg part of logger", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "fb108f2ed9df4da85cae7f35a56ced76f07d352b", "messageHeadline": "log arguments too", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "b2fb53d4bb568ddfd05c6ce705ee11a623686086", "messageHeadline": "Update model.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f95b160b20a78c0e6fe1e164e706350cba1d50ae", "messageHeadline": "Add tqdm progress bar for cachingLM", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "ffc3a45646d990588d90cb7dfdff7dfd081d2f8c", "messageHeadline": "push WIP batched multi-kwarg code", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 93, "deletions": 71}}, {"node": {"oid": "83f957bc395c47490e7758255e3017ef31e6c92e", "messageHeadline": "add Grouper", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 58, "deletions": 0}}, {"node": {"oid": "2fef6bc5cab73d49fe1bab36afea6309792d4dda", "messageHeadline": "update lm_eval.models.__init__", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "abbbc96f2a40272671e34636fe5015583f507b98", "messageHeadline": "remove mention of hf-causal from readme", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 14, "deletions": 13}}, {"node": {"oid": "4f002df5727f1f9deac94c42b9b7984086339e66", "messageHeadline": "remove hf-causal model type", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 433}}, {"node": {"oid": "adef632335427c98dfae67d233686e0208629e99", "messageHeadline": "Update evaluator.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a6d501f06098b703fe7e8c2685b136df21efef79", "messageHeadline": "Merge branch 'misc-cleanup-refactor' of https://github.com/EleutherAI\u2026", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 991, "deletions": 128}}, {"node": {"oid": "89a5a8654c6f0abc4f789f3d8cbe0a9044aa7f6e", "messageHeadline": "update delimiter->fewshot_delimiter", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b3aab393dc022ba21b478492f1d15dd4a9273ed7", "messageHeadline": "squeeze tensors before pad_and_concat", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "513352ae3502dc00329bdab084c1972e7b6fa22c", "messageHeadline": "remove stale commented code", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 7}}, {"node": {"oid": "dced6e96a71768a5d769f0ff37cb4a6554105bcc", "messageHeadline": "add batched generation", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 64, "deletions": 17}}, {"node": {"oid": "4f29f7ccec774de5f47e89a2ade346c5e07f7d1e", "messageHeadline": "merge big-refactor into fix branch", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 987, "deletions": 125}}, {"node": {"oid": "f832c7761ff9aceb77fd26aee597046bfd89bd6c", "messageHeadline": "add additional err handling for get_git_commit_hash", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "8d14b3768619ab6fc49ed53eb1a5b27c7fa53019", "messageHeadline": "make fewshot_docs warning more explicit", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e3960fa0aec6c646cf1e9567f8652a1666ad7bef", "messageHeadline": "add cache hooks to HF", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 16, "deletions": 10}}, {"node": {"oid": "a68c3fa45e76335ffa14484503e3696f196a6a5f", "messageHeadline": "fix mutual info normalization", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 5}}, {"node": {"oid": "6449ab1ac60c35cabd4cd5260cebbf723822d8cd", "messageHeadline": "add use_cache arg", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 16, "deletions": 6}}, {"node": {"oid": "1c0ff96899a177e98f401cbdc8304f69253b9fdb", "messageHeadline": "add CachingLM back", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 104, "deletions": 0}}, {"node": {"oid": "9dea125b2836da6705f1eca542e3853a0d89cda2", "messageHeadline": "Hotfix samplers.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1fa023958c07716bcfbc3d4456dc8e38271c4d66", "messageHeadline": "Merge pull request #565 from fattorib/seq2seq-refactor", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 728, "deletions": 27}}, {"node": {"oid": "d3cfdcf62012447507573bab43bac4a406740cbf", "messageHeadline": "Merge pull request #5 from EleutherAI/seq2seq-support", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "0500fb330ced9648333322838bca619059704c4b", "messageHeadline": "fix edge cases for seq2seq", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "89ad01864fd79a134d0c8ea8f73000017e5cfa88", "messageHeadline": "Merge branch 'big-refactor' into seq2seq-refactor", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 64, "deletions": 3}}, {"node": {"oid": "6122efac0076f67c7344ff19baa6e5ae01d071e9", "messageHeadline": "added for seq2seq greedy_until test", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 18, "deletions": 0}}, {"node": {"oid": "6d8cdef88dadfffb4a5e1eaf8b8b905ac208c71d", "messageHeadline": "rename boolq task name for testing", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 2}}, {"node": {"oid": "783ffd1e94a8254101b5adb6f27552631ae67ea3", "messageHeadline": "merged conflict resolved", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 32}}, {"node": {"oid": "82a21c3613dcf6da9915998f9154871824a5c169", "messageHeadline": "remove pure seq2seq LM class", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 404}}, {"node": {"oid": "621214e19558ec8c5471bf3b2c1c10dce709b6fb", "messageHeadline": "fix issues with encoder_attns, test lambada", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 1}}, {"node": {"oid": "1c409035f15cb2e05919077d305eac5a17a12988", "messageHeadline": "remove stray print in AnthropicLM, add TODO", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "360f7c3e2ee58999f9f76b4b15b70a95e1d248c5", "messageHeadline": "rename hf_merged.py", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "3a6adddb57fc8855f2311020e6b72d8ffa13b9ed", "messageHeadline": "discard ctx toks for causal greedy_until", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 27, "deletions": 10}}, {"node": {"oid": "b48f52053b59ce0a075f91bd912415b7b6714200", "messageHeadline": "more pre-commit", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 160, "deletions": 103}}, {"node": {"oid": "86b71954735db62a090fc1dd40aaaab9904ad266", "messageHeadline": "add start of docstring to HFLM", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 0}}, {"node": {"oid": "0e4f5361f85617de961099da1ca4e41ed73f1f45", "messageHeadline": "init tensors on-device where possible", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 12, "deletions": 6}}, {"node": {"oid": "bc7f52e6a477379a1a0de3d586d1a6b822d33926", "messageHeadline": "automatically unwrap model when needed", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 22, "deletions": 32}}, {"node": {"oid": "1bd6229ca431f086f37a79a444a8329c8b3940de", "messageHeadline": "remove some old code, edge-case seq2seq case", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 19, "deletions": 22}}, {"node": {"oid": "9f36ab18b3487105018dc79bbfdd405f5dc5a850", "messageHeadline": "remove stale comment", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "fb8abcec8d643abf5325e6c76c689f49f4df7cdf", "messageHeadline": "improve pad_and_concat", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 81, "deletions": 15}}, {"node": {"oid": "6d36a9c9fb8d6e509aee036476aae31955e74c0a", "messageHeadline": "allow include_task_folder to use non-leaf dir YAML files", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a4e84b4bb83faa611ce4a77474dec44582e6fc16", "messageHeadline": "add preliminary working HF-auto LM", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 527, "deletions": 0}}, {"node": {"oid": "0b4f88dd5cca745a8daa51688ee809d8a76d9d74", "messageHeadline": "make seq2seq take correct args format", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 63, "deletions": 65}}, {"node": {"oid": "1a6b31a8750a73ef6f05020b81d4cf50b28f9cbe", "messageHeadline": "resolved conflict", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 34, "deletions": 18}}, {"node": {"oid": "1d7d3de55a38b8a2f2ac2c7af485f953842e3cb3", "messageHeadline": "fix small TaskConfig bugs", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 8, "deletions": 2}}, {"node": {"oid": "fd33fb975d419710d771b93d164392fc891969f2", "messageHeadline": "comment on padding method for encoder", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "26a9a44526f3a389e65edb2c7a0727239dd0cb66", "messageHeadline": "multi-device support for loglikelihood_rolling", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 23, "deletions": 8}}, {"node": {"oid": "3589abbbaa0a4d51d3f7bb58abc2e3d3607a60fb", "messageHeadline": "loglikelihood_rolling for seq2seq", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 28, "deletions": 1}}, {"node": {"oid": "3789d34052f6426641aee226aa1c1119fed9fc90", "messageHeadline": "batch support for loglikelihood tokens", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 16, "deletions": 37}}, {"node": {"oid": "0a3b8069f758a59055ca42b2dcb614dc1e50437c", "messageHeadline": "resolved merge conflict", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 24, "deletions": 0}}, {"node": {"oid": "1de7e4a53558ea318b3a1d7170fe871ac70e127f", "messageHeadline": "accelerator multidevice setup", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 21, "deletions": 1}}, {"node": {"oid": "2af4f9e0821cb136d2e847653f68362176cdcd99", "messageHeadline": "commit on clean branch", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 347, "deletions": 0}}, {"node": {"oid": "53ff475ba5826273ed9d7dc39c521fe916d84adc", "messageHeadline": "Merge pull request #4 from EleutherAI/seq2seq-support", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 35, "deletions": 419}}, {"node": {"oid": "e0e9c97d4d7ac95596540ebdf84ce00c1f71c471", "messageHeadline": "remove pure seq2seq LM class", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 404}}, {"node": {"oid": "30e119eddfd14daa803b53149a3150e711a71fa7", "messageHeadline": "fix issues with encoder_attns, test lambada", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 1}}, {"node": {"oid": "d6ae9f2411d041189921cea70bfc30ebb03b0555", "messageHeadline": "remove stray print in AnthropicLM, add TODO", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "ed7fc67504898f214651008aec967c9ecde6f69c", "messageHeadline": "rename hf_merged.py", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "282889ed2f4bb970c1f4687fea48e58961280447", "messageHeadline": "discard ctx toks for causal greedy_until", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 27, "deletions": 10}}, {"node": {"oid": "9a8fee14bae62b4004ad809651aeae8e17b9a2dc", "messageHeadline": "Merge pull request #608 from nikuya3/hellaswag", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 64, "deletions": 3}}, {"node": {"oid": "a7993806a06e5587bb49ac1fbd9947c0fbaaa580", "messageHeadline": "patch some tests", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 21, "deletions": 11}}, {"node": {"oid": "6c80d52ae71d676c0ae79f5e964bfc06ed2f7682", "messageHeadline": "Merge branch 'hellaswag' of github.com:nikuya3/lm-evaluation-harness \u2026", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 204, "deletions": 103}}, {"node": {"oid": "cc89d4f948fdc431b011d6f3c691b3dcf35f0021", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "5be6a53d9e371e41765d535350af0c5acde8654a", "messageHeadline": "Trim white spaces and restore correct order in hellaswag preprocessing", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "3c895d5682f48a5aec436c75a9cc02d526a0cba5", "messageHeadline": "reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "3262fb748f78a2f2b86850937b51558776f8287b", "messageHeadline": "Update hellaswag.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "25dcf3335786bf4c88df3da2330d477e46f4b735", "messageHeadline": "Preprocess answer choices of hellaswag task", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 2, "deletions": 16}}, {"node": {"oid": "a37621f5fe9effd8ac50bb5a10e88ea5c0616ea6", "messageHeadline": "Add regex_replace filter for Jinja templates", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "3968fd0a79ae967d90a83cca9001bb19327554c3", "messageHeadline": "Handle non-serializable fields in task config and result object", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "1409bcf593acefac2a35eaf8e97cf7122fc8a37c", "messageHeadline": "Add evaluate requirement", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "6e482e5bfb37f967f43189f4b2de582467ec9a66", "messageHeadline": "Do not ignore task directories with __pycache__ subdirectories", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "db6912a86b218b2275d16342af290143b3fddf2d", "messageHeadline": "Port hellaswag task", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 68, "deletions": 0}}, {"node": {"oid": "dafc75293f4fa1970cb59a994c4f3d10ceb9cfee", "messageHeadline": "Update hellaswag.yaml", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "5c72066be8ef18443c864bda203d1a771e1001ca", "messageHeadline": "Merge pull request #599 from EleutherAI/fix-gold-aliases", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 79, "deletions": 33}}, {"node": {"oid": "abe05c5d0aff3af8147fd1dbfd4537ac1e6b5e79", "messageHeadline": "remove reference to undef `self.delimiter`", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "a575d9e5092aa5053c2904f090e1b587f6499c99", "messageHeadline": "swap target_delimiter and fewshot_delimiter", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "3dcde17bc7cb7ee0d461d1647a5764c4773acfb2", "messageHeadline": "Merge branch 'big-refactor' into fix-gold-aliases", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 143, "deletions": 69}}, {"node": {"oid": "b8cb513b54412e993d473a7b7dc8ac479d23be7f", "messageHeadline": "Update task.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ebbb6c00f58ef25de338ac65874216c5e4153aa0", "messageHeadline": "Update samplers.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "b85545b088b9ae7bc57209dbb25423927fd9c3c9", "messageHeadline": "Update samplers.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "bec2b500e2177bfa81d2e80a757b57c5c44b959f", "messageHeadline": "Preprocess answer choices of hellaswag task", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 2, "deletions": 16}}, {"node": {"oid": "8c653b93ceaa20223c0290792f035af380ece1cf", "messageHeadline": "Add regex_replace filter for Jinja templates", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "b1b4793c3ecc2882f37204d8e0cc2a75d8d904db", "messageHeadline": "Merge pull request #3 from EleutherAI/seq2seq-support", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 815, "deletions": 133}}, {"node": {"oid": "977817a0ee9ebefcee2f7072147a3734d482e21c", "messageHeadline": "made two delimiter modes", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "6ee8e4e3ecb5f1628843507b43d4cfcd11826d21", "messageHeadline": "fix accuracy -> acc for lambada", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "cbb04a208ba5f031338a3c926e7c56ca30270a94", "messageHeadline": "remove old code", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 35}}, {"node": {"oid": "df18b5a2d7caea8324df8afea313fc2f00bae5cb", "messageHeadline": "remove sh() util", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 9}}, {"node": {"oid": "9cf4a104154d19246912810ee8681eb97fa95528", "messageHeadline": "more pre-commit", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 151, "deletions": 114}}, {"node": {"oid": "306cfadad467945866d2fc8697effe79f83dbb24", "messageHeadline": "add start of docstring to HFLM", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 0}}, {"node": {"oid": "2402b33782cfd1eda8617df61956673d21b97230", "messageHeadline": "init tensors on-device where possible", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 12, "deletions": 6}}, {"node": {"oid": "a757f2934cfab92537fd2fcd4220e4cabbc1100d", "messageHeadline": "automatically unwrap model when needed", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 22, "deletions": 32}}, {"node": {"oid": "605b1cef529d2611fe9ef315ff8775fdb15eb24b", "messageHeadline": "remove some old code, edge-case seq2seq case", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 19, "deletions": 22}}, {"node": {"oid": "24a3940b261c185bddc8c9cb76f1e76b8ac34d76", "messageHeadline": "remove stale comment", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "ef09b104ac30e73cc0cb46a0adfbe7d9f340f772", "messageHeadline": "improve pad_and_concat", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 69, "deletions": 3}}, {"node": {"oid": "e8c84a3870f13c5437646c3c8fd6ec695e0e18e4", "messageHeadline": "allow include_task_folder to use non-leaf dir YAML files", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "193a46321fc6d8d02b0899e2298dc7303c063173", "messageHeadline": "add preliminary working HF-auto LM", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 527, "deletions": 0}}, {"node": {"oid": "e779b52c96dc879d409fb917149831328bd172c0", "messageHeadline": "make seq2seq take correct args format", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 63, "deletions": 65}}, {"node": {"oid": "53ff367194b13ebcb4b0fba192d9302886299d7c", "messageHeadline": "improved stop seqs to hf-causal", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 35, "deletions": 20}}, {"node": {"oid": "be5e118b0cdcf422eb59b4aae45cba3fd9220d9d", "messageHeadline": "fix small TaskConfig bugs", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 8, "deletions": 2}}, {"node": {"oid": "193b0a47670e789adbbe6dd8e8e05e0e92101eda", "messageHeadline": "Handle non-serializable fields in task config and result object", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "cba0eb1d767559f0cf993146d28ee4373206df83", "messageHeadline": "Add evaluate requirement", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "33c482ef8572dae0ab1255b8e633fe2ff6857f86", "messageHeadline": "Do not ignore task directories with __pycache__ subdirectories", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "10a2ccca48229121b62ff2e4ceee5acc7732536b", "messageHeadline": "Port hellaswag task", "author": {"name": "nikuya3", "email": "54780682+nikuya3@users.noreply.github.com", "user": {"login": "nopperl"}}, "additions": 68, "deletions": 0}}, {"node": {"oid": "3c805576f64c3410e2783db4c4e63efd69de8490", "messageHeadline": "reformat for pre-commit", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 8}}, {"node": {"oid": "6b375468012f7b479704fee9b0aa11cc609c3f89", "messageHeadline": "BaseLM to LM", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6a000adbc6ed282ea8cac1f29f9462c4aa90680a", "messageHeadline": "Merge pull request #606 from EleutherAI/model-guide-docs", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 116, "deletions": 14}}, {"node": {"oid": "24321ad80ba725e26be0aa197195efbf8569d18a", "messageHeadline": "pre-commit reformat", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 8}}, {"node": {"oid": "60e8803f9c26f1bd67bd81e0dc28f117be41e741", "messageHeadline": "commit WIP model guide docs", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 90, "deletions": 1}}, {"node": {"oid": "3821762e117390543e58c443d00989b2e9f1e8c0", "messageHeadline": "rename progress.md", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 23, "deletions": 12}}, {"node": {"oid": "77289aed101b6a88e87984e4eec97333981a9218", "messageHeadline": "remove descrip dict from test_evaluator", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "05a7d1da93c4f2a9f5a9ccc4441b9a5b07c36cc5", "messageHeadline": "rename test_description.py", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "194a806d14cb3dfb5f3676a08553b305fd2b151e", "messageHeadline": "add description to config, remove from cmdline args", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 52}}, {"node": {"oid": "dd2b2de9f0a1ed244fb979af66ae99825fd89331", "messageHeadline": "re-add dtype flag", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 20, "deletions": 2}}, {"node": {"oid": "f38f8e20a6c7d30669454bb077dc1ab338f672a0", "messageHeadline": "Merge branch 'big-refactor' into seq2seq-support", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 40, "deletions": 7}}, {"node": {"oid": "1fb166738b5ad44a9e117be8f6b812a444fabc6e", "messageHeadline": "add evaluate to reqs", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "3f0afe02ba243e74b531e9c4e3920cf9f4c3bd83", "messageHeadline": "misc fixes", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 21, "deletions": 5}}, {"node": {"oid": "41677741f2ddff606b1ff185525f8d01b98bf117", "messageHeadline": "Update checklist for OBQA", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c1dbda9412649a93805d9fb641d244ce192f1a3c", "messageHeadline": "Merge pull request #600 from farzanehnakhaee70/add-openbookqa-config", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 21, "deletions": 0}}, {"node": {"oid": "328bb7fcabc4ae25a44ff76c91ebc58b39afe14d", "messageHeadline": "add prost config", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 19, "deletions": 0}}, {"node": {"oid": "aa0326e35da97eb939eb62d170aa7d7fef03f6a4", "messageHeadline": "add openbookqa config", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 21, "deletions": 0}}, {"node": {"oid": "de21abbde3b00a47db6c1c4f85bcaa104c5c1505", "messageHeadline": "update docs + rest of mcqa tasks", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 31, "deletions": 22}}, {"node": {"oid": "ff4b2face314bed3362744cb35d232e1f1a43936", "messageHeadline": "Merge pull request #598 from EleutherAI/patch-multiple-choice", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 13, "deletions": 7}}, {"node": {"oid": "b4dd86e3ac0bca844e4ff54dfed6c7f168941460", "messageHeadline": "minor fix", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0dc67e6a463084a1d99c70d9a38204d88da73410", "messageHeadline": "Merge branch 'big-refactor' of github.com:EleutherAI/lm-evaluation-ha\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "c87fc78873e744f37d4e552ae964995380efeeb1", "messageHeadline": "Update setup.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "4e72f165a7c823dd4dcd89afdb6babd071ebe0e5", "messageHeadline": "fixes for multiple_choice", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 6}}, {"node": {"oid": "bafaef26021c116ebe607ae4796ae0618714318e", "messageHeadline": "Merge pull request #596 from EleutherAI/package-yamls", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 0}}, {"node": {"oid": "d3037f348b90189ac2e9ccd40fee6a2b210d23d3", "messageHeadline": "package yaml files in lm_eval,examples modules", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 0}}, {"node": {"oid": "a6c640d3c70c4356442cc4488aab13b291c15782", "messageHeadline": "Merge branch 'big-refactor' into seq2seq-refactor", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 22485, "deletions": 1987}}, {"node": {"oid": "24e3e3fa9d9cb5180212d4818f0c9b9bf6ceacbe", "messageHeadline": "Merge pull request #590 from gakada/big-refactor-merge", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 21545, "deletions": 54}}, {"node": {"oid": "b7c3580ad95e83d3471c902f15b88b31572869ce", "messageHeadline": "reformatted", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 100, "deletions": 35}}, {"node": {"oid": "86db4a4e88d3f780de2b765753ec52d3d35ef5c0", "messageHeadline": "Update CODEOWNERS", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b06e797ab29e8e9656c2ab154771438282fcc840", "messageHeadline": "Merge branch 'big-refactor' into big-refactor-merge", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 44, "deletions": 3}}, {"node": {"oid": "3fc5bedc450020b4e6780af113f645c0616d0322", "messageHeadline": "Merge pull request #563 from farzanehnakhaee70/big-refactor", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 42, "deletions": 2}}, {"node": {"oid": "5b6847be115d334546cefd420bbe4d1443bb1a18", "messageHeadline": "Merge commit 'refs/pull/563/head' of github.com:EleutherAI/lm-evaluat\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "4cc57af4eb052ca3285ee343368a13bb6e7b0f0e", "messageHeadline": "merged conflict", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 220, "deletions": 134}}, {"node": {"oid": "87fcaf89e56a8e35ecd314a6294903abfbaa476b", "messageHeadline": "Update .gitignore", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "b59fe37934ad1629e7609b1544210c2d923e7e58", "messageHeadline": "Merge branch 'big-refactor' into big-refactor-merge", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 58, "deletions": 1058}}, {"node": {"oid": "5fa8705648304821085c2dfd0549ed605d5bb9e0", "messageHeadline": "Delete task_table.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 268}}, {"node": {"oid": "c7572ba6df672006ffe193ef2f9ae29b80ed2933", "messageHeadline": "Delete triviaqa.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 126}}, {"node": {"oid": "025fa6e84e69e554dd189eef0305539761b9ad5d", "messageHeadline": "Delete gsm8k.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 147}}, {"node": {"oid": "2db1e7cbd2804b90fe8215c6d0a025af95072710", "messageHeadline": "Update __init__.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 4}}, {"node": {"oid": "1347b197d941852415a2a105d45ab8e89c34d04a", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "de02176799037c6a50c3e8943ed6e16e9b775e45", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 25, "deletions": 33}}, {"node": {"oid": "f191542a75010e40bc439eb03b8cc216b19e96a1", "messageHeadline": "Delete task_guide.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 305}}, {"node": {"oid": "8fb119890f100d515b3a7802d88b3fec637fccf1", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "275f8609b002897eded598422c58c2af0fb2ea21", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "81d33055d2de2dc50780480d7d12676ed52fade4", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 56, "deletions": 53}}, {"node": {"oid": "9d3247be3d966361cb66a8b77423842efe3434d7", "messageHeadline": "Merge branch 'EleutherAI:big-refactor' into big-refactor", "author": {"name": "farzanehnakhaee70", "email": "30573681+farzanehnakhaee70@users.noreply.github.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f637b876518e37a31da57c7a7cce29ceaec55632", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c490f16514e31560a3bd2952ccfbb84944fec85f", "messageHeadline": "Merge branch 'big-refactor' of github.com:EleutherAI/lm-evaluation-ha\u2026", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 168, "deletions": 93}}, {"node": {"oid": "8c9fc6b6f185f8343c67285b50bcfe0a08397dc8", "messageHeadline": "Update task.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8a42e3423bf2b281912083030d3e55e3112b25b6", "messageHeadline": "Merge branch 'big-refactor' into big-refactor", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 203, "deletions": 115}}, {"node": {"oid": "b5efc81366711a011b8e414211232348af80ada3", "messageHeadline": "Merge branch 'master' into big-refactor-merge", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 25, "deletions": 1225}}, {"node": {"oid": "b6c70fc84408a96b45b4bf161264b48c25af5e28", "messageHeadline": "Merge pull request #578 from EleutherAI/more-docs", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 167, "deletions": 92}}, {"node": {"oid": "6be66284a09b055602c6e324aca73205da9d1dc2", "messageHeadline": "Merge branch 'big-refactor' into more-docs", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 30, "deletions": 16}}, {"node": {"oid": "400c0199e44adf76278c433e7d2f17b9b4fdca9e", "messageHeadline": "minor fixes to satisify pre-commit", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 67, "deletions": 70}}, {"node": {"oid": "7dec84a02580c5d479c9bf8468debee63a5a154a", "messageHeadline": "Merge branch 'big-refactor' of github.com:EleutherAI/lm-evaluation-ha\u2026", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 35, "deletions": 19}}, {"node": {"oid": "e495e3a099dec85cf046511309c6dc49e8752d18", "messageHeadline": "Merge branch 'master' into big-refactor-test", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 22855, "deletions": 87}}, {"node": {"oid": "b018a7d517cff99d0f0eac2e77fb455566c2a29d", "messageHeadline": "Merge pull request #525 from seopbo/fix-triviaqa", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 20, "deletions": 211}}, {"node": {"oid": "50cf0984e44f8a55c988ea998a7ed1c2e0f938ff", "messageHeadline": "Update triviaqa.py", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "137d0c43e5795be8db840b66bf84745394c2919c", "messageHeadline": "Merge pull request #579 from SONG-WONHO/master", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 16, "deletions": 0}}, {"node": {"oid": "0c53ff4982f1cb0690f0ffbb2314b7ccc4b71177", "messageHeadline": "Merge pull request #580 from EleutherAI/fix-tasks", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 35, "deletions": 21}}, {"node": {"oid": "ced29179324d2e5eec888705d3bb9b5518e02200", "messageHeadline": "Update __init__.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 3}}, {"node": {"oid": "859e592026a30be0780e291e35c3f1828f588a70", "messageHeadline": "rm triviaqa to use HF filtered dev set", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 198}}, {"node": {"oid": "a1160392bd5913f7843f6d77c48bb2d2170e2eac", "messageHeadline": "Change triviaqa dataset path", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "346be595c2b9430a7b8fba6ace72d4c2491f8483", "messageHeadline": "Change accuracy to acc", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e33ae5ab43fdea5b89c9661a23e7afa6d2530557", "messageHeadline": "Update until in greedy_until", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "c08b2d9db63f6e95e2b4debe09955d59a26d0a08", "messageHeadline": "Remove **self._metric_kwargs[key]  from compute as it wasn't existed", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6d355b85dd9d173ca98b3a4d30d0288b220c2292", "messageHeadline": "Fixed typo on num_fewshot", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2ee7121bf21f0d464d5e5661b372cdb75177ea0a", "messageHeadline": "Merge branch 'big-refactor' into more-docs", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "9d06c95346ff393273bbb80379267eef094c0c74", "messageHeadline": "Merge pull request #585 from EleutherAI/fix-quantized-device", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 11, "deletions": 5}}, {"node": {"oid": "2d27f9e12a0be99809dfe5ec1e1949f3f73c665d", "messageHeadline": "add error handling for moving model to device", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 11, "deletions": 5}}, {"node": {"oid": "51d6951cc4a58583407bfd2f7d28d66c9f0dabae", "messageHeadline": "Merge pull request #584 from gakada/perf", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "d4889e1a570c60f2c1b6f90470677b4cba0d31cd", "messageHeadline": "Fix non-callable attributes in CachingLM", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "4c08d72acdf7c6f5cab5d708e8ef400aea08314c", "messageHeadline": "Merge pull request #582 from EleutherAI/fix-max-len", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 18, "deletions": 15}}, {"node": {"oid": "fa43ab2e369229e79480b79c350a4006e1f1280f", "messageHeadline": "fix seqlen issues for bloom, OPT tokenizer check", "author": {"name": "haileyschoelkopf", "email": "hailey@eleuther.ai", "user": {"login": "haileyschoelkopf"}}, "additions": 18, "deletions": 15}}, {"node": {"oid": "f1d251ca65fea5c8f55ef8034fa02c657cc2c8fe", "messageHeadline": "Update output path for the example logger", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 12, "deletions": 10}}, {"node": {"oid": "5042390713aa2a1954faee139a465fc3e999b6ad", "messageHeadline": "Merge pull request #581 from EleutherAI/all_tasks_mutable", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 7}}, {"node": {"oid": "3c284567bddc677f555acb34ecac432b912f9f14", "messageHeadline": "missing ALL_TASKS import", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "483f86d917ba80f384aca966b6aa47868dc82892", "messageHeadline": "set() works but not list????", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "baff256846ac12725486e71ea5cfd10c3ebbd4f3", "messageHeadline": "fix matthews corr. metric", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 60, "deletions": 22}}, {"node": {"oid": "5a766ac52020e8e5ca9ee437825107c324e8a95f", "messageHeadline": "Merge branch 'big-refactor' into big-refactor", "author": {"name": "farzanehnakhaee70", "email": "30573681+farzanehnakhaee70@users.noreply.github.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 1592, "deletions": 1385}}, {"node": {"oid": "0a1ced2214c8714e6a36f640a36ec5393c1e8e4d", "messageHeadline": "fixes some minor issues on tasks. For yaml, the task name should be <\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 32, "deletions": 18}}, {"node": {"oid": "9554f8da4c7dc84afc285d689d4ae2dd830bf6a8", "messageHeadline": "Merge branch 'big-refactor' of github.com:EleutherAI/lm-evaluation-ha\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "44b6ce017a98d4c2704c3ec597fc9aeff81663cd", "messageHeadline": "merge updated big-refactor", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1607, "deletions": 1391}}, {"node": {"oid": "01cfb2ff57a2173ef118215111c1b0be1147bdfb", "messageHeadline": "Update openai_completions.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "990bc542b19ec0ec2fd5a8a028c791ac28d5b4e8", "messageHeadline": "Add - 4bit-related args", "author": {"name": "ynot", "email": "win778500@gmail.com", "user": {"login": "SONG-WONHO"}}, "additions": 16, "deletions": 0}}, {"node": {"oid": "f862a11880ae0d140db50337c02b8b375f4b1bed", "messageHeadline": "Merge pull request #572 from gakada/perf", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 68, "deletions": 59}}, {"node": {"oid": "23dcc12ef2845cde266f02719fe3296417936f7b", "messageHeadline": "Merge pull request #575 from poedator/patch-1", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "811252b8af6e74f65ffc94c0467881eb3b8b97fd", "messageHeadline": "Update evaluator.py - extra .model added", "author": {"name": "Poedator", "email": "24738311+poedator@users.noreply.github.com", "user": {"login": "poedator"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "69ed600d1d19b55704051f3b48e7f78d1a8061b7", "messageHeadline": "fix for non-str output from evaluator", "author": {"name": "Poedator", "email": "24738311+poedator@users.noreply.github.com", "user": {"login": "poedator"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "5026799205ddc2c847995118fd5f1618d73f3c92", "messageHeadline": "update readme for wikitext", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 11, "deletions": 4}}, {"node": {"oid": "11dde9908eeeda29dd8b1e722908fcee670acce5", "messageHeadline": "document usage of template subfolder", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 5, "deletions": 0}}, {"node": {"oid": "2808c0498ef0df9b89f9656f45ae9aefd7e6d886", "messageHeadline": "add a new task template", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 36, "deletions": 0}}, {"node": {"oid": "d3fa8470924cdc94f1036ddd67c03a1c735e1305", "messageHeadline": "address issues raised on task docs", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 15}}, {"node": {"oid": "be1aea6d0a52cddd8e979c263d401bfecb631a4d", "messageHeadline": "Merge pull request #576 from EleutherAI/better-docs", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 42, "deletions": 9}}, {"node": {"oid": "2d54df7efb88039f3021f87bcd49423c31666df3", "messageHeadline": "Merge branch 'big-refactor' into better-docs", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 218, "deletions": 41}}, {"node": {"oid": "966f0686d53ff0d2038248e8b6e6a1a629dbce6b", "messageHeadline": "Merge pull request #577 from EleutherAI/hailey-docs", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 132, "deletions": 27}}, {"node": {"oid": "13a56c9aace2cf4d58b42bda693084e59c598f11", "messageHeadline": "new docs update", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 31, "deletions": 11}}, {"node": {"oid": "c1e50cc5d6016971369b0cc6ce9eeb606278cf21", "messageHeadline": "Merge branch 'hailey-docs' of https://github.com/EleutherAI/lm-evalua\u2026", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 4, "deletions": 8}}, {"node": {"oid": "7894dde5bc3ba7047ab8b9ffbd7bc31b38e2b6a3", "messageHeadline": "can leave out metric details optionally", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "24000871a0e9193b36411524e77863f877503d36", "messageHeadline": "Update advanced_task_guide.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 8}}, {"node": {"oid": "956a78b996f435b2c20d1e7537e8f96c0fa442b4", "messageHeadline": "push examples/ folder", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 32, "deletions": 0}}, {"node": {"oid": "5cab26646bfc113824e62bfd78205e4917779bf3", "messageHeadline": "should not shadow openai pkg name", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b8799c3582e64ec03ee2b33758f6812af02f6e35", "messageHeadline": "Promptsource explainer", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 1}}, {"node": {"oid": "640d98177e04a62ab46eb4556fb21f3b5373317f", "messageHeadline": "update advanced docs", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 66, "deletions": 10}}, {"node": {"oid": "4e764ca02ef2e36aa013ca4bc4ffff45217d7f1d", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "194d881805f8afdcf072c08dd48dbbd575b1f144", "messageHeadline": "added explained to use python function", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 33, "deletions": 7}}, {"node": {"oid": "b9a6a3dfaddf0ae042aa13932b41f20cd82801b6", "messageHeadline": "Update evaluator.py cache_db argument str", "author": {"name": "Poedator", "email": "24738311+poedator@users.noreply.github.com", "user": {"login": "poedator"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b7a1214eabaaa4a8ed52f2e76fd1ebb213120988", "messageHeadline": "WIP docs update", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 94, "deletions": 24}}, {"node": {"oid": "6ae376e3a43caa58b95bb8aa73054a94827bf560", "messageHeadline": "Merge pull request #574 from EleutherAI/StellaAthena-patch-1", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6db83ddc2708192677a4e17c6336c0d92be4bf0c", "messageHeadline": "Rename gpt3.py to openai.py", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "48787d6de82981f301bec01e0018efb7090ace0b", "messageHeadline": "Add a secondary registration of the GPT-3 model class with gooseai", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e131056a5b3c6f31942f1a2f4a0845e1128f8865", "messageHeadline": "Remove the registration of \"GPT2\" as a model type", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "79b972d6d67cf6f4647debe9d22d9c2735fffda8", "messageHeadline": "[Refactor] [WIP] New YAML advanced docs (#567)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 483, "deletions": 958}}, {"node": {"oid": "55eccc29c4d02e724a44e51bed803a824e719730", "messageHeadline": "comment on padding method for encoder", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "5486050fe7cebef5a3d51d168d4da26bd1505959", "messageHeadline": "multi-device support for loglikelihood_rolling", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 23, "deletions": 8}}, {"node": {"oid": "b99c0b41e60dc69b042c2d337bc25a07e6caf9a3", "messageHeadline": "loglikelihood_rolling for seq2seq", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 28, "deletions": 1}}, {"node": {"oid": "79545adbb795991ebc27ba71599af7b9d5071711", "messageHeadline": "Merge remote-tracking branch 'upstream/big-refactor' into seq2seq-ref\u2026", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 916, "deletions": 467}}, {"node": {"oid": "8cec82b26c53174130584d9ae1e41430baa3f342", "messageHeadline": "Add --max_batch_size and --batch_size auto:N", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 68, "deletions": 59}}, {"node": {"oid": "a0c1dbbd573d8e1fcdfa439b08abd71a6a1ba252", "messageHeadline": "add resps and filter_resps to the  example logger", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 7, "deletions": 1}}, {"node": {"oid": "eb7b9095ef5afa73f4c9a3643e20c4f50c5225b3", "messageHeadline": "batch support for loglikelihood tokens", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 40, "deletions": 38}}, {"node": {"oid": "b21c8f3dae152c8dbc93b001e1e2d6a6b93e6cf1", "messageHeadline": "Merge pull request #568 from EleutherAI/revert-537-mmlu", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 14}}, {"node": {"oid": "21970120f9ca2ac02dbb4db63654da4303bb66f2", "messageHeadline": "Revert \"Add MultipleChoiceExactTask\"", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 14}}, {"node": {"oid": "38c7427833f303e1a3b2bfd1ff90d0c502464080", "messageHeadline": "Merge pull request #537 from gakada/mmlu", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 14, "deletions": 0}}, {"node": {"oid": "cc4eab6a2d9ce7626436320d98bb445061410f54", "messageHeadline": "Add Anthropic support (#562)", "author": {"name": "Jason Phang", "email": "jasonphang@nyu.edu", "user": {"login": "zphang"}}, "additions": 112, "deletions": 0}}, {"node": {"oid": "761f0087c82c1b3324dc6c1deff428760efef8f4", "messageHeadline": "Merge pull request #560 from EleutherAI/dataset-metric-log", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1011, "deletions": 333}}, {"node": {"oid": "ae4d9ed2eeec97fd6f8042bd462202c84ee1aaf8", "messageHeadline": "changes for pre-commit", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 56, "deletions": 44}}, {"node": {"oid": "6b72d7b76f3d76b5c626c4e26fe65bd2cf902221", "messageHeadline": "Merge branch 'dataset-metric-log' of github.com:EleutherAI/lm-evaluat\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "224b0854f0606ce0dccf18e30d819c700d5b2f0c", "messageHeadline": "change import origin", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 22, "deletions": 9}}, {"node": {"oid": "76c01e321dc8c2e9b198e0498aecb0ca038719f8", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8171906d0ca0d1741a2f7858cd3f45cc3995b816", "messageHeadline": "resolved latest merge conflict", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 417, "deletions": 175}}, {"node": {"oid": "232632c644f6d612eaba9e105291d15c7d8c629c", "messageHeadline": "[Refactor] Non-greedy generation ; WIP GSM8k yaml (#559)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 393, "deletions": 622}}, {"node": {"oid": "a2e4115854cb429825782226d784098de293ba50", "messageHeadline": "fixed acc_mutula_info calc position", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 11, "deletions": 12}}, {"node": {"oid": "424a428090cc5c98156a0fa1e5f1c5b4731aa6ba", "messageHeadline": "fixed register_model origin and other imports", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 8}}, {"node": {"oid": "ddb7c0f3c9590b1c61381cc0d1846f08aa0b0f99", "messageHeadline": "added error message", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "06eb4e1bee806ce326702afd5eddfda0b7da596f", "messageHeadline": "changed origin of get_model", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "c75ce009a9ccfd0975400da7de0a69b304f09431", "messageHeadline": "fixed error in aggregation", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 4}}, {"node": {"oid": "829280e8c043724aee2ab856ec57e41de9a2ba6c", "messageHeadline": "adjustments on import origin", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 48, "deletions": 17}}, {"node": {"oid": "9c3f72270a3de897bbb7c117de4a6b82c5a23ef9", "messageHeadline": "modified import origin", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "7b0b42c4249539fc47a0e5ee46936d33089df3b4", "messageHeadline": "removed", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 318}}, {"node": {"oid": "a22d8ffa0337f4e629d7a5371c2e6e1ef2edf504", "messageHeadline": "modified import orgin", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 14, "deletions": 8}}, {"node": {"oid": "f216608960b026f47a257d2ac3774dc0bad206fe", "messageHeadline": "MODEL_REGISTRY to registry.py", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 26}}, {"node": {"oid": "9e75a84d709244b0ffcd1fdf6a36ec974c25e553", "messageHeadline": "moved METRIC_REGISTRY to registry.py", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 251, "deletions": 2}}, {"node": {"oid": "9b39df9b2e28c8a1141b337e2d7f2c3b18b9110a", "messageHeadline": "removed import metric", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "79db408862b39c846f8c1c049e88530c9eeda4bf", "messageHeadline": "renamed register.py to registry.py", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 157, "deletions": 105}}, {"node": {"oid": "226063ce5cb00a2bf94544b704860f7cfc765f93", "messageHeadline": "accelerator multidevice setup", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 21, "deletions": 1}}, {"node": {"oid": "28500952e80922657df72c54de5decbed4945915", "messageHeadline": "commit on clean branch", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 347, "deletions": 0}}, {"node": {"oid": "9c89d1fcf334fa28dc0a26c4e48bfdf9f3db0458", "messageHeadline": "Update task_guide.md (#564)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b7cfed1942551c7fcf840f0cd0ea302367a76d12", "messageHeadline": "add example logger", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 30, "deletions": 2}}, {"node": {"oid": "db42dd034a328acc6e46dc202f1f2b0e026896e4", "messageHeadline": "Remove outputs folder from git", "author": {"name": "FarzanehNakhaee", "email": "farzanehnakhaee70@gmail.com", "user": {"login": "farzanehnakhaee70"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "c2fcf688325afb26260395ece2cc77b5c48dcfed", "messageHeadline": "Add MultipleChoiceExactTask", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 15, "deletions": 1}}, {"node": {"oid": "7d0e3ebf3c82f20a0e84bd9f412cfadf961f7ebe", "messageHeadline": "added more registry", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 74, "deletions": 0}}, {"node": {"oid": "ba41ab7cfea215389397c32e0f604bf33ef84d3f", "messageHeadline": "mejor revision into how metrics are properly loaded for all output_types", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 62, "deletions": 46}}, {"node": {"oid": "b3591562e4207c74f743f066be3d2f4c30899b7c", "messageHeadline": "metrics are now in a special folder so that registry can work better", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 320, "deletions": 264}}, {"node": {"oid": "48344fcb1881582cce65990d9dc3a7be14b35904", "messageHeadline": "stderr_for_metric imported from lm_eval.metrics now", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "4b701e228768052cfae9043dca13e82052ca5eea", "messageHeadline": "Add load_in_4bit and fix peft loading (#556)", "author": {"name": "gakada", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 19, "deletions": 16}}, {"node": {"oid": "0a9ad6eecc6b3d3be431a340c2ea3f3ac2698437", "messageHeadline": "much better way to process all metrics chosen", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 82, "deletions": 69}}, {"node": {"oid": "5693abc5a866d6c1523e2177e320608d749dcecb", "messageHeadline": "show list of all datasets", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "01e0626fd119882f6ced8c98df52d12ac9362e51", "messageHeadline": "Merge pull request #557 from kwikiel/master", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6690afaf80336341c8643efef44d48185476e5fb", "messageHeadline": "Fix typo", "author": {"name": "Kacper Wikie\u0142", "email": "5274399+kwikiel@users.noreply.github.com", "user": {"login": "kwikiel"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4e9412d51ee7adf5cb89303f7c88ac3fbb7b0703", "messageHeadline": "Add diagram and workflow overview", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 26, "deletions": 1}}, {"node": {"oid": "63eabe79524cf104ab290a2374e8ee9ada764c94", "messageHeadline": "added `unwrap_model` for generate", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 17, "deletions": 7}}, {"node": {"oid": "6df13d93731338bbb27826e0a7b3b132a4bcb4c7", "messageHeadline": "Merge branch 'big-refactor' of github.com:EleutherAI/lm-evaluation-ha\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 852, "deletions": 24}}, {"node": {"oid": "6d7a24b244f65d93292dba1d1192eeebbbb0a620", "messageHeadline": "fixed typo", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 23, "deletions": 23}}, {"node": {"oid": "82786023ba4c1c728399994523622ba4dcda7af9", "messageHeadline": "fixed warning and info with proper logging", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 6}}, {"node": {"oid": "2976f69bc29aa0eac26484d71929cba36a6683ed", "messageHeadline": "Use max_length in AutoSeq2SeqLM (#551)", "author": {"name": "gakada", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 1, "deletions": 10}}, {"node": {"oid": "92929bd2a0a23b98da63fc250d5895f6f718aef7", "messageHeadline": "Move spaces from context to continuation (#546)", "author": {"name": "gakada", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 5, "deletions": 7}}, {"node": {"oid": "47b888bb4999a52bb822627bf82fc4c3b5152b49", "messageHeadline": "revert to take_first filter", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "c746d1fb27424db616e6807518e9d1617309c1ad", "messageHeadline": "fixing metric for each output_type", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 52, "deletions": 24}}, {"node": {"oid": "a339ffd820131bd2a6c7e70f4324854f433363fa", "messageHeadline": "allow to use alternative methods to use hf datasets, allow configurat\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 24, "deletions": 24}}, {"node": {"oid": "36da9c66327c8959d80e1ea58f898f95ce5d083a", "messageHeadline": "removed prefix \"configurable\" from get_task_name_from_config", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "d1caf4a4cfa53c61176574a0dc3659c2f6683673", "messageHeadline": "Create model_guide.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "76fed776acda05bc3cfd6dc6111e23d867380fb1", "messageHeadline": "Update CODEOWNERS", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "00d33dfe085c813fe9eec12a2131977a9264e814", "messageHeadline": "push task checklist", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 134, "deletions": 0}}, {"node": {"oid": "44eec73bb66d10919c395d6be9cf2b5947c2cd68", "messageHeadline": "push many yaml seed tasks", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 716, "deletions": 22}}, {"node": {"oid": "82401c967486d1a6a7622f6e2ec40f19d7e1a40f", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 10, "deletions": 3}}, {"node": {"oid": "f61974ed05df0c4350957a070b7be3cfb8bc5ce4", "messageHeadline": "fix default .py task filters", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "9deea116a2d32fee75d569168668ef737ff7bb9e", "messageHeadline": "Merge branch 'local-file' into dataset-metric-log", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 27, "deletions": 15}}, {"node": {"oid": "9f62d3acee45933c3e22fa378963e2975253fbd9", "messageHeadline": "Merge pull request #542 from EleutherAI/promptsource-extra", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 2}}, {"node": {"oid": "6e1238cb50a4f892b4696a9b1fedac5156a06bfb", "messageHeadline": "Merge branch 'big-refactor' of github.com:EleutherAI/lm-evaluation-ha\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "e367784884c500bd6c012272b3770bf59538ccb7", "messageHeadline": "Update gpt2.py", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "6f98c7805fb1099c186b2b4e7a7607590151cc69", "messageHeadline": "adjusted to follow pre-commit conventions", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 8, "deletions": 4}}, {"node": {"oid": "6acc5c4738bea53a7d042de0c6ab80cd1cc788aa", "messageHeadline": "result_dict process", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 27, "deletions": 15}}, {"node": {"oid": "9329fc9b40dce4bfe2d7f06a1d2b9ebf15b0057a", "messageHeadline": "add promptsource as an extra", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "23f30926f3ce738e3eee4e6be5c29fb3467e3a6e", "messageHeadline": "Fix LLaMA tokenization issue (#531)", "author": {"name": "gakada", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 205, "deletions": 45}}, {"node": {"oid": "441e6ac1bcae98f4e93cf4ba0bc10dc1e61a6256", "messageHeadline": "changed quantized type signature and default val (#532)", "author": {"name": "Sam Passaglia", "email": "8333102+passaglia@users.noreply.github.com", "user": {"login": "passaglia"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "095d84064b02fc7b126ed293ea97e9dba4299ad2", "messageHeadline": "Merge pull request #519 from gakada/gptq", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 51, "deletions": 11}}, {"node": {"oid": "8c419c83f2305b73394a98f41b01b0cedbf47351", "messageHeadline": "Fix triviaqa task", "author": {"name": "seopbo", "email": "bsk0130@gmail.com", "user": {"login": "seopbo"}}, "additions": 17, "deletions": 9}}, {"node": {"oid": "c11ad4f2f07cd96bc24bdfacfc3a218c031b2768", "messageHeadline": "GPTQ: add auto-gptq extra, add gptq_use_triton parameter", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 16, "deletions": 3}}, {"node": {"oid": "8cff2bea3e3b3179afc461fc079d57f81d2c128a", "messageHeadline": "Extend `dtype` command line flag to `HFLM` (#523)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 17, "deletions": 3}}, {"node": {"oid": "b465cd01ae51dc7803192d9d049fba5a823a322a", "messageHeadline": "GPTQ: fix README and support default names", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 10, "deletions": 9}}, {"node": {"oid": "4e94af6f3728ac27cf06777e792e30c7949b0eae", "messageHeadline": "Merge pull request #522 from EleutherAI/fix-mgpt-fewshot", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 152, "deletions": 0}}, {"node": {"oid": "25699d3ec53334bd47138e8d3777ac57cc8133a4", "messageHeadline": "fixed fewshot prompt by filling [mask]", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 76, "deletions": 0}}, {"node": {"oid": "5a8ac19857ad1b06220e820c339fe2602e6d3442", "messageHeadline": "re implemented fewshot_context method in the class to allow custom pr\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 76, "deletions": 0}}, {"node": {"oid": "b698048dd25081cd6d33c7fbae24b1fda39294ed", "messageHeadline": "better error warning", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "621561009baef8135fb9c5a245d9cebd84497afd", "messageHeadline": "Merge pull request #488 from fattorib/multigpu-feature", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 236, "deletions": 81}}, {"node": {"oid": "d924ca336fd78df3619b39f633f3f464d9314727", "messageHeadline": "Merge pull request #2 from EleutherAI/multigpu-feature-minor-edits", "author": {"name": "ben", "email": "46356259+fattorib@users.noreply.github.com", "user": {"login": "fattorib"}}, "additions": 1170, "deletions": 540}}, {"node": {"oid": "c77fa4619f314de62d96fe8fa306a34805eb5032", "messageHeadline": "Merge branch 'multigpu-feature' into multigpu-feature-minor-edits", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 4}}, {"node": {"oid": "650d3c76419a10fc8edd2e5cfc62361d1ad5c69d", "messageHeadline": "remove device_placement flag and fix rank/world_size assignments", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "bc103ce2b24ccd2001ed45375199ac1cb3567f81", "messageHeadline": "fixes", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 57, "deletions": 79}}, {"node": {"oid": "9f1cb1e791d8c9f06c0d9092506d50ef1e12c0a5", "messageHeadline": "merged with conflict resolved", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1269, "deletions": 838}}, {"node": {"oid": "0375b7922a85f662f916255e6364359cdc984c2b", "messageHeadline": "Merge pull request #520 from EleutherAI/update-config", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 167, "deletions": 44}}, {"node": {"oid": "eb42b01b70e2b86b8d86a94846f49051372abba0", "messageHeadline": "added more promptsource examples", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 161, "deletions": 41}}, {"node": {"oid": "1cf51552fac0dca22ca0519299e3ad06b0c21300", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 3}}, {"node": {"oid": "b296c4f6300fa924320d7d5695f810b446d845f0", "messageHeadline": "Add support for loading GPTQ models via AutoGPTQ", "author": {"name": "gk", "email": "gkpyon@gmail.com", "user": {"login": "gakada"}}, "additions": 37, "deletions": 11}}, {"node": {"oid": "71388a7e34c8831b1217a1a490f13f7608db1c6a", "messageHeadline": "pad out all batches loglikelihood_rolling. confirmed correctness", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 30, "deletions": 64}}, {"node": {"oid": "84ef60eeb3c04bcabd11adfaed26a88a33a5ad9c", "messageHeadline": "Merge pull request #481 from janEbert/json-task", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 129, "deletions": 1}}, {"node": {"oid": "4de8a74eb4b92654b59a5dabced015b9d35e3672", "messageHeadline": "Merge branch 'master' into json-task", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 722934, "deletions": 627}}, {"node": {"oid": "bda688457ef589c6ccdae829a29d8030bf5bbdbe", "messageHeadline": "Merge pull request #492 from juletx/eval-info", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 92, "deletions": 2}}, {"node": {"oid": "96a83d45e751d6970d153082ca50ff08f6f00e2e", "messageHeadline": "Merge pull request #480 from kenhktsui/float-limit", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 8, "deletions": 3}}, {"node": {"oid": "c5ed8cdcdba0fa08ab43df7e8e68f644ae490af4", "messageHeadline": "Merge pull request #501 from EleutherAI/update-config", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 974, "deletions": 490}}, {"node": {"oid": "c10c08a2f71f970820f27275118c216913d6b2f4", "messageHeadline": "unwrap gpt2 model to get n_ctx", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "e4f1dfb6aca73d3c8efc16ea826f108ce8882aaf", "messageHeadline": "improved warning message for processes < total gpus", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "80fe11d8d3bfdd21d85dfb84a63113a8072e4475", "messageHeadline": "keep local changes", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 0, "deletions": 20}}, {"node": {"oid": "c23dd0fd6ccac256fae11900ef0b771156204ed5", "messageHeadline": "Merge remote-tracking branch 'upstream/big-refactor' into multigpu-fe\u2026", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 201, "deletions": 388}}, {"node": {"oid": "65127577d8f8382221c37885b8e587fbf167b0dd", "messageHeadline": "unwrap model to get pos embs", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "f1372139f340ad386d389b5ff86246452d2d395f", "messageHeadline": "update README with instructions for multiGPU evals", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 13, "deletions": 0}}, {"node": {"oid": "c17e3659de75b23c3ca210fff65076450eb0237f", "messageHeadline": "Update README.md", "author": {"name": "Lintang Sutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 4}}, {"node": {"oid": "0996ad39e4a7b3d46d9e12112537da113e182f0e", "messageHeadline": "no need for pre-commit to check custom tags", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "c4c20ff50a104f9512ced9a696c02def68af8bf4", "messageHeadline": "pre-commit stuff", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 338, "deletions": 278}}, {"node": {"oid": "e56b950a0b1a45ed027739d7861905d112d40ca6", "messageHeadline": "able to use prompts from promptsource", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 101, "deletions": 36}}, {"node": {"oid": "1c7521a0591fb5a7683b9c001c4ef81b404668a9", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6399ef9e7884c9a627db7dd4ee6ca8e4b50c7f22", "messageHeadline": "handles prompt from promptsource", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 38, "deletions": 13}}, {"node": {"oid": "e53eb332fc545dcfd23680ae4b9ba3f9e754a3b8", "messageHeadline": "Merge pull request #477 from juletx/results", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 21063, "deletions": 0}}, {"node": {"oid": "d13271933bb7955a27374a7431b54532a755b81b", "messageHeadline": "Merge pull request #483 from janEbert/out-dir", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "e7a212ffa0643b81d92f5afab18aeb38e1f66c5b", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 3}}, {"node": {"oid": "c41bb266a3bd805d110f08512015839a2a9ef8ee", "messageHeadline": "removed", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 55}}, {"node": {"oid": "1405697fe55eaed566ba21b85777cb6a5e0cb4df", "messageHeadline": "edit", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 5}}, {"node": {"oid": "c37268bb60ef3a5c2a2742cbf2767931838db198", "messageHeadline": "update fewshot_docs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 15, "deletions": 5}}, {"node": {"oid": "b7f896c7594e8a26d80d12360a6f765af48c2d3e", "messageHeadline": "fixed so that fewshot from argparse is passed to the task", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "1c0a1f8e140765abb068a4c1e9424f76d037c0e6", "messageHeadline": "condition if kwargs is None", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 3}}, {"node": {"oid": "1e0d27ba2e839847062f35bf81bcb6a2d7580ce1", "messageHeadline": "rename", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 48, "deletions": 0}}, {"node": {"oid": "64912d8102a94126c21e43e05649a53320932eea", "messageHeadline": "minor edit", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 18}}, {"node": {"oid": "d75da0ddf30bda95e58d4b5a96d8eca213386a92", "messageHeadline": "updated filter process", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "c3dabb32649a184ca41b6a0c40edcfd601faa472", "messageHeadline": "updated filter process", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 51, "deletions": 26}}, {"node": {"oid": "4a3c1f19cfc03304084afc22342cf8a48a541938", "messageHeadline": "set template_aliases as conditional", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 7}}, {"node": {"oid": "7d6b39ed1969631a8bdfaba1c36b2851b6593672", "messageHeadline": "fewshot_doc", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 12, "deletions": 4}}, {"node": {"oid": "8d608117e9ecc68b1bbf583152c6ab4c1b183e52", "messageHeadline": "added logging process", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 47, "deletions": 28}}, {"node": {"oid": "4923a7ce7ccd5af07ced8fbe07a650b066dcf3ec", "messageHeadline": "fixed process", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "e786cbac6755d87b0a51fd9ed09712bdb4caa0e2", "messageHeadline": "detect if doc_to_x is a function", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 6}}, {"node": {"oid": "cbbe3b236406b451ad9b69a4b612fec19cb4ff30", "messageHeadline": "added more samples", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 81, "deletions": 0}}, {"node": {"oid": "cfbfa71c5269d96f28c93fe8aac871e1c9affb32", "messageHeadline": "can process !function", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 55, "deletions": 0}}, {"node": {"oid": "7aee2dfff3ecc61e5437b5c2b18756fa227f9c19", "messageHeadline": "moved functions out and some fixes", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 14, "deletions": 45}}, {"node": {"oid": "66c5819428ca10cba552a923a0d3193c1097f65b", "messageHeadline": "can process doc_to_text and doc_to_target as function", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 28, "deletions": 7}}, {"node": {"oid": "51b795cf529a3085eb59f2148d16c5623c8589c7", "messageHeadline": "adjustments to ALL_TASKS", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "64d07df7e1397e1ab5d096825bb4a994da3755fa", "messageHeadline": "delete files", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 714}}, {"node": {"oid": "439c7c6cb76deab0ad832a82ff8525b077392c53", "messageHeadline": "arc yaml set as 1 group", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 5, "deletions": 4}}, {"node": {"oid": "a5f5d92b2186e270c340ad613c3a2bd4bee20a78", "messageHeadline": "adjust super glue cb", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 27, "deletions": 26}}, {"node": {"oid": "600918bb7ec167e10e9b7f3dd36461d7a1a8ff19", "messageHeadline": "can list other yamls to be included and loaded", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 37, "deletions": 1}}, {"node": {"oid": "3cdc47c51066769fb4131845760a571283c2329c", "messageHeadline": "removed commented lines", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 0, "deletions": 20}}, {"node": {"oid": "2091a0f1e2fc76c681489103d1a091a1144807fd", "messageHeadline": "no need for yaml loader function", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 22}}, {"node": {"oid": "a179c47260a478674a70765ea3b44093e7f18b6d", "messageHeadline": "python func of tasks and yamls are both in the tasks/ with yamls bein\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 780, "deletions": 23}}, {"node": {"oid": "ca0b8d45c7943d60b58207912dfe0ac761422079", "messageHeadline": "modified how yaml and python functions are added to groups and task r\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 156, "deletions": 57}}, {"node": {"oid": "275857a1eac2f417ce9b0de4132ef34b1825cca3", "messageHeadline": "changes to import and loading yaml", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "2196490cfe3903b73b1aaa491634d8ff3c028f8f", "messageHeadline": "TaskConfig with yaml", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 7, "deletions": 2}}, {"node": {"oid": "92a508560f806b5c451b24c18741810613616e20", "messageHeadline": "Merge remote-tracking branch 'upstream/master' into results", "author": {"name": "Julen Etxaniz", "email": "juletxara@gmail.com", "user": {"login": "juletx"}}, "additions": 1443, "deletions": 270}}, {"node": {"oid": "d145167959c2b1826d900524912cb99c44d5fb30", "messageHeadline": "Add multilingual datasets (XCOPA, XStoryCloze, XWinograd, PAWS-X, XNL\u2026", "author": {"name": "Julen Etxaniz", "email": "juletxara@gmail.com", "user": {"login": "juletx"}}, "additions": 1444, "deletions": 246}}, {"node": {"oid": "af913422cde6c5e31a160a90560b667e6e3b2478", "messageHeadline": "update write out variable name", "author": {"name": "Julen Etxaniz", "email": "juletxara@gmail.com", "user": {"login": "juletx"}}, "additions": 9, "deletions": 13}}, {"node": {"oid": "99b0a42d79f9fe79e3aba3b4d35f79b96f4c3465", "messageHeadline": "update parameter names and add docs", "author": {"name": "Julen Etxaniz", "email": "juletxara@gmail.com", "user": {"login": "juletx"}}, "additions": 39, "deletions": 30}}, {"node": {"oid": "8299ab3b4afb839381d14710114fed03dd1f3c00", "messageHeadline": "process yaml and registered tasks through args.task", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 30, "deletions": 14}}, {"node": {"oid": "41ec3bc4a570c16f92a474467f6b3114b0e66f32", "messageHeadline": "ignore egg-info", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "fc09c8f7a18af0965a083c37541a99f11725a13a", "messageHeadline": "minor fix", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 3}}, {"node": {"oid": "05550ef342762129394b4f36574032dba88e34bc", "messageHeadline": "Merge pull request #490 from jquesnelle/auto-batch-size-fix", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 21, "deletions": 18}}, {"node": {"oid": "f71bffb0ce2c91d193797332fe69c171dbc0c5ad", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "7bf74f16147419d3c381c0919c16add33c4e7c44", "messageHeadline": "update", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2e046ce32c34c32e2d640b52804161687982bf3e", "messageHeadline": "add --write_detailed_eval_info to dump JSON with prompts and completions", "author": {"name": "Julen Etxaniz", "email": "juletxara@gmail.com", "user": {"login": "juletx"}}, "additions": 88, "deletions": 3}}, {"node": {"oid": "52ee80a666d85452b19b689d371b5535dfad96c1", "messageHeadline": "adjusted to adapt to modiefied registry process", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 26, "deletions": 15}}, {"node": {"oid": "4bc837d4cac26bf089dc93c33dda42356d75463d", "messageHeadline": "modified registry process", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 86, "deletions": 4}}, {"node": {"oid": "35730acec7f54867470781a04c8bd9477157381a", "messageHeadline": "moved METRICS related dicts", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 17, "deletions": 32}}, {"node": {"oid": "d9862c587e452cb9542ed1c3bf0d52ffbbd2fa05", "messageHeadline": "moved registry processes here", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 29, "deletions": 5}}, {"node": {"oid": "484fa09043ee2e37b709312dc927b654933dd878", "messageHeadline": "simplify", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 2, "deletions": 5}}, {"node": {"oid": "cf79b1c6fd93206b6c4035430e180868518bf700", "messageHeadline": "configurable can be overidden with newer configs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 13, "deletions": 125}}, {"node": {"oid": "d424f26bce4cfec10c76c1367ecf299e93c10df9", "messageHeadline": "fix adaptive batch crash when there are no new requests (e.g. when pu\u2026", "author": {"name": "Jeffrey Quesnelle", "email": "jq@jeffq.com", "user": {"login": "jquesnelle"}}, "additions": 21, "deletions": 18}}, {"node": {"oid": "8f859cd2bad0dee75801757c0c088b0b153ceee3", "messageHeadline": "balance documents among hosts for wikitext", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 6, "deletions": 4}}, {"node": {"oid": "3cfd23a1c6cfc472f124db1f270c2c1db8f9e2ce", "messageHeadline": "drop comments about pytorch DP", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 2, "deletions": 4}}, {"node": {"oid": "4d3ea67af79918a0f87cf9d1307f8518ee1e4cbc", "messageHeadline": "fix warning issue if single device", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "cac8528147597bd8a0d25efa54cdaced53258e59", "messageHeadline": "alternate step so that we can change num_fewshots", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 6}}, {"node": {"oid": "53a4817e93b522d5d4cd24d3bd0f16f958c0632a", "messageHeadline": "Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evalu\u2026", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 183, "deletions": 389}}, {"node": {"oid": "f6b76f5de75dd95ff3f16f90931f36ddc9ab32c1", "messageHeadline": "Merge pull request #486 from EleutherAI/yaml-parameterize", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 91, "deletions": 11}}, {"node": {"oid": "8fc04fe5ef2d428374db27f77e3ef6ec4dbac0d0", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 11, "deletions": 4}}, {"node": {"oid": "44a4c374ceeaaaed69c034ed0406865eee4691c0", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 12, "deletions": 11}}, {"node": {"oid": "41d16c2bb72c674b209705597afffed5f04f3f11", "messageHeadline": "muliple prompts", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 23, "deletions": 0}}, {"node": {"oid": "f1beac007acfd7dbc846e861f95b51edbc03bbde", "messageHeadline": "save git hash to results dump", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 15, "deletions": 1}}, {"node": {"oid": "38244e157d1664fc5c8adb5ce1776c03b90dbe84", "messageHeadline": "add yaml registering decorator", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 76, "deletions": 10}}, {"node": {"oid": "95642aa63123632e835bb5bcb3e52ee1c66e2d12", "messageHeadline": "add mutual info, code cleanup", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 89, "deletions": 336}}, {"node": {"oid": "c473d7e0e989cd5fadd5ba61cd20d4fe9572f8cb", "messageHeadline": "Create output path directory if necessary", "author": {"name": "janEbert", "email": "janpublicebert@posteo.net", "user": {"login": "janEbert"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "3226ed648c455fb69c49fc4c95f2ab79b0a7a611", "messageHeadline": "Add perplexity task on arbitrary JSON data", "author": {"name": "janEbert", "email": "janpublicebert@posteo.net", "user": {"login": "janEbert"}}, "additions": 129, "deletions": 1}}, {"node": {"oid": "46f37ec002b92dbbdc17d26456f7b9a301ba13a8", "messageHeadline": "warning if launched incorrectly on multiGPU machine", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 19, "deletions": 8}}, {"node": {"oid": "c4447dabdf60467bdc8903029c25b0e2de428c99", "messageHeadline": "clean up cumul batches for rolling + remove unused print", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 4, "deletions": 6}}, {"node": {"oid": "e64924412be899d1bdf8f1d45bc0f517635cdb05", "messageHeadline": "world_size condition for padding", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "074099f902d3a42651cbca65200fa0063acfb756", "messageHeadline": "clarify comments around padding", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 4, "deletions": 5}}, {"node": {"oid": "afbf8e6639c86490cc086eb949eac89b55ce58ab", "messageHeadline": "add more mpt and llama results", "author": {"name": "Julen Etxaniz", "email": "juletxara@gmail.com", "user": {"login": "juletx"}}, "additions": 1800, "deletions": 0}}, {"node": {"oid": "082d6db3ffcbebb4f354b12ad1756ba5a971f742", "messageHeadline": "Merge pull request #448 from nikhilpinnaparaju/issue-439", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "3fda1195e183f95f8da574b4c3a911c49a10f150", "messageHeadline": "allow float limit to represent data portion", "author": {"name": "Ken Tsui", "email": "kenhktsui@gmail.com", "user": {"login": "kenhktsui"}}, "additions": 8, "deletions": 3}}, {"node": {"oid": "0e849758b42dfa5b82ffd82c7f3cea5854db0c81", "messageHeadline": "update task table with missing tasks", "author": {"name": "Julen Etxaniz", "email": "juletxara@gmail.com", "user": {"login": "juletx"}}, "additions": 51, "deletions": 7}}, {"node": {"oid": "8a7077017886d5829a7c6a9265babcfdad2c707a", "messageHeadline": "add opt and mpt model results", "author": {"name": "Julen Etxaniz", "email": "juletxara@gmail.com", "user": {"login": "juletx"}}, "additions": 5406, "deletions": 0}}, {"node": {"oid": "2ac318a98db31b93c1a10c210a0b3cea7582290a", "messageHeadline": "add basic markdown tables with results", "author": {"name": "Julen Etxaniz", "email": "juletxara@gmail.com", "user": {"login": "juletx"}}, "additions": 2718, "deletions": 0}}, {"node": {"oid": "14043a0f762702bbb356073f50c066bcd9b275b2", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 6, "deletions": 0}}, {"node": {"oid": "21e128d801300b2ca9f466efb2331049d7e2593a", "messageHeadline": "add bloom, xglm and llama results", "author": {"name": "Julen Etxaniz", "email": "juletxara@gmail.com", "user": {"login": "juletx"}}, "additions": 11140, "deletions": 1}}, {"node": {"oid": "32d7593dc02fe9699e1849a641beca199aa9f109", "messageHeadline": "partial support for document-level padding", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 66, "deletions": 16}}, {"node": {"oid": "a8d9a449d978bcf857cce26d3b28235bf5e19d5a", "messageHeadline": "assert + help message if script called incorrectly with multigpu", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "159c1a9328f1d9b909fd72125a3ed985ac031b6f", "messageHeadline": "limit for doc iterator", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a7d6cb2dccbbabaaee987c911b26f17bba369920", "messageHeadline": "record _rank/_world_size for single GPU", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "0542d35d5e56768dd9041ef9b88b90256970d843", "messageHeadline": "Sort task names to keep the same order always (#474)", "author": {"name": "Julen Etxaniz", "email": "juletxara@gmail.com", "user": {"login": "juletx"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "eddd627a1c6df6d8b5602017f02d4fd59777ae36", "messageHeadline": "add utility func for slicing iterators, only tqdm on main process", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 17, "deletions": 11}}, {"node": {"oid": "629bcfba2ced04ceffc5848c567837bb7d97f9aa", "messageHeadline": "bugfixes missed from local branch", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 15, "deletions": 13}}, {"node": {"oid": "09e91431ced3fc626a3511a0904a9ae850ec92a2", "messageHeadline": "accelerate added to requirements in setup.py", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "d4c5315a3402e79eb4ba316cb1420b66f26d21cf", "messageHeadline": "sync working changes with upstream", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 153, "deletions": 75}}, {"node": {"oid": "bdc1af905826d15006e772b8ebe128e5841fe68e", "messageHeadline": "added comment", "author": {"name": "Nikhil Pinnaparaju", "email": "nikhilpinnaparaju@gmail.com", "user": {"login": "nikhilpinnaparaju"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "3c210d4e4e0f965c49c642410f43958ae7588c30", "messageHeadline": "Undo minor version increment", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f6fb399a6eec83c3477d09eb6df36753f9315de0", "messageHeadline": "Bump the Python requirement to 3.9", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "f4355f932f23a28e3d5478c2f2fd758178dcf670", "messageHeadline": "Merge pull request #472 from juletx/patch-1", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 4, "deletions": 6}}, {"node": {"oid": "cede13c6107af04829476275d4fb443af20edebc", "messageHeadline": "Fix bugs introduced in #394 #406 and max length bug", "author": {"name": "Julen Etxaniz", "email": "juletxara@gmail.com", "user": {"login": "juletx"}}, "additions": 4, "deletions": 6}}, {"node": {"oid": "9a87719720d2f8b8e79d8ec1fa0e493688e0c671", "messageHeadline": "Merge pull request #394 from fattorib/auto-batching", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 92, "deletions": 19}}, {"node": {"oid": "d6ceced5ee81a0afae7ffcd8158abe273f044814", "messageHeadline": "Merge branch 'master' into auto-batching", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 699827, "deletions": 600}}, {"node": {"oid": "05e97d8482fab38651469dd4763ea327aba64fe6", "messageHeadline": "modify layout of yaml vs vanilla tasks", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 9, "deletions": 48}}, {"node": {"oid": "2da74953dee265e347298a788b4fd09f2c0344c2", "messageHeadline": "Merge pull request #438 from EleutherAI/configurable-tasks", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 478, "deletions": 116}}, {"node": {"oid": "d2b167579f39261778576f027f2d4a97e81722c8", "messageHeadline": "Merge branch 'big-refactor' into configurable-tasks", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 15, "deletions": 13}}, {"node": {"oid": "1f23061b80795f255e056a51925cd795d58ccc4e", "messageHeadline": "Merge pull request #467 from EleutherAI/configurable-mcqa-ppl", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 330, "deletions": 67}}, {"node": {"oid": "337419b842312bbade17b4999412561946071771", "messageHeadline": "add configurable ppl task", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 194, "deletions": 8}}, {"node": {"oid": "487f7811a864faf401989f9ce8ebe129124e2c4b", "messageHeadline": "prelim. multiple choice support", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 140, "deletions": 63}}, {"node": {"oid": "4d21ab6b2713cc3a8b4fa7574e89c62ef504e75f", "messageHeadline": "address PR comments", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 22, "deletions": 2}}, {"node": {"oid": "fa686d04171c5c25755f3bd8c045ffd9795d24a9", "messageHeadline": "Merge pull request #465 from EleutherAI/wip-hailey-refactor", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 14, "deletions": 12}}, {"node": {"oid": "9d8e0532f92522633794f6d307b4849f9ada4efb", "messageHeadline": "change id_ to idx in instance", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 14, "deletions": 12}}, {"node": {"oid": "fc4428dc47a099340ea19bf752d0a8b0f3e298f7", "messageHeadline": "Removing tests as they seem to not be working correctly.", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 63}}, {"node": {"oid": "9dd22497c17294f5ba84fe8d8f943d7ff27d895e", "messageHeadline": "added newline to fix precommit check breaking", "author": {"name": "Nikhil Pinnaparaju", "email": "nikhilpinnaparaju@gmail.com", "user": {"login": "nikhilpinnaparaju"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "22b7fdc5babff58320ca5096b470454e065c744f", "messageHeadline": "alternative more elegant way of setting pad token for gpt2 generate", "author": {"name": "Nikhil Pinnaparaju", "email": "nikhilpinnaparaju@gmail.com", "user": {"login": "nikhilpinnaparaju"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "1b604e16636cdcf7111b14d7dead2fbc767e789f", "messageHeadline": "Use EOS token as PAD token to get rid of huggingface model warning", "author": {"name": "Nikhil Pinnaparaju", "email": "nikhilpinnaparaju@gmail.com", "user": {"login": "nikhilpinnaparaju"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "dd9fb896eea80933627b519df1fe47b0f45fe4a0", "messageHeadline": "Merge pull request #447 from nikhilpinnaparaju/issue-437", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "7376b0fd0c83b43d608b93666c81059dd7b1bf90", "messageHeadline": "Updated handling for device in lm_eval/models/gpt2.py", "author": {"name": "Nikhil Pinnaparaju", "email": "nikhilpinnaparaju@gmail.com", "user": {"login": "nikhilpinnaparaju"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "602abceb8ae066b1f9cc8071d0c91009d5f97be1", "messageHeadline": "Add non-programmatic BIG-bench-hard tasks (#406)", "author": {"name": "yurodiviy", "email": "44850998+yurodiviy@users.noreply.github.com", "user": {"login": "yurodiviy"}}, "additions": 699688, "deletions": 19}}, {"node": {"oid": "e7f49cca7f4598d9997903f7fc751b8fd6269909", "messageHeadline": "updated yaml", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 16, "deletions": 12}}, {"node": {"oid": "3ec68d388ed5450bd9516de3155cf0b833772873", "messageHeadline": "added opt8ion to use prompts, removed typos on metric kwargs", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 10, "deletions": 5}}, {"node": {"oid": "cea713dc053affac6aa04e7e29a9099934ea6a24", "messageHeadline": "can process looglikelihood requests", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 41, "deletions": 19}}, {"node": {"oid": "84191b83bd4dd94df9fa67092b2291c850ae1631", "messageHeadline": "registries for metrics", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 25, "deletions": 0}}, {"node": {"oid": "7ad96ded94e3177084231f7c1554db99d8130d1d", "messageHeadline": "yaml for lambada and sglue", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 28, "deletions": 0}}, {"node": {"oid": "33b01a2b84217b81ce8a5864247ef75b124dda3f", "messageHeadline": "metric list easier to define args", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 6, "deletions": 7}}, {"node": {"oid": "f107ae296092dbbb3479cb02b23d494c35e360d7", "messageHeadline": "ported changes here", "author": {"name": "lintangsutawika", "email": "lintang@sutawika.com", "user": {"login": "lintangsutawika"}}, "additions": 33, "deletions": 17}}, {"node": {"oid": "e47e01beea79cfe87421e2dac49e64d499c240b4", "messageHeadline": "Merge pull request #435 from EleutherAI/haileyschoelkopf-patch-1", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "91b8ff6b7e6026daa0e22b93f8f6c2cbca5c7e8c", "messageHeadline": "document wildcard support in README", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "2a9da9fb3588ca6ed6aeea4aceca6d43623e9f94", "messageHeadline": "add metric + agg registries", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 91, "deletions": 45}}, {"node": {"oid": "460584ca6bd8d195ba06c4cff217652c44311861", "messageHeadline": "add template_vars + make jinja rendering stricter", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 18, "deletions": 10}}, {"node": {"oid": "f275301a093851aca80750418fe2e5f42bc6f98c", "messageHeadline": "make tasks and models registered by decorators", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 432, "deletions": 317}}, {"node": {"oid": "bdd8cc72651b95d24439f44e629a4e0fbf83f3f1", "messageHeadline": "Update README.md", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6179fdf5b4dcacffb33b5f64306aa63d5a67356e", "messageHeadline": "Merge pull request #427 from EleutherAI/hotfix-hf-causal", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 33, "deletions": 31}}, {"node": {"oid": "62d7937be852a48645b5b2b1a3d975edd52f74b3", "messageHeadline": "document changes in readme", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "442d47b715c6834306c04f794190243e89fb5d11", "messageHeadline": "hotfix: make greedy_until work for Accelerate HF model", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 31, "deletions": 29}}, {"node": {"oid": "e7c18e5319502d8ec95c0f7e8352c60da4fe1abf", "messageHeadline": "refactor Instance dataclass", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 12, "deletions": 61}}, {"node": {"oid": "977b281aa70c1f7c3b6fbc2a62ac046f4da9b342", "messageHeadline": "Add support for load_in_8bit and trust_remote_code model params (#422)", "author": {"name": "Phil Wee", "email": "pkw2010@nyu.edu", "user": {"login": "philwee"}}, "additions": 42, "deletions": 17}}, {"node": {"oid": "221186c7aeaa977978a335615bc84023216d3254", "messageHeadline": "Merge pull request #418 from zanussbaum/master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "1881deb678976fdb3a7068655c4235d5bc539101", "messageHeadline": "fix: add seq2seq peft", "author": {"name": "Zach Nussbaum", "email": "zanussbaum@gmail.com", "user": {"login": "zanussbaum"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "d2a9b759186c08254b7e156a43bd473619460239", "messageHeadline": "in-place replace main with lm-eval2, keeping old git history", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 1779, "deletions": 11437}}, {"node": {"oid": "814940e87ebc1059d2cc00438a1fc9c6b510e78c", "messageHeadline": "Merge pull request #416 from trail-of-forks/fix-setup", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "83d45912203a5f9a1f61ec07abd2f7ae0b9536e6", "messageHeadline": "Update setup.py dependencies", "author": {"name": "Riccardo Schirone", "email": "riccardo.schirone@trailofbits.com", "user": {"login": "ret2libc"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b315725ab43cebbffe2e5282cba2aab090b9f814", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "5c95b9070a9c3134a71dbd1b2ec671fcb90d1c17", "messageHeadline": "Merge pull request #414 from zanussbaum/master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 54, "deletions": 1}}, {"node": {"oid": "83c2ff169f8349c3ed1d281ea75994d396ce0b03", "messageHeadline": "Merge branch 'master' into master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "7548311a4791ab11ce9f1009acf2bd3c0425661e", "messageHeadline": "Cleaned up wording a little", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "69826e2c45a0da1b27cea71492cd3a6220a3fbdc", "messageHeadline": "docs: update readme", "author": {"name": "Zach Nussbaum", "email": "zanussbaum@gmail.com", "user": {"login": "zanussbaum"}}, "additions": 10, "deletions": 0}}, {"node": {"oid": "9142999ebc94315875dfa8bcbad9de4677d58506", "messageHeadline": "update type arguments for hf-causal to accept str for batch_size", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e87c0838a01fdad1ff818bdb6bd2449ceabd3858", "messageHeadline": "compute multiple forward passes in autobatcher to improve robustness \u2026", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 5, "deletions": 7}}, {"node": {"oid": "44275ae98f0d70ef2bf8a8cf164a0f5e3874ce53", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "fcbe193e328612582492db86d5eb7879f8ef67f2", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "94ce276c3cc0672d8add805c5f9bfa009d0f0165", "messageHeadline": "fix: reqs for peft + sentencepiece tokenizers (llama model)", "author": {"name": "Zach Nussbaum", "email": "zanussbaum@gmail.com", "user": {"login": "zanussbaum"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "2b9a6e94a3d071c6281f2b50c5237a9ed62839d0", "messageHeadline": "feat: peft model evals", "author": {"name": "Zach Nussbaum", "email": "zanussbaum@gmail.com", "user": {"login": "zanussbaum"}}, "additions": 41, "deletions": 0}}, {"node": {"oid": "d86de51d08718a7bd9df45903c5ded16f22147a7", "messageHeadline": "remove double assign typo of test_batch", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "42c6b7dfc120e3cb660355436e309f3a891c535a", "messageHeadline": "additional external call to empty_cache + gc collect", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 11, "deletions": 8}}, {"node": {"oid": "50eb80ba5f23bde87cb80bbe1f1633bb98038e58", "messageHeadline": "Merge pull request #407 from EleutherAI/fix-install-instructions", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 7, "deletions": 3}}, {"node": {"oid": "8b9c0da5cc0915f379f545e135ec0093bc2d9565", "messageHeadline": "update installation instructions", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 7, "deletions": 3}}, {"node": {"oid": "ccc31842f68a52b49186e55a6c4bfee79881d3c3", "messageHeadline": "Merge pull request #403 from Jeffwan/fix_readme", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "92e5632e53a01642827b17df924b6fc2f0a6c2b4", "messageHeadline": "Change device parameter to cuda:0 to avoid runtime error", "author": {"name": "Jiaxin Shan", "email": "seedjeffwan@gmail.com", "user": {"login": "Jeffwan"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "99304fe585c39d8997dfdf0058fa199a94b58081", "messageHeadline": "wrap self._model_call with F.log_softmax + remove empty_cache, not ne\u2026", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 2, "deletions": 4}}, {"node": {"oid": "8a89b30cf70b6242e35188c9192c41cf4e6cc34d", "messageHeadline": "empty cuda cache after determing largest possible batch size", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "97f936beff2af5b56762147085a8857c72576e61", "messageHeadline": "add accelerate to deps", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "83dbfbf6070324f3e5872f63e49d49ff7ef4c9b3", "messageHeadline": "Hosting wikitext on HuggingFace  (#396)", "author": {"name": "ben", "email": "46356259+fattorib@users.noreply.github.com", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 256}}, {"node": {"oid": "16ac11af53f038dfe224cc0fd4bcee97efb1c428", "messageHeadline": "Merge pull request #391 from fattorib/track-upstream-master", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 1, "deletions": 220}}, {"node": {"oid": "f1dedbf07b75c7908e91684c96d129f963cfc3b3", "messageHeadline": "ll_rolling computes adaptive bs separately and passes computed bs to \u2026", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 34, "deletions": 14}}, {"node": {"oid": "7502ff3d82dcfc89716d0eb918521331dbddb995", "messageHeadline": " transfer data to eleuther HF org", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b824fc9154a498bc11af6493d0d81c4d1ed0c955", "messageHeadline": "accelerate import not needed", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "d5720d5f7977ef8a02149a2ea580ec361dda0772", "messageHeadline": "single GPU automatic batching logic", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 36, "deletions": 13}}, {"node": {"oid": "945a5b010428e108bbd4146f5d8e864e8aa9898f", "messageHeadline": "Merge branch 'EleutherAI:master' into track-upstream-master", "author": {"name": "ben", "email": "46356259+fattorib@users.noreply.github.com", "user": {"login": "fattorib"}}, "additions": 653, "deletions": 9}}, {"node": {"oid": "db5013873ecda91a2645cd36bac0e95a015841f3", "messageHeadline": "inspect import not needed", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "2e522e2cb96d5fc56aad2c39575b19159eaadebf", "messageHeadline": "Merge pull request #381 from EleutherAI/bigscience_refactor", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 653, "deletions": 9}}, {"node": {"oid": "d09f32ce55eb7ab6058728feb673a9dcf6a0feec", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 14, "deletions": 9}}, {"node": {"oid": "1581db0b384806df7ad9e87a4450b782497b4c58", "messageHeadline": "remove unused import for eval-harness", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "3c8479babdd9ba46c13e38396644874286e05a43", "messageHeadline": "arithmetic task class now uses HF remote builder", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f022d30330971047588fa592bda8028fdd7582ce", "messageHeadline": "remove old dataset builder script under datasets for arithmetic", "author": {"name": "Benjamin Fattori", "email": "ben.fattori@mail.utoronto.ca", "user": {"login": "fattorib"}}, "additions": 0, "deletions": 217}}, {"node": {"oid": "9bb0de9c07fb322ca93638dc01835e14afc313ee", "messageHeadline": "Merge pull request #390 from sxjscience/master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 7, "deletions": 7}}, {"node": {"oid": "859f96fd01cdc06668764f3ee748a1dd307a3fb0", "messageHeadline": "enable \"low_cpu_mem_usage\"", "author": {"name": "Xingjian Shi", "email": "xshiab@connect.ust.hk", "user": {"login": "sxjscience"}}, "additions": 7, "deletions": 7}}, {"node": {"oid": "3d14707a1054e174553d7e4ec99d1cb8905735bb", "messageHeadline": "Merge pull request #384 from jon-tow/gpt2-vocab-slice", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1a545447178898aa3f07b413c982964b12cb80e5", "messageHeadline": "hotfix(gpt2): Remove vocab-size logits slice", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f9eca2c8160be8c20ecc956b7ff545f880160d0e", "messageHeadline": "Add accuracy metric to crows-pairs (#380)", "author": {"name": "Hailey Schoelkopf", "email": "65563625+haileyschoelkopf@users.noreply.github.com", "user": {"login": "haileyschoelkopf"}}, "additions": 30, "deletions": 26}}, {"node": {"oid": "ea3df930592dd92467be4a917291182697f9c914", "messageHeadline": "Added CrowSPairs (#379)", "author": {"name": "Aflah", "email": "72096386+aflah02@users.noreply.github.com", "user": {"login": "aflah02"}}, "additions": 309, "deletions": 0}}, {"node": {"oid": "db1f55ff01034ffc22e21b5e0e2af72f177f0256", "messageHeadline": "first step: add HF+accelerate implementations", "author": {"name": "haileyschoelkopf", "email": "hailey.schoelkopf@yale.edu", "user": {"login": "haileyschoelkopf"}}, "additions": 639, "deletions": 0}}, {"node": {"oid": "2578030794013e8312fbf695b7efdc914dc85285", "messageHeadline": "Update CODEOWNERS", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f079e322b857714fcef1ada9e78ddc606fe51e84", "messageHeadline": "Added ToxiGen task (#377)", "author": {"name": "Tom Hartvigsen", "email": "tomh@mit.edu", "user": {"login": "Thartvigsen"}}, "additions": 74, "deletions": 0}}, {"node": {"oid": "8c048e266a22a1c85ccbdb0c209ac712e4f39989", "messageHeadline": "Update Pile Test/Val Download URLs (#373)", "author": {"name": "ben", "email": "46356259+fattorib@users.noreply.github.com", "user": {"login": "fattorib"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "f29597bea99352ce426b538458071cbaa76537b1", "messageHeadline": "Merge pull request #370 from jon-tow/update-lambada-openai", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 3, "deletions": 140}}, {"node": {"oid": "2bae14d20daff8d1a72846e0b9b1988d6741bb65", "messageHeadline": "Update `lambada_openai` multilingual data source", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 140}}, {"node": {"oid": "57f8ff24ca492b4cbd9c86fd9db7cfa519d79284", "messageHeadline": "Merge pull request #366 from jon-tow/bump-triviaqa-version", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "f7aeeff24c384175efbc03b32fbd5aeb843f7d05", "messageHeadline": "Bump `triviaqa` version", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "3035550d09f5a6906591ad802cf0e807bd0749f0", "messageHeadline": "Update `actions/setup-python`in  CI workflows (#365)", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "0742f4d7a5337855914e24d821484653f2cae65f", "messageHeadline": "Merge pull request #364 from jon-tow/triviaqa-dataset", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 50, "deletions": 6}}, {"node": {"oid": "3aff8606ef62bf182970172b08a188bb443434dc", "messageHeadline": "Run pre-commit", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 7, "deletions": 3}}, {"node": {"oid": "299e950596b14d67eade9c0180fdd193d7106fa2", "messageHeadline": "Replace stale `triviaqa` dataset link", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 46, "deletions": 6}}, {"node": {"oid": "62ca18400ebe0fc0dbb14274b27170d2d5ae9e3d", "messageHeadline": "Bump version to 0.3.0", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1d8107bfe1c5bcd67f8317ad88568cb2a8fc7ae8", "messageHeadline": "Merge pull request #362 from EleutherAI/cleanup-for-release", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 434, "deletions": 440}}, {"node": {"oid": "1e5d55d9c5377a0184b3b7221f3393cf333c1386", "messageHeadline": "Remove redefinition of `jieba`", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "0699d7741a7fb0be027d345bc3208e99a8a16ac8", "messageHeadline": "Cleanup `README.md` and package deps", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 435, "deletions": 439}}, {"node": {"oid": "fdd3dbc3b2871d6a0a5433bfa42f56490ee058ab", "messageHeadline": "Merge pull request #357 from jon-tow/add-lambada-orig", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 132, "deletions": 56}}, {"node": {"oid": "1386065dddd95f54b4bebb6a4015a218cf4947f9", "messageHeadline": "Run pre-commit", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8f442ec12468b1ebed2b88b4e83bf5f7ef02487e", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-\u2026", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "9ec03596ece78f9ffdadc272286d40b04c011119", "messageHeadline": "Merge pull request #358 from jon-tow/update-pre-commit", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0bca43033239b6f4097b04a741831105b8f75343", "messageHeadline": "Revert \"Remove old `LAMBADA` dataset\"", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 131, "deletions": 0}}, {"node": {"oid": "63bdb4b07c3366a99a60ff59c2fa528f96e9d2ee", "messageHeadline": "Remove old `LAMBADA` dataset", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 131}}, {"node": {"oid": "97f752137a925e0c90a18e99fbc6f9f31a7ae194", "messageHeadline": "Remove unsupported `gitlab` link in pre-commit config", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "bfa57687e48d44782e376fe5def71c9d8c8aa472", "messageHeadline": "Add the original `LAMBADA` dataset", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 132, "deletions": 56}}, {"node": {"oid": "4f0410a4be0049729078376ce36a42dc308b6e38", "messageHeadline": "Add `TextSynth` API (#299)", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 378, "deletions": 186}}, {"node": {"oid": "ddd97430d2dd492b66643c46fa10d30f62be1460", "messageHeadline": "Merge pull request #330 from jon-tow/remove-dataset", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 4, "deletions": 288}}, {"node": {"oid": "80dbfcbd76f77898b19b12132328b4d821b44e33", "messageHeadline": "Merge pull request #352 from jon-tow/update-deps", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8c69f17e57f22577888b75369d5d8b2a160fa9d4", "messageHeadline": "[deps] Use minimum versioning for `numexpr`", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2598f990372e17f39ca432b03f0c279f6fa6118b", "messageHeadline": "Merge pull request #343 from EleutherAI/fix-tokenizer-init", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "3dc27bd2d6cd0e3843cb9ba8021e80217f050edc", "messageHeadline": "Remove stale tokenizer comment", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "5b1becebf30adc44ecd542e1a5f03cd828448e1b", "messageHeadline": "Move `TODO`s into one source location", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 5, "deletions": 4}}, {"node": {"oid": "ff83e9b2269a03c07e372fda064f89c707fde402", "messageHeadline": "Manually concat tokenizer revision with subfolder", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "b0b76d87fc84b2bfdfa88b71ce4592f35f707428", "messageHeadline": "Merge pull request #336 from richhankins/fix_disjoint", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "834de32fc4e864fe3acbf815be64dd712891f590", "messageHeadline": "Fix make_disjoint_window for tail case", "author": {"name": "Rich Hankins", "email": "rich@augmentcode.com", "user": {"login": "richhankins"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "13ed134354bc7c3da455b5876875d2b3d68e50db", "messageHeadline": "Merge pull request #333 from OpenGPTX/squad_fix_keyerror", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1a1531857bc2e2756dfc8ca7c24e88c372b544aa", "messageHeadline": "fix key access in squad evaluation metrics", "author": {"name": "Konstantin Schulz", "email": "Konstantin.Schulz@dfki.de", "user": null}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4645584fb92c77ccd7a1835bdfd70bfa04f7e83f", "messageHeadline": "Merge pull request #331 from EleutherAI/researcher2-patch-1", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 14, "deletions": 15}}, {"node": {"oid": "a95f4b56888a33c7d52d4ea3c882a2c543734f11", "messageHeadline": "Update decontamination.md", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 14, "deletions": 15}}, {"node": {"oid": "1f8a8c1dc95a475f0b6fd3cc082e89a674c072d7", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-\u2026", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 4499, "deletions": 2807}}, {"node": {"oid": "b4c0275dca8f74e5023cf48638c05e6ec00e29d0", "messageHeadline": "Remove custom datasets that are in HF", "author": {"name": "jon-tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 4, "deletions": 278}}, {"node": {"oid": "b0acb3379d2fa8e15561cea033be422bff144f30", "messageHeadline": "Merge pull request #325 from EleutherAI/update-datasets-readme", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 5, "deletions": 3}}, {"node": {"oid": "f1c963976dbd957c2badaaa28ad39719a3b86839", "messageHeadline": "Add EOF newline", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 1, "deletions": 1}}, {"node": {"oid": "3798515de9d33ddc96f0f5f4d9720ef2ffec2728", "messageHeadline": "Update datasets/README.md", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 5, "deletions": 3}}, {"node": {"oid": "6174e428d0a3ed5ad66813a0793a7a4da212c1cf", "messageHeadline": "Merge pull request #322 from Muennighoff/master", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "9baa294c0d925896f267063f97e6c2666ef4d89f", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-\u2026", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 4, "deletions": 5}}, {"node": {"oid": "e0396a4e123a5c58c3d50150b49e3d69ab3a07ce", "messageHeadline": "Merge pull request #321 from EleutherAI/decontam-import-fix", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 4, "deletions": 5}}, {"node": {"oid": "4587b7186b9c72f060cadc56041b984f5952d3f1", "messageHeadline": "Add bootstrap_iters kwarg", "author": {"name": "Niklas Muennighoff", "email": "n.muennighoff@gmail.com", "user": {"login": "Muennighoff"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "07ba1e683e5694697d8e9dcdd80526dcc28b1b9e", "messageHeadline": "Format evaluator", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 1, "deletions": 0}}, {"node": {"oid": "f81db68e8714da34f7d2e15c02cca430bdeba49e", "messageHeadline": "Fix undeclared identifier error", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8d5b116ce21e67431852d95d3d72f64c90a43df0", "messageHeadline": "Fix `ModuleNotFoundError` in evaluator", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 2, "deletions": 4}}, {"node": {"oid": "a2cada5d8e1534c636a6bd43ae98eb16eb6240a7", "messageHeadline": "Merge pull request #317 from EleutherAI/Mistobaan/add-pre-commit", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 3635, "deletions": 2855}}, {"node": {"oid": "83507c4bb7f2abba8e5d15fca9faf7d9582433d3", "messageHeadline": "Update `task_guide` and add `pre-commit` dev dependency", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 7, "deletions": 2}}, {"node": {"oid": "8c997e532c7b578e8c304a6af2ac31546f9d8d16", "messageHeadline": "Revert `tests/testdata` changes and address flake8 issues", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 655, "deletions": 633}}, {"node": {"oid": "d95a433304c50ccdf526cc1efe309fe202bee00d", "messageHeadline": "fix codespell", "author": {"name": "Fabrizio Milo", "email": "mistobaan@gmail.com", "user": {"login": "Mistobaan"}}, "additions": 39, "deletions": 36}}, {"node": {"oid": "121b7096ab608a3ef8a73957c0f6efae053b5f15", "messageHeadline": "add pre-commit", "author": {"name": "Fabrizio Milo", "email": "mistobaan@gmail.com", "user": {"login": "Mistobaan"}}, "additions": 4180, "deletions": 3430}}, {"node": {"oid": "7a03811879c7677e4a2c66dff6374ea2ddafb189", "messageHeadline": "Merge pull request #301 from StephenHogg/task_doc", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 11, "deletions": 2}}, {"node": {"oid": "11f614b0138f165441d90deab89d4f6476f7531c", "messageHeadline": "Merge branch 'master' into task_doc", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 4995, "deletions": 1376}}, {"node": {"oid": "e00d682f4daa4b36921069218681384c623616df", "messageHeadline": "Merge pull request #261 from EleutherAI/researcher2", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 1072, "deletions": 161}}, {"node": {"oid": "eb8163e96682aaf8899e7ab62dc7d49d8f2df83d", "messageHeadline": "Merge pull request #315 from jon-tow/unpin-datasets", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "cf074822dfe044103eef02ac285dd906dda29a7d", "messageHeadline": "Merge pull request #316 from jon-tow/master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1123, "deletions": 3073}}, {"node": {"oid": "7585ec563b51741e2c0d1dc519016756c2d91e8c", "messageHeadline": "Revert \"Merge branch 'master' into master\"", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 1123, "deletions": 3073}}, {"node": {"oid": "5fe7e2c09eadf4fcdfc5db9dbbfa29777431ac45", "messageHeadline": "Merge branch 'master' into master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 3073, "deletions": 1123}}, {"node": {"oid": "73ca0545b5068055ea15c97bc71f9a65426adf0c", "messageHeadline": "Unpin `datasets`", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 1, "deletions": 1}}, {"node": {"oid": "37afce3b92f2ba34f5b5a147422d3269cb0649be", "messageHeadline": "Merge pull request #32 from bigscience-workshop/kkawamu1/gem_xsum", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "f4600a9f51df2db218f029d4863506c47ff47918", "messageHeadline": "Merge pull request #31 from bigscience-workshop/max-gen-fix", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "7dd1ef9599adc4f169f415b1cc37387ecb4fd2ef", "messageHeadline": "Fix naming", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 1, "deletions": 1}}, {"node": {"oid": "372ca6f511ceac5d21327d0575d6dc4c8f190ccf", "messageHeadline": "Merge pull request #20 from JanKalo/master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 304, "deletions": 0}}, {"node": {"oid": "2e0b659a9ead8a862cbf321d8294d40e4e62153c", "messageHeadline": "Merge pull request #27 from bigscience-workshop/add_HuffPost", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 58, "deletions": 0}}, {"node": {"oid": "9af7387259127c7b449e717a079f46e96a8c0c4a", "messageHeadline": "Fix max generation length", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2c16579716374fc13d19e550e009b9f5e5c435e2", "messageHeadline": "integrate HuffPost to EvalHarness", "author": {"name": "KhalidAlt", "email": "ekh@Protonmail.com", "user": {"login": "KhalidAlt"}}, "additions": 58, "deletions": 0}}, {"node": {"oid": "49f117ed2f58ab48c45cc3540387acdfe7461739", "messageHeadline": "Update lama.py", "author": {"name": "KhalidAlt", "email": "49135754+KhalidAlt@users.noreply.github.com", "user": {"login": "KhalidAlt"}}, "additions": 22, "deletions": 20}}, {"node": {"oid": "ad23a8613e699c17111e47f216f010c414539f98", "messageHeadline": "Merge pull request #26 from bigscience-workshop/max-gen-fix", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 17, "deletions": 118}}, {"node": {"oid": "674e05641fbbe7e8912d90fcfbb27fa924b450d5", "messageHeadline": "Update lama.py", "author": {"name": "KhalidAlt", "email": "49135754+KhalidAlt@users.noreply.github.com", "user": {"login": "KhalidAlt"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "eda365f6d4b0fc4250f426a0968ea0039601b82f", "messageHeadline": "Fix max generation limit", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 17, "deletions": 118}}, {"node": {"oid": "1444a36c677058c8b1e810a31fc8b324719ecce8", "messageHeadline": "Merge branch 'master' into master", "author": {"name": "KhalidAlt", "email": "49135754+KhalidAlt@users.noreply.github.com", "user": {"login": "KhalidAlt"}}, "additions": 138, "deletions": 37}}, {"node": {"oid": "1367690543139e3b05e49a8a120d2c539f96d76b", "messageHeadline": "Update __init__.py", "author": {"name": "KhalidAlt", "email": "49135754+KhalidAlt@users.noreply.github.com", "user": {"login": "KhalidAlt"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "00903bc3395406b94755f01d6fc03c8945305734", "messageHeadline": "Update __init__.py", "author": {"name": "KhalidAlt", "email": "49135754+KhalidAlt@users.noreply.github.com", "user": {"login": "KhalidAlt"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "6559ba0f17a23b5ca21833c6ef25fccc2239d6c7", "messageHeadline": "Merge branch 'add_lama' into master", "author": {"name": "KhalidAlt", "email": "49135754+KhalidAlt@users.noreply.github.com", "user": {"login": "KhalidAlt"}}, "additions": 250, "deletions": 0}}, {"node": {"oid": "14b8ec7ec3996a7c7906e2a2691bcfd369e54e1b", "messageHeadline": "Merge pull request #309 from dirkgr/PytestUpdate", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 17, "deletions": 20}}, {"node": {"oid": "22155f7d02b5a7b3fc95a2b325ec0d43eb7b20dd", "messageHeadline": "Merge pull request #25 from oskarvanderwal/master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 70, "deletions": 0}}, {"node": {"oid": "e9f8f654da55785a34108397c6981175c0cf2e0f", "messageHeadline": "Update `task_guide.md` instructions for dev environ setup", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 1, "deletions": 1}}, {"node": {"oid": "bc9f6fc59e98eaf24e6a5ad58ea90241dd824c9a", "messageHeadline": "Rename all `_load_doc`s to `_process_doc`", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 5, "deletions": 5}}, {"node": {"oid": "0816eba0733a596b1e7cb599415a0d098f244b01", "messageHeadline": "Rename bigsciencelama.py to lama.py", "author": {"name": "KhalidAlt", "email": "49135754+KhalidAlt@users.noreply.github.com", "user": {"login": "KhalidAlt"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "827e2d511baaee53ebf8dbc0b6308f83abf48252", "messageHeadline": "Merge pull request #306 from jon-tow/add-swag", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 63, "deletions": 0}}, {"node": {"oid": "1da3d71932b3f1c31af4b8deff87a3befeeb5dce", "messageHeadline": "Merge pull request #305 from jon-tow/add-triviaqa-dataset-features", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 34, "deletions": 9}}, {"node": {"oid": "235f8d3fb33edfcd2062a605d2c095fcf6f65b19", "messageHeadline": "Merge pull request #308 from jon-tow/add-templates", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 274, "deletions": 63}}, {"node": {"oid": "23ddba77da4b9325c3aebd0abf28107c07c62098", "messageHeadline": "Update CODEOWNERS", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "073b0808adca794bab1dd0ceccfea63753fb32da", "messageHeadline": "Merge branch 'bigscience-workshop:master' into master", "author": {"name": "Oskar van der Wal", "email": "56364990+oskarvanderwal@users.noreply.github.com", "user": {"login": "oskarvanderwal"}}, "additions": 166, "deletions": 67}}, {"node": {"oid": "2d861a291b82729e1aa927935e464cdba761b513", "messageHeadline": "Added CrowsPairs for English and French", "author": {"name": "Oskar", "email": "o.d.vanderwal@gmail.com", "user": {"login": "oskarvanderwal"}}, "additions": 70, "deletions": 0}}, {"node": {"oid": "29bff88dbd067e8ac30718fc0e3d33b7665f9c82", "messageHeadline": "Updated T0 and GPT-J", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 51, "deletions": 18}}, {"node": {"oid": "b62d1becafc2ff9c907e6448d973195215cfc3f6", "messageHeadline": "Update stopping criteria for few-shot", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 17, "deletions": 19}}, {"node": {"oid": "50d029612b0001d534889fbb11cabd0f083c4171", "messageHeadline": "Merge branch 'master' into master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 591, "deletions": 53}}, {"node": {"oid": "1e44199c0f0c82c957476facae02fa619d7582a7", "messageHeadline": "Merge branch 'master' into add_lama", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 909, "deletions": 111}}, {"node": {"oid": "5e59320b709fd08ee24fd3fd098ea17cb5a9ad8a", "messageHeadline": "Modified stopping criteria for T5 and GPT-2", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 47, "deletions": 19}}, {"node": {"oid": "235494f4815f61c80086ca213c723ae12c7939f6", "messageHeadline": "comment unimplemented task", "author": {"name": "KhalidAlt", "email": "ekh@Protonmail.com", "user": {"login": "KhalidAlt"}}, "additions": 2, "deletions": 3}}, {"node": {"oid": "58abbac2f9919a60561a96755237c268ced520ef", "messageHeadline": "add dataset description", "author": {"name": "KhalidAlt", "email": "ekh@Protonmail.com", "user": {"login": "KhalidAlt"}}, "additions": 7, "deletions": 118}}, {"node": {"oid": "c46ff9e4a6e9bad5a433f3797c7da3ee65780061", "messageHeadline": "Merge branch 'master' of https://github.com/bigscience-workshop/lm-ev\u2026", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 831, "deletions": 30}}, {"node": {"oid": "6c86eb479f27b51191aa2d3830a48525b3a96505", "messageHeadline": "Fixed generation trunction for GPT-2 and T5", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 17, "deletions": 13}}, {"node": {"oid": "383318fe0b096767299ea65a7ba6bead7e5a1345", "messageHeadline": "add lama task", "author": {"name": "KhalidAlt", "email": "ekh@Protonmail.com", "user": {"login": "KhalidAlt"}}, "additions": 402, "deletions": 1}}, {"node": {"oid": "a963ab81fb8b07fdc37592655ba7f6ef13bac730", "messageHeadline": "Merge pull request #21 from tomlimi/tomlimi/fix_fs_write_out", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 7, "deletions": 7}}, {"node": {"oid": "256de63bfd6d091993d7990d3e9eaabf6f78ef4c", "messageHeadline": "Merge pull request #22 from bigscience-workshop/add-axg", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 36, "deletions": 0}}, {"node": {"oid": "10492f4b97975c70a3c6e89f04f87aebf5fa9121", "messageHeadline": "Add `axg` and `axb` to `super_glue`", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 36, "deletions": 0}}, {"node": {"oid": "449905de579c1b696d0b3b5cf2a8c292b7dab5df", "messageHeadline": "Merge pull request #17 from tomlimi/tomlimi/gender_bias", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 115, "deletions": 0}}, {"node": {"oid": "44e1ddb22a14a003109e217d41719eb3700c313b", "messageHeadline": "Merge branch 'master' into tomlimi/gender_bias", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 379, "deletions": 22}}, {"node": {"oid": "9d60dd763b1990aab72bcc90df2a8838807665bf", "messageHeadline": "Merge pull request #19 from kasnerz/kasnerz/generation_tasks", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 103, "deletions": 3}}, {"node": {"oid": "577376910a1e076a50045fd681122011c2397014", "messageHeadline": "Merge branch 'master' into kasnerz/generation_tasks", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 276, "deletions": 19}}, {"node": {"oid": "d47f4534c3562c87eeadfe8ab9f1f808d521cc4d", "messageHeadline": "Merge pull request #16 from jordiclive/webnlg_challenge", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 125, "deletions": 19}}, {"node": {"oid": "c166e2696a2c070538250d690f54a42e72e62627", "messageHeadline": "Merge pull request #13 from Shashi456/mlsum", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "966b70eafc667412d43caf9215447d4e7d89a3cc", "messageHeadline": "Merge pull request #18 from jordiclive/simplification", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 151, "deletions": 0}}, {"node": {"oid": "4be489119248316b34df81bc0eda483f6f5a96a4", "messageHeadline": "e2e_nlg_cleaned: Disable stopping criteria", "author": {"name": "Zdenek Kasner", "email": "kasnezde@fel.cvut.cz", "user": {"login": "kasnerz"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "49f0699cd1ee28d1d10530491a9acce207bea233", "messageHeadline": "Fixed few-shot context generation or datasets without a training split", "author": {"name": "tomlimi", "email": "t.limisiewicz@gmail.com", "user": {"login": "tomlimi"}}, "additions": 7, "deletions": 7}}, {"node": {"oid": "e04aab6d09f52ef935c2087822e8e2b3d84fdea7", "messageHeadline": "added task to init", "author": {"name": "Jan-Christoph Kalo", "email": "kalo@is68.ifis.cs.tu-bs.de", "user": null}, "additions": 8, "deletions": 0}}, {"node": {"oid": "27a8f231f5934556473b8ca62d4b18fe470a59d0", "messageHeadline": "added bigscience-lama evaluation script.", "author": {"name": "Jan-Christoph Kalo", "email": "kalo@is68.ifis.cs.tu-bs.de", "user": null}, "additions": 44, "deletions": 0}}, {"node": {"oid": "6588bc746d62c1097b43c670a848178276e5284d", "messageHeadline": "e2e_nlg_cleaned: Update stopping criteria to \"\\n\\n\", exclude an accur\u2026", "author": {"name": "Zdenek Kasner", "email": "kasnezde@fel.cvut.cz", "user": {"login": "kasnerz"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "b7f070d8a2c50199b95ba3e3f9b94ee51fb2b2b8", "messageHeadline": "Merge remote-tracking branch 'origin/master' into kasnerz/generation_\u2026", "author": {"name": "Zdenek Kasner", "email": "kasnezde@fel.cvut.cz", "user": {"login": "kasnerz"}}, "additions": 756, "deletions": 241}}, {"node": {"oid": "ec80b255d35c15facd81d0a2059fc510da220a09", "messageHeadline": "add docstring", "author": {"name": "jordiclive", "email": "jordan.clive19@imperial.ac.uk", "user": {"login": "jordiclive"}}, "additions": 49, "deletions": 0}}, {"node": {"oid": "8cd9162496866dd476e7d10672c0991bf7666e10", "messageHeadline": "GEM Simplification", "author": {"name": "jordiclive", "email": "jordan.clive19@imperial.ac.uk", "user": {"login": "jordiclive"}}, "additions": 102, "deletions": 0}}, {"node": {"oid": "d42d54233647f867c0880cfc73d481e1fcfb69b1", "messageHeadline": "adding challenge sets", "author": {"name": "jordiclive", "email": "jordan.clive19@imperial.ac.uk", "user": {"login": "jordiclive"}}, "additions": 125, "deletions": 19}}, {"node": {"oid": "9aba053ab0b4a2e022accf86c290882ed79e0679", "messageHeadline": "Remove custom max_length method", "author": {"name": "Zdenek Kasner", "email": "kasnezde@fel.cvut.cz", "user": {"login": "kasnerz"}}, "additions": 0, "deletions": 4}}, {"node": {"oid": "5a62765b89aa3dacff40a0e6e42207bba36c4e18", "messageHeadline": "Revert changes in base.py", "author": {"name": "Zdenek Kasner", "email": "kasnezde@fel.cvut.cz", "user": {"login": "kasnerz"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "7010e452e85c68610b7a542d8018a43e3de4c5f6", "messageHeadline": "WinoBias support added", "author": {"name": "tomlimi", "email": "t.limisiewicz@gmail.com", "user": {"login": "tomlimi"}}, "additions": 115, "deletions": 0}}, {"node": {"oid": "9cd70235f803a5c768f62e7518eda222b24546d6", "messageHeadline": "Merge pull request #15 from bigscience-workshop/update-mlsum-task-name", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "22ea032f2e8c22b83a8f67008d05ef86c3251206", "messageHeadline": "Fix `mlsum` task names after split update", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 2, "deletions": 2}}, {"node": {"oid": "efd319b35c317edcb57f33f4e5adbbcc89591366", "messageHeadline": "Merge pull request #14 from bigscience-workshop/fix-gpt-j", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "32d0658c271c09f42c8b2f36bfe4a698b4c4532a", "messageHeadline": "Add `eot_token` property to gpt-j", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 4, "deletions": 0}}, {"node": {"oid": "0a720fc5a0d39661983d7181568cab5920bfd6f9", "messageHeadline": "Merge branch 'bigscience-workshop:master' into mlsum", "author": {"name": "Pawan Sasanka Ammanamanchi", "email": "Shashi456@users.noreply.github.com", "user": {"login": "Shashi456"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "a80d78db9749d6795f366ce5d988a5247924371c", "messageHeadline": "Fix functions", "author": {"name": "Shashi456", "email": "pass2pawan@gmail.com", "user": {"login": "Shashi456"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "f65e196ecdfb018b05d9ceacaf387bc18d9cce03", "messageHeadline": "Merge pull request #5 from Shashi456/mlsum", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 127, "deletions": 1}}, {"node": {"oid": "6d5031acdaf700620bbb76aa857f200f781dbe53", "messageHeadline": "Prettify task __init__", "author": {"name": "Shashi456", "email": "pass2pawan@gmail.com", "user": {"login": "Shashi456"}}, "additions": 0, "deletions": 5}}, {"node": {"oid": "666bab5f4d164e7181430de708610823f6adc9e5", "messageHeadline": "Merge branch 'master' of https://github.com/Shashi456/lm-evaluation-h\u2026", "author": {"name": "Shashi456", "email": "pass2pawan@gmail.com", "user": {"login": "Shashi456"}}, "additions": 163, "deletions": 0}}, {"node": {"oid": "9abb1814ba6dcf90b18563979d4da65da0184977", "messageHeadline": "Update gem_mlsum", "author": {"name": "Shashi456", "email": "pass2pawan@gmail.com", "user": {"login": "Shashi456"}}, "additions": 44, "deletions": 9}}, {"node": {"oid": "6e56cd0d0fc7d89b974e9dee7489584e425790e2", "messageHeadline": "Merge pull request #7 from bigscience-workshop/kkawamu1/gem_xsum", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 163, "deletions": 0}}, {"node": {"oid": "7e25c240dd6ccccc45821daa33f13a11fd734af8", "messageHeadline": "Merge branch 'master' into mlsum", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 461, "deletions": 240}}, {"node": {"oid": "fce17ee1d4fc9634b5a64d9dfc7770d4d93c6a19", "messageHeadline": "Merge pull request #12 from tttyuntian/master", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 11, "deletions": 55}}, {"node": {"oid": "3ee4da8ef34d6e3e8b4614c73cb0cf81f183492a", "messageHeadline": "Added blimp task template", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 2, "deletions": 54}}, {"node": {"oid": "7da753f2ec5f004f54084d347400fef0a92e2605", "messageHeadline": "Adding null prompt support in T5", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 9, "deletions": 1}}, {"node": {"oid": "20820c3ce32528f477408cb2ff4826fbe444f8a1", "messageHeadline": "Merge pull request #11 from bigscience-workshop/update-context-separator", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 12, "deletions": 5}}, {"node": {"oid": "597bf7478db239a5794e8d43cd02820502c53d28", "messageHeadline": "Add `\"\\n###\\n\"` example separator", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 12, "deletions": 5}}, {"node": {"oid": "567e24c92306b351af3705ab05b2f64e5450d03f", "messageHeadline": "Merge pull request #10 from tttyuntian/master", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 165, "deletions": 101}}, {"node": {"oid": "3c390461fee944a913b17f5cfa47bb9a57b6c666", "messageHeadline": "Modified T0, T5 to BaseLM", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 148, "deletions": 100}}, {"node": {"oid": "ecde9d2e7cd1d5caff00cfbf178e291d899b0b22", "messageHeadline": "Fixing stopping criteria", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 17, "deletions": 1}}, {"node": {"oid": "e51880d8f0673964ffe581d0c20c06f322bd3c54", "messageHeadline": "Merge pull request #9 from jordiclive/webnlg", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 40, "deletions": 1}}, {"node": {"oid": "afcea2670c9a90640d383bd7667290acd92bb57b", "messageHeadline": "webnlg", "author": {"name": "jordiclive", "email": "jordan.clive19@imperial.ac.uk", "user": {"login": "jordiclive"}}, "additions": 40, "deletions": 1}}, {"node": {"oid": "1cd4ec01fa6d6b3ffeae5d3054f74c14435ff0d5", "messageHeadline": "Merge pull request #8 from dirkgr/PytestUpdate", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 16, "deletions": 18}}, {"node": {"oid": "38a240ce23e40ddc9111481ee8c9cb12db8ac471", "messageHeadline": "Merge branch 'master' into PytestUpdate", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1848, "deletions": 1042}}, {"node": {"oid": "33f5572a218c8f751de7e1c4915db8a199bb6b55", "messageHeadline": "Delete main.yml", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 48}}, {"node": {"oid": "e9e5305d91819c27bce7a99f82550bcb69ae3221", "messageHeadline": "Turning on data flags", "author": {"name": "Shashi456", "email": "pass2pawan@gmail.com", "user": {"login": "Shashi456"}}, "additions": 7, "deletions": 7}}, {"node": {"oid": "fc781c2fb1475be8e8e0de1fb31716799fa7a209", "messageHeadline": "Create main.yml", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 48, "deletions": 0}}, {"node": {"oid": "025547c9e219d4fd3a6381d3da754c46130a2865", "messageHeadline": "Merge pull request #6 from cjlovering/master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 219, "deletions": 62}}, {"node": {"oid": "e5bc43544b6b1da81e88a9c9082d6831740dbc8a", "messageHeadline": "Merge pull request #4 from cjlovering/cjlovering/output_examples", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 219, "deletions": 62}}, {"node": {"oid": "408e780207782b54ca31d8c069410ae9e6f05982", "messageHeadline": "Save logging information even if the saving example is false", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 10, "deletions": 12}}, {"node": {"oid": "f42f168cfe1675afed378a103ed17f4a14423142", "messageHeadline": "Add MLsum Tasks", "author": {"name": "Shashi456", "email": "pass2pawan@gmail.com", "user": {"login": "Shashi456"}}, "additions": 96, "deletions": 0}}, {"node": {"oid": "960f378050491d84f86fce756939e7da7891b22f", "messageHeadline": "Added field for comment; serves as an empty placeholder", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "60dc4ad2c0cdf35800648acb9b0eccfa8a857a1b", "messageHeadline": "Add field to track if it was the original task", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "549991995972f786b2ddcbf11be030c8196970d2", "messageHeadline": "Merge pull request #2 from cjlovering/master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1667, "deletions": 1019}}, {"node": {"oid": "18af502b3e794c9801bd9f61a3dc46eb86315968", "messageHeadline": "Merge pull request #5 from Shashi456/update-promptsource-version", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "56b2aa94dc13071ffcd45763859031e33395970e", "messageHeadline": "e2e_nlg_cleaned: Remove custom aggregation() and higher_is_better() m\u2026", "author": {"name": "Zdenek Kasner", "email": "kasnezde@fel.cvut.cz", "user": {"login": "kasnerz"}}, "additions": 0, "deletions": 22}}, {"node": {"oid": "0033f1b60f11e6a7010644cee4a7a1ee88d15b73", "messageHeadline": "Align BLEU computation with ROUGE", "author": {"name": "Zdenek Kasner", "email": "kasnezde@fel.cvut.cz", "user": {"login": "kasnerz"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "795b7912b7e8498d16650b2259c30b760dfa967a", "messageHeadline": "e2e_nlg_cleaned: Add custom code for filtering out invalid examples", "author": {"name": "Zdenek Kasner", "email": "kasnezde@fel.cvut.cz", "user": {"login": "kasnerz"}}, "additions": 28, "deletions": 2}}, {"node": {"oid": "3eb3d8b7c395da8393ccc3fd8542b0aa9fae249c", "messageHeadline": "e2e_nlg_cleaned: Exclude QA prompts for now", "author": {"name": "Zdenek Kasner", "email": "kasnezde@fel.cvut.cz", "user": {"login": "kasnerz"}}, "additions": 10, "deletions": 1}}, {"node": {"oid": "256a800ab041914e72e6dd34c1fbea870d39d546", "messageHeadline": "Update to the version of transformers which supports multiple stoppin\u2026", "author": {"name": "Zdenek Kasner", "email": "kasnezde@fel.cvut.cz", "user": {"login": "kasnerz"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f777d8401bfc06eada1e0d6686f743612139e069", "messageHeadline": "Add a main part of e2e_nlg_cleaned", "author": {"name": "Zdenek Kasner", "email": "kasnezde@fel.cvut.cz", "user": {"login": "kasnerz"}}, "additions": 90, "deletions": 3}}, {"node": {"oid": "60a07f106bcfb8c10976661ad12c715ee59f4511", "messageHeadline": "Use eval-hackathon branch for installing prompt-source", "author": {"name": "Pawan Sasanka Ammanamanchi", "email": "Shashi456@users.noreply.github.com", "user": {"login": "Shashi456"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "13d5304817ed102dc6b13d2b3cc967056888196c", "messageHeadline": "Enable saving all the examples.", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 217, "deletions": 61}}, {"node": {"oid": "448bcbfbf4aada7974da85c4b23e9fd449768524", "messageHeadline": "Merge pull request #3 from cjlovering/add-rouge", "author": {"name": "Charles Lovering", "email": "charles_lovering@brown.edu", "user": {"login": "cjlovering"}}, "additions": 63, "deletions": 7}}, {"node": {"oid": "d40a7ce007ec886c184dcfdaaa37bd4662c6ffaa", "messageHeadline": "Merge branch 'master' of https://github.com/cjlovering/ps-eh", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 477, "deletions": 32}}, {"node": {"oid": "62a706fcf0bfb937390355b342482558f9a01291", "messageHeadline": "Add doc getter methods to template", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 35, "deletions": 0}}, {"node": {"oid": "ff89667f61aee65eec298b73ca7d7253476fb9b4", "messageHeadline": "Add ROUGE metric to `PromptSourceTask`", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 63, "deletions": 7}}, {"node": {"oid": "5b0d95a087736650b1a6620c93d0df0b755a626d", "messageHeadline": "Merge pull request #2 from cjlovering/cjlovering/request_args", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 19, "deletions": 15}}, {"node": {"oid": "a3a9a7c2d162e7ec59fa1c6bfc435e6214e5f797", "messageHeadline": "Added GPT-J", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 121, "deletions": 0}}, {"node": {"oid": "21d897db56e8bf72bb0a2bdb3ace62cc7cba0bee", "messageHeadline": "Updated the requests so that its easier to understand.", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 19, "deletions": 15}}, {"node": {"oid": "4f85bcf98d173af452791c128a7b50cfd5718b85", "messageHeadline": "Updated doc: If the answer choices is empty, then it is generation; e\u2026", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 10, "deletions": 17}}, {"node": {"oid": "af3cccc837badf0bba60e7d194069a6a4924902e", "messageHeadline": "Merge branch 'master' of https://github.com/cjlovering/ps-eh", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 81, "deletions": 6}}, {"node": {"oid": "716c87d659ca08d2c52bd5b0a503a7ad8623321f", "messageHeadline": "Added t0, t5, mt5", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 327, "deletions": 0}}, {"node": {"oid": "94218002b139a7e8a4847cd52d95f86ccc92c4eb", "messageHeadline": "Add multi-reference ROUGE support", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 61, "deletions": 0}}, {"node": {"oid": "4941a8bb30af33bd5238505913213374eb2613d5", "messageHeadline": "Merge pull request #1 from cjlovering/cjlovering/gen_max_len", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 20, "deletions": 6}}, {"node": {"oid": "9384ec917dd1e33ea2e406a682fcd411a3eab01b", "messageHeadline": "A dependency required this but it was not installed by default", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 20, "deletions": 6}}, {"node": {"oid": "02ec78893e0e6a2977c90c7b375dcce5070f1e42", "messageHeadline": "Merge branch 'master' of github.com:cjlovering/ps-eh", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 93, "deletions": 0}}, {"node": {"oid": "b8c203cd052a3808ed469fba924bb88264c76eb6", "messageHeadline": "A dependency required this but it was not installed by default", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "ca414b0eadfcf707120413a20f8b95dc5ade3115", "messageHeadline": "Merge branch 'master' of https://github.com/cjlovering/ps-eh", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 1, "deletions": 1}}, {"node": {"oid": "941fe268bf7f0a9252b460a6655a8e478c01c3b7", "messageHeadline": "Add `PromptSourceTask` template", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 93, "deletions": 0}}, {"node": {"oid": "08358031894c2f3fe86a93e1ef546fe62a9156cd", "messageHeadline": "Remove stopping_criteria and set max_generation_length to 64", "author": {"name": "ken", "email": "ken_kawamura@brown.edu", "user": {"login": "kkawamu1"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "842139aecda71d08902b7c240b976615d8b2ccc4", "messageHeadline": "Merge branch 'master' of github.com:cjlovering/ps-eh", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "23a6ee7f174494ac2cab337bc9180547c126d6bf", "messageHeadline": "Changing stopping criteria for coqa", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "61b5729293061d4d840b585a62b420e8b08790af", "messageHeadline": "Modified stopping criteria for gpt2", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0b8cb8b92a10ffb5309b2205245887cd4a34f883", "messageHeadline": "Merge with master branch", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 88, "deletions": 22}}, {"node": {"oid": "96ea7ddce14daaabe3da8ec3a9ce520e72c9c535", "messageHeadline": "Added stoppping criteria for generation", "author": {"name": "Tian Yun", "email": "tttyuntian@hotmail.com", "user": {"login": "tttyuntian"}}, "additions": 66, "deletions": 2}}, {"node": {"oid": "b2838b8d119cec0765b233ba619168556d8ddc10", "messageHeadline": "Rename task specific  to", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 8, "deletions": 6}}, {"node": {"oid": "1dcca55ca4022b045bfdfec6624e99b5c2ed5dd2", "messageHeadline": "Minor updates to documentation.", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 8, "deletions": 20}}, {"node": {"oid": "d4c0009315197a9ba91adb3b4989f4c1c2325905", "messageHeadline": "Added default behavior for bleu to the promtsourcetask class", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 51, "deletions": 9}}, {"node": {"oid": "f39c27c266fd366031ce392f0565a1e679cfd3a9", "messageHeadline": "Rename task specific  to", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 43, "deletions": 6}}, {"node": {"oid": "c27e29e156a0beed0b1cac742b2a8fa6f9ad9fc9", "messageHeadline": "Force no caching while testing", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 2, "deletions": 0}}, {"node": {"oid": "f33717c24b23f0a1ac6a2d20e59a6bc7303332db", "messageHeadline": "Merge branch 'master' of github.com:cjlovering/ps-eh", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 3, "deletions": 12}}, {"node": {"oid": "c26f1d4c9b0399f3d395038427a720792bcca065", "messageHeadline": "Update the token encode", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 53, "deletions": 31}}, {"node": {"oid": "b988137d6a4e29ca28d2146192a87db6fc3b63b4", "messageHeadline": "Fix `wsc` subset name", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 3, "deletions": 12}}, {"node": {"oid": "422380cc12e34284d50a78742ec878ea7d277ff9", "messageHeadline": "Print table across task and prompt", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 32, "deletions": 4}}, {"node": {"oid": "053afe0540bbdb099b7184c5aee32a5695915fa5", "messageHeadline": "Merge branch 'master' of github.com:cjlovering/ps-eh", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "0cdcc9891f5fbd182d2f30ce2f0c17bb0afc7330", "messageHeadline": "Turned off generation tasks for now. Changed process to look at the m\u2026", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 118, "deletions": 27}}, {"node": {"oid": "0e578306dc8bd15e24a17aa848560d211de49b94", "messageHeadline": "Add `HANS`", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 63, "deletions": 0}}, {"node": {"oid": "b1a3c6e3a53d5dc6be23334aa77ba0e10f4ab2a0", "messageHeadline": "QQP with PS integration.", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 1, "deletions": 30}}, {"node": {"oid": "26e94211bc40c6b8d2a355bc381b72234a8eac25", "messageHeadline": "MRPC with PS integration.", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 1, "deletions": 16}}, {"node": {"oid": "b44aa554d7084de825c8c3660337881526c2f028", "messageHeadline": "QNLI with PS integration.", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 1, "deletions": 22}}, {"node": {"oid": "4c201b97dd4cf1dbfdde316afddbce7caea2bbe9", "messageHeadline": "SST with PS integration. (It was already done.)", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 0, "deletions": 19}}, {"node": {"oid": "e49cf8da0912debaa0e5759e3434466d39598fd7", "messageHeadline": "SST with PS integration. (It was already done.)", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 30, "deletions": 78}}, {"node": {"oid": "31a019c2d60750f8c149ea1e9dcf357635fec7d0", "messageHeadline": "Temp. commenting our e2e", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 29, "deletions": 38}}, {"node": {"oid": "ec2eb11736a364bf395d112241b0368b98091e2f", "messageHeadline": "Add promptsource to setup.py", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 4, "deletions": 3}}, {"node": {"oid": "c93093b63430ad9b75391504a68118d0c90dd437", "messageHeadline": "Removed the default option for an acc task", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 27, "deletions": 61}}, {"node": {"oid": "4ae2ab3747d2037d5ea87b9e180a2a297aa0cd7c", "messageHeadline": "Add `higher_is_better` & `aggregation` defaults to `PromptSourceTask`", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 10, "deletions": 0}}, {"node": {"oid": "6ec93da2497bda2402b35960b977f89943d1cb8f", "messageHeadline": "Add `eos_token` property", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 8, "deletions": 5}}, {"node": {"oid": "34f591afde1e30632e38604887ce9df89b149c92", "messageHeadline": "Add multiple tasks", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 81, "deletions": 387}}, {"node": {"oid": "2bfa451846844599af80b60215bfbe580650fb5d", "messageHeadline": "Fix prompt source rank choice accuracy", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 69, "deletions": 110}}, {"node": {"oid": "9f38846153ddade1844ca5b6bbaf3a43268fbd85", "messageHeadline": "Fix task name to template creation", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 42, "deletions": 24}}, {"node": {"oid": "9484eecc419af07c23359825eb6ffef5b7ba70b4", "messageHeadline": "Fix coqa", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 30, "deletions": 24}}, {"node": {"oid": "7d282b5f11e8424842fce36a1ebb51f76006f7ae", "messageHeadline": "Add PromptSourceTask to the updated tasks", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "88745155ae4759e4d83684306218ab5c8acd1be2", "messageHeadline": "Initial integration", "author": {"name": "cjlovering", "email": "cjlovering@wpi.edu", "user": {"login": "cjlovering"}}, "additions": 480, "deletions": 318}}, {"node": {"oid": "a1b271dd33907cf1ca2f761a2560edfdd8695ab0", "messageHeadline": "Fix typo", "author": {"name": "ken", "email": "ken_kawamura@brown.edu", "user": {"login": "kkawamu1"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "b4ad893c08d8f1acea4ec4d618df2fe7eb228473", "messageHeadline": "Merge master", "author": {"name": "ken", "email": "ken_kawamura@brown.edu", "user": {"login": "kkawamu1"}}, "additions": 450, "deletions": 185}}, {"node": {"oid": "8c83a8219d5faf7cbac5ff34a8ecbbc2eace21eb", "messageHeadline": "Add SPLIT", "author": {"name": "ken", "email": "ken_kawamura@brown.edu", "user": {"login": "kkawamu1"}}, "additions": 13, "deletions": 5}}, {"node": {"oid": "6a2b94b20ad36891bc832cafb6191a69dc21e2c2", "messageHeadline": "Add GEM/xsum", "author": {"name": "ken", "email": "ken_kawamura@brown.edu", "user": {"login": "kkawamu1"}}, "additions": 155, "deletions": 0}}, {"node": {"oid": "c901185943eccd94b6dbc242b58fbadb58ef916c", "messageHeadline": "Update `task_guide.md` instructions for dev environ setup", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c0e1d8cd29b89358d8ca64cd9b149e99d621a6f7", "messageHeadline": "Merge remote-tracking branch 'dirkgr/PytestUpdate' into PytestUpdate", "author": {"name": "Dirk Groeneveld", "email": "dirkg@allenai.org", "user": {"login": "dirkgr"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "e9acf86d2f1ee9f708726b4fde3a158c37ad950e", "messageHeadline": "Move black and pytest to be dev dependencies", "author": {"name": "Dirk Groeneveld", "email": "dirkg@allenai.org", "user": {"login": "dirkgr"}}, "additions": 3, "deletions": 4}}, {"node": {"oid": "f20ba95b5be3e7652f8348285c9a10db1a3dfc35", "messageHeadline": "Rename all `_load_doc`s to `_process_doc`", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 5, "deletions": 5}}, {"node": {"oid": "aeb7e846628da1e2dbecb0340edeaef433b487d4", "messageHeadline": "There is no point in passing in a RNG if you're re-seeding it every t\u2026", "author": {"name": "Dirk Groeneveld", "email": "dirkg@allenai.org", "user": {"login": "dirkgr"}}, "additions": 4, "deletions": 6}}, {"node": {"oid": "7ab6470159a69404c80d31a2a824b0fee46009f8", "messageHeadline": "Use `_process_doc` like the other tasks do", "author": {"name": "Dirk Groeneveld", "email": "dirkg@allenai.org", "user": {"login": "dirkgr"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "a080b5ad7d5c3fd6a343657140fd37cac7f59587", "messageHeadline": "Adds missing __init__.py files", "author": {"name": "Dirk Groeneveld", "email": "dirkg@allenai.org", "user": {"login": "dirkgr"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "d2455d3d2bd443f5ae2a90a59a0c0fc8ae1140bf", "messageHeadline": "Make this importable", "author": {"name": "Dirk Groeneveld", "email": "dirkg@allenai.org", "user": {"login": "dirkgr"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "932730363aed12c9b124ab921af180714cb842f9", "messageHeadline": "Updates PyTest dependency", "author": {"name": "Dirk Groeneveld", "email": "dirkg@allenai.org", "user": {"login": "dirkgr"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a0fa0454b5e44b3fc91f74215989e1f005ea3676", "messageHeadline": "Add templates and update docs", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 274, "deletions": 63}}, {"node": {"oid": "9b6c45a845568542a6e5d778c38ddbaa1023083c", "messageHeadline": "Add testdata", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 2, "deletions": 0}}, {"node": {"oid": "80f3e064b3fe029fe54f9b6d77ce8be566d66f3b", "messageHeadline": "Add `SWAG`", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 61, "deletions": 0}}, {"node": {"oid": "ab6883b101ba68c9987190ae7299578d13d7fe50", "messageHeadline": "Fix decontamination queries", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 2, "deletions": 2}}, {"node": {"oid": "70772cb26ff1986b9cf6157d67f36083798c4f8b", "messageHeadline": "Add dataset features to `TriviaQA`", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 34, "deletions": 9}}, {"node": {"oid": "9b4bfb6a76f2f8f757581e4d93a8dbeeedf41bd5", "messageHeadline": "Merge changes", "author": {"name": "jon-tow", "email": "jon-tow@Jons-MacBook-Pro.local", "user": null}, "additions": 13, "deletions": 15}}, {"node": {"oid": "e0cfeb907a829698fd1226c65a2f49d8e6477885", "messageHeadline": "Merge branch 'master' into researcher2", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 4671, "deletions": 1266}}, {"node": {"oid": "0a6a9b7ee13805cfff1a7a47c97668d6e337eb73", "messageHeadline": "Delete extractive spans comment", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6caa0afd96a7a7efb2ec4c1f24ad1756e48f3aa7", "messageHeadline": "Merge pull request #300 from jon-tow/hf-dataset-refactor", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3557, "deletions": 1145}}, {"node": {"oid": "9434722c87df4c5054603a9a555cc1d01841ccdb", "messageHeadline": "Fix docstring", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "39c8c87f94e5c7fe351353446986dfd339e3bc6d", "messageHeadline": "Adopt `_process_doc` method", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 86, "deletions": 56}}, {"node": {"oid": "d79a4389e4e1eeb83bc9430514b3cc0f77477fd0", "messageHeadline": "Add specific `load_dataset` args", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 33, "deletions": 8}}, {"node": {"oid": "0a67b8542578f8c82f7af029e2455093c62819f8", "messageHeadline": "Suggested updates for the task guide", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 12, "deletions": 3}}, {"node": {"oid": "4f8b99f68cdc8ee368f9de5f236b2ad33b70ca02", "messageHeadline": "Whitespace formatting", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 8, "deletions": 13}}, {"node": {"oid": "ad164f36cb8983de5c33a51acaf5f41dba5acfc5", "messageHeadline": "Revert `pile` coverage omit", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "daa1f4363c49cbc3c0a9e28e2af0f0d4cb5d2a3f", "messageHeadline": "Omit  task coverage", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "c2e3298767e54be8ef33c1870d6b56d2e000a8e6", "messageHeadline": "Fix  dataset value feature", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "897ae76d84f106e137880e9fe4823f02664689aa", "messageHeadline": "Add custom `image`less `HeadQA` dataset", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 153, "deletions": 1}}, {"node": {"oid": "13317f8cea9a220c073e78104b0062dd078d69af", "messageHeadline": "Fix training docs caching", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 20, "deletions": 16}}, {"node": {"oid": "7c9da71441aab0670663a18336488143b1ddf9b8", "messageHeadline": "Refactor `Task` download", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3334, "deletions": 1128}}, {"node": {"oid": "f9b8115158131e2485cf4d025faf111a602de0c3", "messageHeadline": "Update sort_13_gram_buckets.py", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "985f1f1ac3772d64f61e22dacffe18212d55df93", "messageHeadline": "Update decontamination.md", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "4411e788b4588d17c1098a227567d6ac0b6469a4", "messageHeadline": "More Changes For PR", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 164, "deletions": 13}}, {"node": {"oid": "7064d6b900760a189ff32b1751e8c1934884debf", "messageHeadline": "Bump version to 0.2.0", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "fcaea0e39ee96da06bdeed282a816f35ae850098", "messageHeadline": "Merge pull request #296 from jon-tow/gsm8k-fix", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "f91a70d64a1ad0cea9eb7c7ed903c268b7764447", "messageHeadline": "Add positional args to `gsm8k` `download_file`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "cec2fe23eaa4cda5521b518c3f3461480169e5e9", "messageHeadline": "Merge pull request #294 from jon-tow/asdiv-remove-fewshot-description", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 5}}, {"node": {"oid": "d14ab6d5d2a63594fe86d1b7742adc6c91f55f64", "messageHeadline": "Remove `Asdiv` `fewshot-description` method", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 5}}, {"node": {"oid": "3c0fa5a602ce60c17b7790fbc48539a1cdfcd7b7", "messageHeadline": "Merge branch 'researcher2' of https://github.com/EleutherAI/lm-evalua\u2026", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 715, "deletions": 272}}, {"node": {"oid": "f495bfb4f6d35620b8d200701fcac99926d0ef25", "messageHeadline": "Refactor for PR", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 407, "deletions": 505}}, {"node": {"oid": "263eb48d1082b4d429ce3e7d05912ff3fcbe3bed", "messageHeadline": "Merge pull request #292 from jon-tow/task-doc-update", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1126, "deletions": 130}}, {"node": {"oid": "b82e1a87257c0fe6089c5431acb2ee6acebec1d9", "messageHeadline": "Update `task_guide.md` for new citation structure", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 9}}, {"node": {"oid": "9e0b7102dc4324bb4896b04eb69a49c1606bdd58", "messageHeadline": "Correct misspellings", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "3f13d15f4586ec9d77a051cd93a11c76e3bc662b", "messageHeadline": "Make citations module-level constants", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 480, "deletions": 319}}, {"node": {"oid": "a1aceacd607fd0fb7fe8f871567270edeb0a6954", "messageHeadline": "Add citations and descriptions to all tasks", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 912, "deletions": 71}}, {"node": {"oid": "57569bbb74f90ee2019087da06b2d66898e1790d", "messageHeadline": "remove flags", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 17}}, {"node": {"oid": "bf19a7a19e3306d63b9393f81697bd06c9efef88", "messageHeadline": "Merge branch 'master' into researcher2", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 714, "deletions": 271}}, {"node": {"oid": "663b781bfab9168c2bbf5cf9b99e199f685c91db", "messageHeadline": "Merge pull request #291 from StephenHogg/check_integrity_docs", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 9, "deletions": 0}}, {"node": {"oid": "fff69d420fe38ed54996c0b65ecea71be437979b", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "db1d67f612bc6bddd013480d90ccd7cde53c73f7", "messageHeadline": "Add comment to README explaining check integrity flag", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 10, "deletions": 0}}, {"node": {"oid": "93cf95d062d36b717742cb3c1a3ca8e530d7cb58", "messageHeadline": "Merge pull request #290 from StephenHogg/check_integrity", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 44, "deletions": 3}}, {"node": {"oid": "40c1e05c102f929b4f4d4621ead0b61e660ada16", "messageHeadline": "add flag", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "606b98f71a0ad5e4ba5b3ee24406c31cd296310f", "messageHeadline": "Change function signature so package root is always found from utils.py", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 3, "deletions": 3}}, {"node": {"oid": "4d8dde31440cc6279b57c2806de1aa8903e5f178", "messageHeadline": "Add integrity check", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 41, "deletions": 2}}, {"node": {"oid": "0f283a9c1214f33b1f3172687e1ec480688874ef", "messageHeadline": "further interface changes", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 9, "deletions": 11}}, {"node": {"oid": "b246009951d13a07b8cd6a80a0951ea52b2e3f2b", "messageHeadline": "pull stuff into lm_eval.decontamination", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 438, "deletions": 151}}, {"node": {"oid": "3c37ea9caf8ef8982116c4a8abb2a4e4b92eb854", "messageHeadline": "Merge pull request #264 from StephenHogg/qasper", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 220, "deletions": 0}}, {"node": {"oid": "815f165c971c2df5691f02b78c783c29db17b971", "messageHeadline": "Don't create requests for TODO metrics", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 11, "deletions": 11}}, {"node": {"oid": "f7aaff08b2dbe3352439724111cfd0f4a43a50b9", "messageHeadline": "Mark unanswerable as TODO", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 5, "deletions": 6}}, {"node": {"oid": "8990517490150b0675024c0f095f0b0e61032762", "messageHeadline": "Merge pull request #289 from jon-tow/gsm8k-testdata", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "98d75af05352d8a9f1948197f1c9a3d12e2081ca", "messageHeadline": "Changes for PR", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 17, "deletions": 37}}, {"node": {"oid": "54065e6a60e3127b5b507e01ae1bfe48d8fefc92", "messageHeadline": "Add `gsm8k` test data", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "be55ea8ceec01b1e1981145c13ea7cac7d5a66c1", "messageHeadline": "Align casing between doc creation and evaluation", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 2, "deletions": 2}}, {"node": {"oid": "8f0e176541ec84b78c69fed3a41fa857afca11fd", "messageHeadline": "Remove extractive spans score; mark as TODO", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 3, "deletions": 15}}, {"node": {"oid": "cef6aa8d76e0ac91352ce88283db198ba45b3de7", "messageHeadline": "Merge pull request #260 from jon-tow/gsm8k", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 141, "deletions": 0}}, {"node": {"oid": "f6fb7f6a7e85be93d82c25c67666625e57ccc0bb", "messageHeadline": "gpt3: print exceptions", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "edc9449fdad55c029516333795ae1f889c9a21ba", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2cc968ba6a3555c24ce24f3092eceb77c3f3182c", "messageHeadline": "Merge pull request #288 from EleutherAI/leogao2-patch-1", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 263, "deletions": 251}}, {"node": {"oid": "71647775d77290821d6a4450b0c06159f52fe8ba", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 263, "deletions": 251}}, {"node": {"oid": "c5131e8a23d1ad88dd82c63e39a0e278f4d36da8", "messageHeadline": "Merge pull request #263 from jon-tow/hendrycks_math_checksum_update", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 22, "deletions": 8}}, {"node": {"oid": "c74ca57c51c5fb0889b955878f00fe6d60ba393c", "messageHeadline": "Merge pull request #286 from StephenHogg/255", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 7, "deletions": 7}}, {"node": {"oid": "d895bfe6ada71e699d185c5ceca5a75a4001b5c0", "messageHeadline": "Update dummy model classmethod args", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 7, "deletions": 7}}, {"node": {"oid": "af9766d1e67e79fdc3a7de444ca37c5dc3389692", "messageHeadline": "Remove incorrect comments", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 0, "deletions": 2}}, {"node": {"oid": "894c0697417539b51bfcccb34fc7f4f27b1d2e19", "messageHeadline": "Name changes as per leogao2", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 16, "deletions": 6}}, {"node": {"oid": "96f3e5b3699d25f5db43c8c4c4b792a560eea7b7", "messageHeadline": "Disable extractive spans", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 22, "deletions": 13}}, {"node": {"oid": "c6a356964a9ae3ba879322d810ca1ae13c522131", "messageHeadline": "Include extraction in process_results; fixes per test results", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 106, "deletions": 39}}, {"node": {"oid": "c2f1247460aff769ac96813f6934bc36479b5937", "messageHeadline": "exponentiate unanswerable score; rename dict key", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 4, "deletions": 3}}, {"node": {"oid": "7ed5e29ea79e3c0075740652dc936f3bd6fd5a48", "messageHeadline": "Add citation; update process_results and construct_requests", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 60, "deletions": 4}}, {"node": {"oid": "79492627fdc9ef31af054caa1aa02f7552f36258", "messageHeadline": "Add initial draft of QASPER; register with package; yet to complete p\u2026", "author": {"name": "Stephen Hogg", "email": "shogg@emvision.com.au", "user": null}, "additions": 92, "deletions": 0}}, {"node": {"oid": "e18ef6e162e5b7f5d9582d998fa9c980703a3540", "messageHeadline": "pin best-download to 0.0.9", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "7d37dd4aab83a6d63098d14e3ad0059c3736beeb", "messageHeadline": "Revert \"Update requirements.txt\"", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "dcef7c8f630244f79a687f361db073dc58bd75c9", "messageHeadline": "Update cbt.py", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "49dc29bdfc857f59dfc46b425e3533a814424d91", "messageHeadline": "Bump Hendrycks `MATH` because of updated dataset", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 22, "deletions": 8}}, {"node": {"oid": "05590e11522e78a19b9b1109b710ad79f8ae5de6", "messageHeadline": "Update requirements.txt", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "dae7b868611b66cf262b0fd8eb39c018a8de8203", "messageHeadline": "Added decontamination to remaining evals", "author": {"name": "Quentin Gregory Anthony", "email": "qanthony@login01.expanse.sdsc.edu", "user": null}, "additions": 223, "deletions": 1}}, {"node": {"oid": "c75b3413a7091ce2e48417b614816477c1362914", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-\u2026", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "341663a92f0c35171eb9cd464acf6ee3bdebfc55", "messageHeadline": "Small fixes", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 3, "deletions": 7}}, {"node": {"oid": "55e62507b9d260f9b7f963fc6ff0993f66ef140c", "messageHeadline": "Merge branch 'master' into researcher2", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 1972, "deletions": 1004}}, {"node": {"oid": "bb0eafbb0216480d6a8093d361c984950f9e6bb4", "messageHeadline": "Added more eval tasks and fixed directory error tasks", "author": {"name": "Quentin Anthony", "email": "qganthony@yahoo.com", "user": {"login": "Quentin-Anthony"}}, "additions": 38, "deletions": 0}}, {"node": {"oid": "26f0233fa4b6ca5b2d663a017dc4352ac528648a", "messageHeadline": "Merge pull request #259 from jon-tow/headqa-ux", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "9b472bc91fd94c4b6858ddece4eb66013585457d", "messageHeadline": "Create contamination.py", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 147, "deletions": 0}}, {"node": {"oid": "03c15e065e6d4c68e9ecbc9b78c45450bf8a9976", "messageHeadline": "Update `headqa` deprecation warning to display on init only", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "f04a91c1e604edb3d8242fd7ee7f106674fe52ee", "messageHeadline": "Add grade school math `GSM8K`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 141, "deletions": 0}}, {"node": {"oid": "1c52e917cf9196a72dee111ff3a107b115e9da96", "messageHeadline": "cb: add testdata", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "c3d941b98af79417dce99d71f14d1f0d337c78bb", "messageHeadline": "Merge pull request #254 from bigscience-workshop/thomas/fix_cb", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "4849702e1b457af5480e10569ee79d171b44fead", "messageHeadline": "Merge pull request #256 from jon-tow/update-asdiv", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 8, "deletions": 5}}, {"node": {"oid": "caff4f1099349f599a3bdeb7725bb2add06d741b", "messageHeadline": "Conform `asidv` to the new description api", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 8, "deletions": 5}}, {"node": {"oid": "be40299e8d0f398a0a0539ed9da19f91d223399d", "messageHeadline": "Merge pull request #252 from bigscience-workshop/thomas/fix_best_down\u2026", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b421f0529a4d2bf1ef8e68cd74671be44010c37a", "messageHeadline": "Bump version", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ea3fd79bd2309fec6e2fea74aaa072a967173dc3", "messageHeadline": "Fix CB", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "2d9fc25632fb0e48c8d6f7d183252800ce02afcd", "messageHeadline": "Missed asdiv", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ed6931e7e6c422bd17029c13a61cbbeef11e453c", "messageHeadline": "Merge pull request #250 from bigscience-workshop/thomas/fix_best_down\u2026", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 25, "deletions": 26}}, {"node": {"oid": "78824d7faee422c8ce55432612a20c71925d3481", "messageHeadline": "Merge branch 'master' into thomas/fix_best_download_version", "author": {"name": "Thomas Wang", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 535, "deletions": 363}}, {"node": {"oid": "cc2381213103eb8d980b000fecfd1706e4e17ba7", "messageHeadline": "Merge pull request #243 from bigscience-workshop/thomas/fix_multirc", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 10, "deletions": 6}}, {"node": {"oid": "170ae09602b724db69fbbc06f12abf7ce2a23165", "messageHeadline": "Merge pull request #226 from jon-tow/evaluator-description-option", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 317, "deletions": 340}}, {"node": {"oid": "02a4def21b4de5f29c6314fb73abe1810d194128", "messageHeadline": "Update blimp fewshot_context", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 11, "deletions": 2}}, {"node": {"oid": "c65412e530c1780ec7b2b970373e9e83f796ab20", "messageHeadline": "Actually it shouldn't be hard to fix it to be compatible with future \u2026", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 26, "deletions": 26}}, {"node": {"oid": "5792030c1011aaba8c6373f84e5c3744535724a4", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm-evaluation-harness \u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1010, "deletions": 208}}, {"node": {"oid": "b2f6bce86c62a40498d863d103bd3c25c6d7f672", "messageHeadline": "Fix fewshot_context method handling", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "76dc60938b43ec471149640e7ed954f3229fbc3e", "messageHeadline": "Best-download have backward compatibility issue", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c3bec45fa95a39271a0311768da8b2aa85e64e8e", "messageHeadline": "Merge branch 'evaluator-description-option' of https://github.com/jon\u2026", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "d2636b4e71d2eb23faf6af547c53ed139b79d376", "messageHeadline": "Enforce `rnd` args with assertions", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 12}}, {"node": {"oid": "d09561fe8cee57775b31bc1adfe6696c3c0f514e", "messageHeadline": "Update description_guide.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8728710c0846a744a99687d363f9cb8f0f375fd5", "messageHeadline": "Merge pull request #244 from rokosbasilisk/asdiv", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 120, "deletions": 0}}, {"node": {"oid": "70a9c476819405c7ebafa580e4a07ae590aa367c", "messageHeadline": "Merge pull request #242 from igor0/bits_per_byte", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 55, "deletions": 6}}, {"node": {"oid": "8dbd24f6603fa219991f4621545f0ca0edc30709", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm-evaluation-harness \u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 34, "deletions": 11}}, {"node": {"oid": "ff58b389cc41eb68647e645a149ddc77bc23ed41", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm-evaluation-harness \u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 836, "deletions": 203}}, {"node": {"oid": "a67c17e00996ab37a270657269adecf4310e9509", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm-evaluation-harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 824, "deletions": 193}}, {"node": {"oid": "9d87d47696f73c18597e12c74c16f001f0497ffe", "messageHeadline": "Delete test_cache.db", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "cb3babd3620cc597f3399c17622db7bddec99a39", "messageHeadline": "Improve pile test efficiency", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 4}}, {"node": {"oid": "4b3dee673a776de7be988b40a1a9c40b27e04e4d", "messageHeadline": "asdiv: space convention", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "33315a1f50b163856e3c83f1ea514654c7367deb", "messageHeadline": "pile/wikitext: add testdata", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 46, "deletions": 0}}, {"node": {"oid": "83e1a1166cb2df68accf353737e1d0450a719517", "messageHeadline": "removed strip_bracket function", "author": {"name": "rokosbasilisk", "email": "thecomplexitytheorist@gmail.com", "user": {"login": "xagi-dev"}}, "additions": 1, "deletions": 5}}, {"node": {"oid": "72d7cc0cc545b187b9cb243bc1fb73d350657298", "messageHeadline": "remove _strip_bracket function", "author": {"name": "rokosbasilisk", "email": "thecomplexitytheorist@gmail.com", "user": {"login": "xagi-dev"}}, "additions": 3, "deletions": 15}}, {"node": {"oid": "0fe2e80ad2d656747e8136abf51df6bfb5cf31be", "messageHeadline": "Merge pull request #245 from bigscience-workshop/thomas/improve_boolq", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "6653cc509e810752eeda2349c20c315a18c7fd5d", "messageHeadline": "Bump the version number for all tasks based on PerplexityTask", "author": {"name": "Igor Ostrovsky", "email": "igoros@gmail.com", "user": {"login": "igor0"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "04635731923251fd79500dc0e1e356f43e666df5", "messageHeadline": "remove unrequired files&add pin commit hash", "author": {"name": "rokosbasilisk", "email": "thecomplexitytheorist@gmail.com", "user": {"login": "xagi-dev"}}, "additions": 5, "deletions": 556}}, {"node": {"oid": "73d0ae5e448c36ff663fa82376c78f6b43a6cee8", "messageHeadline": "Add testdata for multirc v1", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "d7e3248990acc20e48c2facc34840384acfdc1e0", "messageHeadline": "Pretty sure questions need to be paragraph dependent also", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 5, "deletions": 4}}, {"node": {"oid": "0bde758966b7787d368412cc8ab02869d3463523", "messageHeadline": "Revert \"Add capital letters\"", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f42a8d6619a321ca270b1159702f2d9874cbb375", "messageHeadline": "Add testdata for boolq v1", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "5a53c3684523cd873bdef18cb8264fecd82a3f21", "messageHeadline": "pile: Switch download over to backup host temporarily", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "0d9d47da05170e3c9629b76360f614104f654056", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "97ca18e0854dbf01b68d0f77c249f5ebe64ec057", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "50ac7dff3dc9ecdd4de913c38cd1f6f2749f4cad", "messageHeadline": "Merge pull request #240 from bigscience-workshop/thomas/fix_head_qa", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 22, "deletions": 4}}, {"node": {"oid": "440216db7b1a3faebc0b1cb085e20192e13ce386", "messageHeadline": "Questions in BoolQ don't have interrogation punctuation at the end", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "bce9f289f8eee207fbc56288492f1db0807961eb", "messageHeadline": "remove apps", "author": {"name": "rokosbasilisk", "email": "thecomplexitytheorist@gmail.com", "user": {"login": "xagi-dev"}}, "additions": 0, "deletions": 130}}, {"node": {"oid": "b0a123102305a8b55ef3af24b535929a60642b52", "messageHeadline": "add asdiv task", "author": {"name": "rokosbasilisk", "email": "thecomplexitytheorist@gmail.com", "user": {"login": "xagi-dev"}}, "additions": 817, "deletions": 0}}, {"node": {"oid": "6597c3479de3b3e2a11ff37c4f8a4161f926bf4a", "messageHeadline": "Implement decontamination of evals against training set 13 grams.", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 209, "deletions": 24}}, {"node": {"oid": "23a420674b49476c5be66b0bf4e0d487ca38f1c7", "messageHeadline": "Add capital letters", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "df1fc6c355c771eb090efabb230bb267bbb1e885", "messageHeadline": "Fix multirc", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "383605122c428437dae452e543e5a73deadbe63f", "messageHeadline": "Fix bits_per_byte metric in PerplexityTask", "author": {"name": "Igor Ostrovsky", "email": "igoros@gmail.com", "user": {"login": "igor0"}}, "additions": 6, "deletions": 3}}, {"node": {"oid": "e34c6bd606ab68d3b73f19dc3732bc28465a6fa0", "messageHeadline": "Fix README", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "666b615f2f80e213354dc5dc78aa85e639d48ff9", "messageHeadline": "Fix README", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 146, "deletions": 146}}, {"node": {"oid": "377a1f459969b66a8eeeb755239e5067813c71fd", "messageHeadline": "Merge remote-tracking branch 'origin/master' into thomas/fix_head_qa", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 1357, "deletions": 502}}, {"node": {"oid": "0e232f7af6ff6a48bd78b3ff3752549e723056de", "messageHeadline": "Update new `task` arg and task dict getter", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 7, "deletions": 5}}, {"node": {"oid": "57d0718a718243d6ee64a995f3da8fc2146bbd29", "messageHeadline": "Remove all `provide_description` argument uses", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 5}}, {"node": {"oid": "a34bbe695d8da0eaf8e488b082efd8ad1ffe1745", "messageHeadline": "Remove more `provide_description` uses", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 40, "deletions": 9}}, {"node": {"oid": "d86aabc4aa78406ee22114752790287d4fa18fce", "messageHeadline": "more changes", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 7, "deletions": 5}}, {"node": {"oid": "7b2b2a23689cab7faf90fdf05105a235713d57ea", "messageHeadline": "Make simple_evaluate take LM and Task objects directly too", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 35, "deletions": 13}}, {"node": {"oid": "f16e8b5cb9734154b6f8efee0b7829f23e3f3227", "messageHeadline": "Merge pull request #237 from igor0/blimp", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 794, "deletions": 185}}, {"node": {"oid": "66fffcfc6dacd441f7a4fbc350054b085908eb79", "messageHeadline": "wnli: add testdata for v1", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "22c4124fcc91d9552d1b71d9ae9655da7ef8d390", "messageHeadline": "Add new testdata", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "cdab2c033c75f22c5a0dc2b1c5e004f8ae4413cc", "messageHeadline": "headqa: maintain backwards compatibility", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 8, "deletions": 1}}, {"node": {"oid": "e8e4873a27179082bbb4cd13a7414434725954d0", "messageHeadline": "Merge pull request #239 from bigscience-workshop/thomas/fix_wnli", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 7, "deletions": 8}}, {"node": {"oid": "a48e3cf2c9f865fea3b3bafa2b3c4ab21011c78c", "messageHeadline": "Bump version", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "5855f482c48f7f23efac13bcd38370e7ffc17470", "messageHeadline": "Allow users to specify en headqa or es", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 11, "deletions": 4}}, {"node": {"oid": "682e5c44c1036f00c2e283ef02bf21a23c0a021a", "messageHeadline": "FIX wnli", "author": {"name": "thomasw21", "email": "24695242+thomasw21@users.noreply.github.com", "user": {"login": "thomasw21"}}, "additions": 6, "deletions": 7}}, {"node": {"oid": "5f48dfc2ab16917ff5b18862c522bbf42a46f053", "messageHeadline": "Add BLiMP", "author": {"name": "Igor Ostrovsky", "email": "igoros@gmail.com", "user": {"login": "igor0"}}, "additions": 794, "deletions": 185}}, {"node": {"oid": "aea963a10c0d4baa43c6585ab11771e1ebce4415", "messageHeadline": "Format for consistency", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "8ebe36b25f81be9a9cd237156a2037bf750d6189", "messageHeadline": "Add positional arg deprecation decorator", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 85, "deletions": 9}}, {"node": {"oid": "d34ae3cfc9fba7f17da48c8a33ea87f2a9d27995", "messageHeadline": "Add `description_dict` to results config", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "70f92733af76ae1c8ad1524f4c1712f7ff9c05ea", "messageHeadline": "Fix doc reference", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "acf76b50b6f69917a09379415111c0af3f00c5b3", "messageHeadline": "Add `description_dict` docs and update `task-guide`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 50, "deletions": 37}}, {"node": {"oid": "1bc6cdb160efa0bbc3198a15d2cf428a06158aec", "messageHeadline": "Add type info to doc-string", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "744482bb183deab3abf5c2206b217422221aaee1", "messageHeadline": "Fix task example link", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e54380d882436c5136a2b474e341dfb26cd672a1", "messageHeadline": "Fix task example link", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "3f06b6032194e75ec3b6b9fdbd08597d37af6a55", "messageHeadline": "Remove unused import", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "09e1de936d7deb7938a313679b45057dfe31ef82", "messageHeadline": "Fix assertion error message", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "e3ddcfc277a449b4974159c4b275c0fd0cfc72c5", "messageHeadline": "Remove needless call", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 3}}, {"node": {"oid": "10dd7d38535a3695546fc3d762fc37596950ad93", "messageHeadline": "Make `evaluate` and `simple_evaluate` description args consistent", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 12, "deletions": 11}}, {"node": {"oid": "d1319950d7836102b9a2eaa359319143b2f1e00c", "messageHeadline": "Add newline to end of file", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "09cd76c76be8f22d1b80e656069028306ee73e1b", "messageHeadline": "Fix assertion error string", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "3fdff22f5986a895766bcb905493886b3156b781", "messageHeadline": "Remove `print` from test", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "d7a8ab24ab8778351ae8651093d5883ef72f1ec7", "messageHeadline": "Add basic `description_dict`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 47, "deletions": 41}}, {"node": {"oid": "ee53be219bf3f1ef485a725db6b7047b09d0eb63", "messageHeadline": "Add `provide_description` arg for backward compat", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 35, "deletions": 40}}, {"node": {"oid": "7a357971b553a871ad955190fc883ec76499cf9e", "messageHeadline": "Merge branch 'evaluator-description-option' of https://github.com/jon\u2026", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "1d04c42d2c9b92094c0fd6b7497120ce76984590", "messageHeadline": "Merge", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 25, "deletions": 13}}, {"node": {"oid": "ff314d62f38cd1699267df948b3e1f480412f92b", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-\u2026", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 607, "deletions": 441}}, {"node": {"oid": "df5d7cf024f201f3863c305a32f180ea68a599bd", "messageHeadline": "Merge pull request #229 from EleutherAI/lm_refactor", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 608, "deletions": 435}}, {"node": {"oid": "9590f3664e278f2e320e70c883c3c5cdcbca1380", "messageHeadline": "Bump version to 0.1.0", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c7ae6df1048532ff6cc13ce6c51c4bd57c1d2eea", "messageHeadline": "Update evaluator.py", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "564e06127ffdef1817a69cd7cdce816d452750ce", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-\u2026", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 97, "deletions": 24}}, {"node": {"oid": "a6bb79b4071d3bf77917b02af97c21965f6de8ff", "messageHeadline": "Update test_gpt3.py", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "01b98d2d639f742fdeea2392c6c306b13905cc96", "messageHeadline": "Update test_gpt3.py", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "7c011370f25f56eb14ed3f869f8dd7def594d1af", "messageHeadline": "Get rid of get_token_logprobs", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 21}}, {"node": {"oid": "ae63eeb7d11f3d83af4642383835774eda1b16c1", "messageHeadline": "Merge pull request #232 from zphang/lm_refactor_update1", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 35, "deletions": 4}}, {"node": {"oid": "67e2bf8b96c5f6a9e065a0da111235fb0561ae9b", "messageHeadline": "Merge pull request #234 from EleutherAI/race_fix", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "784a5c5459e05dcc0f0ba1296c0a366b1b01e941", "messageHeadline": "Pin datasets to 1.15.1", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "6b6e6d77f3ad0fdd5ea0082a293422a1fe3668f5", "messageHeadline": "Update req_type to request_type", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "3b386db4d3a433fbad32fc165eacd5cf4c1c9c02", "messageHeadline": "Make everything request_type", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 11, "deletions": 11}}, {"node": {"oid": "24fde4a65f0d4125d61a1dfaa49e7c00d10dedde", "messageHeadline": "Update base.py", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "13916e326472011fb5e2d6c2c8f9a538eff3b226", "messageHeadline": "Pin datasets to 1.16.0", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "37bfb4f55e982fa25b46204bec6b3e0ebfa8bfc8", "messageHeadline": "Merge pull request #228 from jon-tow/truthfulqa-ver-bump", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 86, "deletions": 16}}, {"node": {"oid": "376855d552f9fc5fb384feb0b4ac79b7e6bf9469", "messageHeadline": "Loosen tolerances that were intended for `float64` values", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "377a4fe2f3c3b972196103e977b268e0ddb92acc", "messageHeadline": "pytest.approx doesn't support recursive dicts", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 13, "deletions": 1}}, {"node": {"oid": "8a97b28ff7f2557c9bed66f5f17b19008737fccb", "messageHeadline": "Merge pull request #230 from EleutherAI/losen_reqs", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "53f6bc34a534d247cd2786babb990b03f07417ca", "messageHeadline": "update GPT3 test data and more docs", "author": {"name": "Jason Phang", "email": "jasonphang@nyu.edu", "user": {"login": "zphang"}}, "additions": 35, "deletions": 4}}, {"node": {"oid": "76ebb7927df062fb90e3736b7daae35f75127def", "messageHeadline": "Some refactor/clean-up, and GPT-3 update", "author": {"name": "Jason Phang", "email": "jasonphang@nyu.edu", "user": {"login": "zphang"}}, "additions": 367, "deletions": 228}}, {"node": {"oid": "a3d996172aeabc7020fecb978418f1b46530f398", "messageHeadline": "Add pytest.approx to version_stable", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "33ccde8602d46a58b724145f63f6462d71e2d9f5", "messageHeadline": "Add `truthfulqa` back to the task registry", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "36edc465987ad73ca565eb7106e16290be6b5208", "messageHeadline": "Increase verbosity level of `pytest`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e2bbe79a08d3aaf4c5a7d4da5c9d1389e0bc1fdf", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-\u2026", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "134150a2d635788b1f8bde70b7dc9f46a787ead1", "messageHeadline": "Comment out truthfulqa before tests are fixed", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "8325257572e28946fc8257186f66c6e26f5228d2", "messageHeadline": "Add newline separator for correct `rougeLsum`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 6, "deletions": 0}}, {"node": {"oid": "6350d6528aa4aca04a581f996134c77b474df1eb", "messageHeadline": "Update setup.py", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "61b14994e05053037d3b04f052640743ac3102ea", "messageHeadline": "Revert testdata", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "91e7f1823bf0d678253e819be7e5b47f2e319019", "messageHeadline": "Less strict versioning for black", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2ff1f5a2e8646dfde285c6803b21b8a2b651e697", "messageHeadline": "Add imports and fix comment", "author": {"name": "Leo Gao", "email": "lg@openai.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 4}}, {"node": {"oid": "8ef7a5154afc1c8ad12fa4444cd69c9ac01a702d", "messageHeadline": "Switch gpt3 to BaseLM", "author": {"name": "Leo Gao", "email": "lg@openai.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "fea936e69b8519e7afb25619a2061b21d2452684", "messageHeadline": "Merge TokenizedLM and TorchLM into BaseLM", "author": {"name": "Leo Gao", "email": "lg@openai.com", "user": {"login": "leogao2"}}, "additions": 158, "deletions": 163}}, {"node": {"oid": "b0746b3515b56bf6c082dda3aed37c8fb8a8d3f9", "messageHeadline": "Remove `t5` imports`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "e2de79daabbf6eb41c5b0d3612db6e67d7d0fea3", "messageHeadline": "Fix `rouge-score` version", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e3c74cc007557c6821686ed04ffb6cb332ab53d8", "messageHeadline": "Add newline to EOF", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "9e3368664457ba1ddcd634a706faffa72fbcec9a", "messageHeadline": "Merge branch 'truthfulqa-ver-bump' of https://github.com/jon-tow/lm-e\u2026", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f123c8cf43bf4f308e6818987cfc530e93863ae5", "messageHeadline": "Merge branch 'truthfulqa-ver-bump' of https://github.com/EleutherAI/l\u2026", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "0f27316c92028406d28679c7edc2b06d79ab8085", "messageHeadline": "Remove `t5` dependency", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 53, "deletions": 8}}, {"node": {"oid": "4a891f673af411583dea511ed7f4a0871ce49a09", "messageHeadline": "Merge remote-tracking branch 'origin/master' into truthfulqa-ver-bump", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ff8de12027f6d2d774bf56c3885f75122a37377c", "messageHeadline": "Pin python version to 3.9.7", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a73e80c7defb67f595476bb15303f0507f4f34b3", "messageHeadline": "Add testdata", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "05ed92a4bf0dfd5b1a0573d02711b455b93faa8f", "messageHeadline": "Bump the `truthfulqa` version to 1", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "724d728d6f4dba65a99d50502da21d06e0d6c8ca", "messageHeadline": "Merge pull request #225 from JunShern/fix-mkdir-with-parents", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "c59bfde8e82e8991d58bb12c6c288ce64cb1fd14", "messageHeadline": "Merge pull request #227 from jon-tow/truthfulqa-dependency-fix", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "0b7a79217ee0514b38aea59bc8c5a7bfb45a594d", "messageHeadline": "Merge remote-tracking branch 'origin/master' into evaluator-descripti\u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "3bdba226de828a03621010366cba5c8b98aaf39d", "messageHeadline": "Merge remote-tracking branch 'origin/master' into fix-mkdir-with-parents", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "ddf4339fa124971c486094cab55f68771bfe5e0a", "messageHeadline": "Merge branch 'master' into truthfulqa-dependency-fix", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "d19c5850f7533873004d14e0b8ce213fad6706d4", "messageHeadline": "Pin tensorflow-estimator==2.6.0", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "e8e652d353ea4944fe77e4a6b9f8375df966f676", "messageHeadline": "Merge remote-tracking branch 'origin/master' into fix-mkdir-with-parents", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "792a7656d32920c3021710e10e1750ce6a82b7ef", "messageHeadline": "Merge remote-tracking branch 'origin/master' into evaluator-descripti\u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "4748281bcc40843dca083fdd00621ec00662afd7", "messageHeadline": "Merge remote-tracking branch 'origin/master' into truthfulqa-dependen\u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "53df56d4f0f2df7b25a983e4808c6c4734ca651c", "messageHeadline": "Pin python to 3.9.4", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "84fe796e1c5f50985c92169ea68dc0e54c0b93c9", "messageHeadline": "Remove unused `bleurt` dependency", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "722579247e3bcc91aa5922decf8cbf70a9bd07e0", "messageHeadline": "Pin the datasets to the original source", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "c406a6204755dac18a91236c7b48134bc4d2dbb6", "messageHeadline": "Clear CI cache", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2cfbfd72d88466cf18c359ef75504531fdb5c396", "messageHeadline": "Update `truthfulqa` dataset checksums", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "57e064d0be0dc567d24aad596ebcb9cd3798d652", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-\u2026", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "8ac992696564547de95655ec250d16dc4bd41782", "messageHeadline": "Replace the `fewshot_description` API with a `description_dict` based\u2026", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 147, "deletions": 318}}, {"node": {"oid": "e7dd78efc54332d0a832650246f81927832a4208", "messageHeadline": "Always mkdir with parents", "author": {"name": "Jun Shern Chan", "email": "junshern.chan@motional.com", "user": null}, "additions": 4, "deletions": 4}}, {"node": {"oid": "84aa15c6e4cb65adf39c2dccf91a799cc7e6440a", "messageHeadline": "Fix testdata for truthfulqa", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "b67aec37cf047f4daae243b0d26f3be910daca5e", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "06d9be66491c878a850ed7d1af932b8ec8157360", "messageHeadline": "Merge pull request #181 from jon-tow/task-guide", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 283, "deletions": 0}}, {"node": {"oid": "fab2a6d8194b53305b00503fc0247bddf3c9f14f", "messageHeadline": "Fix spacing issue", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 9, "deletions": 6}}, {"node": {"oid": "de7d6614c5f74a6cf88a9d9a1c9e8fbd6bf14e77", "messageHeadline": "Re-order sections and adjust header structure", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 36, "deletions": 27}}, {"node": {"oid": "a5541c0224835747679305405a275eb2b45db4bf", "messageHeadline": "Update task-guide.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "cf8da97035814b7a2b2f49e49f6ef973484bd2a0", "messageHeadline": "Update task-guide.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 18, "deletions": 7}}, {"node": {"oid": "7f24a08b03d64b8466400a7bc9d42f6987ee2833", "messageHeadline": "Refactor LM organization for more reuse", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 275, "deletions": 247}}, {"node": {"oid": "33a94c5479c075404fdd84f597e8d9428aedd53a", "messageHeadline": "Merge pull request #219 from jon-tow/truthfulqa", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 373, "deletions": 1}}, {"node": {"oid": "85ffb7fc1d3d2e67e0b273a21de799d6efd7e886", "messageHeadline": "Add underscores to metrics", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 46, "deletions": 46}}, {"node": {"oid": "0538d4c94f934c07d47f828d4512c9d8f1d4c16f", "messageHeadline": "Add truthfulqa test data", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "dd59b0ef30611932d6f122c38990585e1d8d8327", "messageHeadline": "Adhere to spacing convention", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "e5066c6962bec3ca70c0ef6de613674e2a14b6c1", "messageHeadline": "Make gpt2 a little more tokenizer agnostic", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 23, "deletions": 13}}, {"node": {"oid": "7f8d667678f74b3c78318d48d8c0f81c6021d01c", "messageHeadline": "UPdate cost estimate", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 16, "deletions": 6}}, {"node": {"oid": "7a058d69cbb68f4cdc678c7d149ce52d217c6a36", "messageHeadline": "Improve interface and upgrade results dict format", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 73, "deletions": 42}}, {"node": {"oid": "b569f4838ecb003948f538eedcdcb492b41489fd", "messageHeadline": "add support for HF revision", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "3b81a361cfcdc0708bf332c18d53c09d84e958f0", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm-evaluation-harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 57, "deletions": 0}}, {"node": {"oid": "7f7b717dc9b7973e5c108e5330b4b9c8597191ac", "messageHeadline": "gpt2: fix bad multi gpu behavior and typecheck inputs", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 7, "deletions": 1}}, {"node": {"oid": "17c47812f3cb3666411d9f92c4078184d355c968", "messageHeadline": "Implement `TruthfulQA`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 369, "deletions": 1}}, {"node": {"oid": "3b11c19114fde0d4628edb83889a4a9238e9f4f6", "messageHeadline": "Merge branch 'task-guide' of https://github.com/jon-tow/lm-evaluation\u2026", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "4d147bddc1efab9e68664f3e5b9fc4c64699eca0", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-\u2026", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 4187, "deletions": 1050}}, {"node": {"oid": "011cc8913b0808603a892044b7db77fefad93ab0", "messageHeadline": "fix up some wording", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 22, "deletions": 15}}, {"node": {"oid": "f299be8f142d7f32f4798ffc6e257aafc0708e40", "messageHeadline": "Add missing colon to function call", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "dc937d4b70af819c5695e09d94e59e4cdb1e40ad", "messageHeadline": "Create CITATION.bib", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 26, "deletions": 0}}, {"node": {"oid": "69a2b49b440417b4d4722ea0d795393733f3278c", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 31, "deletions": 0}}, {"node": {"oid": "e037ef0aaafef2d67008713f1a55cca3c1ee1f03", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 7, "deletions": 3}}, {"node": {"oid": "8c79ccc0366a7603fa043e7547bf54927a239ced", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "feee4ed4cfa6b5f042a282afcebe840cb5ca15c0", "messageHeadline": "Merge pull request #216 from EleutherAI/fix_download_if_exists", "author": {"name": "sdtblck", "email": "46172032+sdtblck@users.noreply.github.com", "user": {"login": "sdtblck"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "031677b6fefd4eb86a3d4de7bdc564abd4c81864", "messageHeadline": "fix file download for existing files", "author": {"name": "Samuel Weinbach", "email": "samuel.weinbach@aleph-alpha.de", "user": null}, "additions": 1, "deletions": 1}}, {"node": {"oid": "49dc058d18a1eab9f4775bf494cba526f39111a1", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 0}}, {"node": {"oid": "72d39b7770ad03fca194fcf14a46e656f8c06004", "messageHeadline": "Rename package", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b50b1ffd4811d7d4a3a06550738ed20f92e1ad25", "messageHeadline": "Add MT lambada multilingual testdata", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 10, "deletions": 0}}, {"node": {"oid": "28fa2ddd7d2cef7c2e9af56c7088907426791b43", "messageHeadline": "Merge pull request #215 from EleutherAI/sdtblck-patch-1", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 40, "deletions": 11}}, {"node": {"oid": "754f1f244ffaf764466a9c2b3435ca58615209e8", "messageHeadline": "Fix lambada multilingual tasks", "author": {"name": "sdtblck", "email": "46172032+sdtblck@users.noreply.github.com", "user": {"login": "sdtblck"}}, "additions": 40, "deletions": 11}}, {"node": {"oid": "9d46382b7b31c24762bf94a82a69fc62d8349b53", "messageHeadline": "Move jieba and nagisa imports into their respective functions", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "13710677a4b6ac278f1017c78a4ba18c6402cace", "messageHeadline": "Merge pull request #204 from EleutherAI/lambada-multilingual", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 53, "deletions": 0}}, {"node": {"oid": "52f270dcf7a8a78101c34937eda6cd04542e8f13", "messageHeadline": "triviaqa OOM fix", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "e4766cd74e2fc9225fca572f78d659df56a938da", "messageHeadline": "Add zh/ja v1 data", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "dd3b4f7a05da5a3bec925a478556f71c20fd74e8", "messageHeadline": "evaloator: take min bootstrap steps for blue/chrf/ter", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "d67c77beab707a682adbdb7e76a7b3f414badf1e", "messageHeadline": "Merge pull request #214 from Muennighoff/tokenization", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 36, "deletions": 3}}, {"node": {"oid": "4d2c1522fad5df7a4c6c3834c872092436c84f2e", "messageHeadline": "Merge pull request #213 from silentv0x/master", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 127, "deletions": 82}}, {"node": {"oid": "82ec4f538a50c2c07e8644f75393e4da3eda507f", "messageHeadline": "evaluator: fix bleu/chrf/ter taking forever", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "c0f0f7e1a696a16756f1835eaf8366d2b041c73b", "messageHeadline": "Update translation.py", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 8, "deletions": 2}}, {"node": {"oid": "f046d331d68921cd49febabddaaf29cb0342d10c", "messageHeadline": "Add space tokenization for JA/ZH", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 28, "deletions": 1}}, {"node": {"oid": "81122e3d9d8a90e9e8fac6dd0952565d29ad28a0", "messageHeadline": "Update version and testdata for DROP", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "5dbeba0da552450461a264a6853039fd015422ae", "messageHeadline": "Bug fix", "author": {"name": "silentv0x", "email": "incessantjunk@gmail.com", "user": {"login": "silentv0x"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "35d809c30ca41670d457f24b9c1a2b11bcc47fa4", "messageHeadline": "Fixes DROP implementation", "author": {"name": "silentv0x", "email": "incessantjunk@gmail.com", "user": {"login": "silentv0x"}}, "additions": 124, "deletions": 81}}, {"node": {"oid": "198ca7329b3d370885a3656dfdcc2cc92b1cb405", "messageHeadline": "fix squad breakage due to HF change", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 8, "deletions": 1}}, {"node": {"oid": "fc0d23776ca8f108980e7a743a3e95c5d4498388", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 35, "deletions": 25}}, {"node": {"oid": "e35386d9ab3ed2f400da718a26fc8ce6c0595a7f", "messageHeadline": "Fix version issue", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "592b2a23d208da02dcc7988d86282e9ca0fc2bc9", "messageHeadline": "Merge pull request #210 from EleutherAI/StellaAthena-patch-1", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 16, "deletions": 5}}, {"node": {"oid": "adec7faab69641f018527e9127abcdf3a87e560c", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "ba3fa9c87e1fa00c9ad7a79fa1739ecc318a04a7", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "f6e7ae258ca08b381ae1aaba692fd68340717430", "messageHeadline": "Update main.py", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "93ea091e7d24df77529b131e9f8db6d263eecdb8", "messageHeadline": "Updated readme for clarity", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 14, "deletions": 3}}, {"node": {"oid": "af0b1e26a8a938509a4034fa84784a492e10095d", "messageHeadline": "Merge pull request #209 from EleutherAI/sdtblck-patch-1", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 7, "deletions": 5}}, {"node": {"oid": "75519897b9aa454999f880fd84d9280876012913", "messageHeadline": "Don't download lambada if it already exists", "author": {"name": "sdtblck", "email": "46172032+sdtblck@users.noreply.github.com", "user": {"login": "sdtblck"}}, "additions": 7, "deletions": 5}}, {"node": {"oid": "db85d5aea6d86999114ad767c079c709910fda7f", "messageHeadline": "Add missing functools import", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "a10b05c576d90313babb8f71801ae6834fc668d5", "messageHeadline": "Add missing json import", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "3473ea80ae808dea477a8ab71120b4a0f22d0191", "messageHeadline": "Merge pull request #206 from jon-tow/mutual-fix", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 7, "deletions": 3}}, {"node": {"oid": "cf5823cff6ac6cb150a541d7052872b39416ebca", "messageHeadline": "Merge branch 'master' into lambada-multilingual", "author": {"name": "sdtblck", "email": "46172032+sdtblck@users.noreply.github.com", "user": {"login": "sdtblck"}}, "additions": 283, "deletions": 1}}, {"node": {"oid": "11f0e6d8eaa269a031aa407116d3862198bc5c96", "messageHeadline": "Indicate that it's machine translated in task name and comments", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "0fec45555a4ad4a53782c84bc08d4e9d9212058f", "messageHeadline": "Add mutual v1 test data", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "6b6303d4cb3fae65a16a6ae2ec6d6453c2b1c9f5", "messageHeadline": "Stable file order and don't lower", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "69c8345673fa2102bbba919df2b58e11493d909f", "messageHeadline": "Fix `MuTual` loglikelihood request bug", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "6fa1a4fa320dab5ec8a44ffbbe8b30f1cd53705e", "messageHeadline": "Merge pull request #201 from jon-tow/mc-taco", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 136, "deletions": 0}}, {"node": {"oid": "9676b93fd7581c99e3dd3e60205c7311f9f84966", "messageHeadline": "Add MC-TACO stable test data", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "3e1bcf712e654a53d974961dc53c721675d6c8f0", "messageHeadline": "Add MuTual test data", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "8516571b0187b499c89f76d723c28df08a3b93a1", "messageHeadline": "Merge pull request #198 from jon-tow/mutual", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 142, "deletions": 0}}, {"node": {"oid": "cb8a1b2363cb868acbdf84d641090c4bb52de922", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness \u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "9a58cee6bf7fbe7adcf31bf795e482889c2413fd", "messageHeadline": "Grammar cleanup in the task docstring", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "40bdb0c46b61128e71f9b37c95600f10aca1297e", "messageHeadline": "Update __init__.py", "author": {"name": "sdtblck", "email": "46172032+sdtblck@users.noreply.github.com", "user": {"login": "sdtblck"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "5b008dc053c5493c731c4150370cb59e746731f3", "messageHeadline": "Update lambada_multilingual.py", "author": {"name": "sdtblck", "email": "46172032+sdtblck@users.noreply.github.com", "user": {"login": "sdtblck"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a92d47eb7515b11f2dd7e2652a56f65c928e0ed4", "messageHeadline": "Update __init__.py", "author": {"name": "sdtblck", "email": "46172032+sdtblck@users.noreply.github.com", "user": {"login": "sdtblck"}}, "additions": 7, "deletions": 0}}, {"node": {"oid": "4cd4b05cd56103d38f6513feb79032b11eef3238", "messageHeadline": "Create lambada_multilingual.py", "author": {"name": "sdtblck", "email": "46172032+sdtblck@users.noreply.github.com", "user": {"login": "sdtblck"}}, "additions": 40, "deletions": 0}}, {"node": {"oid": "213149ad8eff1eb4318ba68da4064049188dba89", "messageHeadline": "Merge pull request #203 from jon-tow/best-download-update", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "6b0a89260cc07ef4fb57db0ccc906a9af10b6975", "messageHeadline": "Update `download_file` calls to reflect api arg name change", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c7fd34c25c4bba39e3122f3e0e2b909990aa513e", "messageHeadline": "Replace `wget` downloading with `best_download`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "e5db2a1c862bfcca7d6a8de7b870ba30e17f722d", "messageHeadline": "Implement `MC-TACO`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 134, "deletions": 0}}, {"node": {"oid": "ebcb28f2482fcaa4cd59b4aab4419853a9cf6b44", "messageHeadline": "Add detokenizer", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 19, "deletions": 3}}, {"node": {"oid": "d235355a9a78ea2442344262920a9ea26bc8c237", "messageHeadline": "Remove unnecessary parentheses", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "31150fac760c026c6be923628a81e44026672ca4", "messageHeadline": "Fix data path location", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 7, "deletions": 6}}, {"node": {"oid": "d7076a634ec7cb95c0e98ab794cf1dc8e612d5d2", "messageHeadline": "Implement `MuTual`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 122, "deletions": 0}}, {"node": {"oid": "be893e9df2526c01e30adb93d1e5fb916566f441", "messageHeadline": "Update readme", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 10, "deletions": 0}}, {"node": {"oid": "0f4e3df48c28d992a829c7000e02310568810be7", "messageHeadline": "Fix typo", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "893ef63fdedf0f2275aa8b29d20d40a98e7b59b8", "messageHeadline": "Test training docs too and don't cache downloads", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 13, "deletions": 2}}, {"node": {"oid": "7cbd0bba4aab4e8f90583e77123126f01e562fe7", "messageHeadline": "Look at all docs of task in test", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "dc648d842976d1771d1628d4b423120891918321", "messageHeadline": "Merge pull request #195 from EleutherAI/sdtblck-patch-1", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 3}}, {"node": {"oid": "41b0ae2291bf4188aadb25a7d4ff6cfe1b9338b4", "messageHeadline": "Update test data for coqa v1", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "f14571fbc7cebccd9258b044e5d8ccf26e57846a", "messageHeadline": "Actually do things correctly this time", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "278863b50c7dd4dce6a8d284c4547c454d94f9e0", "messageHeadline": "Bump CoQA version", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2078adbacc13291b5c52990cc3e361dba12f9884", "messageHeadline": "Make CoQA take the first line of output excluding any leading whitespace", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "7069583be0435fc0a1407377e43f8236472fe2c2", "messageHeadline": "Set dummy secret key", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "21a1178c530fdc81d6a68dd934572a4fc79ce093", "messageHeadline": "Add openai dependency", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "c59cd3340388b05f677a055f1755cc2f34383246", "messageHeadline": "Add gpt3 tests", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 90, "deletions": 1}}, {"node": {"oid": "cc474a9448eb3565ebc0c05b6ccfd65f32baf5c6", "messageHeadline": "Change generate until sequence in coqa to be \\nQ:", "author": {"name": "sdtblck", "email": "46172032+sdtblck@users.noreply.github.com", "user": {"login": "sdtblck"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "9b9d8dd7652eee05d911f5d94598ea02fb01bdfe", "messageHeadline": "Exclude abstractmethod and notimplemented stuff from coverage", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 18, "deletions": 1}}, {"node": {"oid": "8f8a1dff64a8e1717812f13eeeb16597ddd33585", "messageHeadline": "Remove a bunch of dead code", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 281}}, {"node": {"oid": "6e5b3cde9063a328039993af31fb47f4bc514c9a", "messageHeadline": "Add stable version hash for prost", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "51a567cf935f9c46220bbdf401e98d5db47639f8", "messageHeadline": "Merge pull request #194 from jon-tow/prost-task", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 58, "deletions": 0}}, {"node": {"oid": "f4e380816fdff6999365f507f8c265f7fdb49077", "messageHeadline": "Implement `PROST` task", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 58, "deletions": 0}}, {"node": {"oid": "f9cc0267360382ef5b7c7cbd831c6b3b4a857c47", "messageHeadline": "Use hashed version stability test instead", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 366, "deletions": 340}}, {"node": {"oid": "10d4b64a4c5c083064c2af19dc0e51fd33b38ffd", "messageHeadline": "Minor WikiText fix", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "5e31e40e82046b8c4a9766234448805aa299116a", "messageHeadline": "Implement WikiText", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 89, "deletions": 124}}, {"node": {"oid": "efa99cb2677517b06c8021f05a1bb2fda7eea971", "messageHeadline": "TriviaQA memory fix", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 9, "deletions": 15}}, {"node": {"oid": "c4ecbd6d19b5001c81a10984f0d4241d195c03c0", "messageHeadline": "Omit unimplemented tasks from coverage", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 11, "deletions": 15}}, {"node": {"oid": "142ee24fa3c66d595b803641d9fed51ab96a3e91", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "04a81391c3659d22658642562250050b66b31a2a", "messageHeadline": "Make codecov use token", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e87200ed1ae2a0ccd20ee78e5e88993f8bd31276", "messageHeadline": "Rename CI pipeline", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f685cecf06d06aede54a2120474b84e28b776ae9", "messageHeadline": "Fix broken coverage", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "3b990af0baeff08a7ed2115d296c7bebdd41491d", "messageHeadline": "make hendrycks math order deterministic", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 8, "deletions": 8}}, {"node": {"oid": "90429563fe1150d3bbee7d86fd7504b7f45173b0", "messageHeadline": "Actually fix hendrycks math", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 4}}, {"node": {"oid": "2810c634fef4978d7945ee88e1818360ec14296e", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "004b63e306ebf8a8c41233a5011131aadc1c41c0", "messageHeadline": "Fix hendrycks MATH to use new download link", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "54efc9cd49c63975ad73d179d9f9f75bd4dc56cf", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "36c616749a63fbe826ec88128f57b04a73f973fb", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "b6776b82f8141f9820b6d9bd353345e8896eb9c0", "messageHeadline": "Bump best_download version", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "c9f81dc9b5cc6ca746e77fca9099c6166d6387d1", "messageHeadline": "Update python-app.yml", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "20229183f72676ed9bd11fb663278b80ea883702", "messageHeadline": "convert all enabled tasks to use best_download for better download ro\u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 40, "deletions": 28}}, {"node": {"oid": "85277915897af5920f7db39d22716cb2159eae9b", "messageHeadline": "fix bug where interrupted pile download doesnt correctly resume", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 4}}, {"node": {"oid": "17c825d1828c9c7f9108268c5888a8bdf2295930", "messageHeadline": "Add tests to make sure task definitions don't change without the vers\u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 431, "deletions": 0}}, {"node": {"oid": "105fa9741ff660f6a62c2eef0d2facfde36dda41", "messageHeadline": "Add task versioning", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 93, "deletions": 8}}, {"node": {"oid": "f76e636772e24e55af4ad2c0f2fdc69edc32b8a7", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "9d6d291f5fbd381cc408ab847d8ec0be96ffe7b2", "messageHeadline": "make create_from_arg_string fully general", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 9, "deletions": 7}}, {"node": {"oid": "031e6acabfd14b8e01930a155faccbd5d33bbab4", "messageHeadline": "Add missing () in fewshot-no-training-path", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "2bb67c988b7cd3a75b516c0f7b4ab7118a835edc", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1068, "deletions": 102}}, {"node": {"oid": "9c76176bd1b108139d1632f2eccc3b9ebaa6eaef", "messageHeadline": "naturalqs: fix spacing", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "aecf13dd53de0c9aa5cffff58b067492f4521ce6", "messageHeadline": "fix hendrycksMath for \\fixed without curly braces", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 13, "deletions": 6}}, {"node": {"oid": "9bbf2406d1615d82a887d85f6ccf83f130c14d9f", "messageHeadline": "remove erroneous entry for Pile-CC", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "99d40991b75d3429c9cfa3a7a73ba123b33796a3", "messageHeadline": "Merge pull request #183 from EleutherAI/researcher2", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1067, "deletions": 99}}, {"node": {"oid": "3ddfa4a632d1ac1e091dcf6cffe46930c91b05d2", "messageHeadline": "Merge pull request #182 from EleutherAI/sdtblck-patch-1", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 3}}, {"node": {"oid": "8a61f7710bad228261daadb9e79e18c85b612c0d", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 186, "deletions": 189}}, {"node": {"oid": "47e3ae6c21e8de62107cda2acb6b039968c0f02d", "messageHeadline": "Add column to task table", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "43f4d418e4785472e65df164a0a4da5103bcdeae", "messageHeadline": "Remove PileCommonCrawl (empty task)", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 4}}, {"node": {"oid": "2a254220fa1fb99fbf2e92bf4ac21f10a9e80cf2", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 180, "deletions": 184}}, {"node": {"oid": "7549befa4f0d3cb6033be82b4f5987144de188a1", "messageHeadline": "Add inits", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "d0277a3fbb59fa2f326e0a1201805b475b80c19e", "messageHeadline": "Update janitor.py", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 76, "deletions": 76}}, {"node": {"oid": "7cb91943a6efb0cff12d9db6246f74d308d82822", "messageHeadline": "Janitor testing and changes", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 518, "deletions": 16}}, {"node": {"oid": "2a11129d188860b528befc98395827acac4cc22e", "messageHeadline": "Add pile 13gram processing.", "author": {"name": "researcher2", "email": "32028883+researcher2@users.noreply.github.com", "user": {"login": "researcher2"}}, "additions": 466, "deletions": 0}}, {"node": {"oid": "ab8da1f06045d6919b0cd221cac8e5edc38181ca", "messageHeadline": "switch lambada over to wget while researcher2 fixes best-download", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 6, "deletions": 5}}, {"node": {"oid": "ebe9e6b90abdc6667e7ad95dc426b4fdf3aea2fc", "messageHeadline": "Update gpt2.py", "author": {"name": "sdtblck", "email": "46172032+sdtblck@users.noreply.github.com", "user": {"login": "sdtblck"}}, "additions": 1, "deletions": 3}}, {"node": {"oid": "a14adbe0a2c851319ea7f744b5842eaf76fc0d12", "messageHeadline": "Add lmd dependency", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "fbcb3983c30929fcc0de75b58e881a56b911dd21", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 184, "deletions": 99}}, {"node": {"oid": "e6decf02afaa231fd802670626614381459ce4bf", "messageHeadline": "gpt2: extract model call", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 11, "deletions": 1}}, {"node": {"oid": "7a39d68b345ecc8b70c863b8209cf82650734438", "messageHeadline": "Make bootstrap_iters configurable and fix some tests", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 17, "deletions": 14}}, {"node": {"oid": "0a400af9d0586a14b1482cc796c0dc65712b0d5b", "messageHeadline": "Update task table", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 184, "deletions": 99}}, {"node": {"oid": "0fa316f8204798fc22f1e5dae1f19a9628be93f7", "messageHeadline": "Make PerplexityTask fit the framework a bit better", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "3830713f093106e36c408345b3e25f16c97aba79", "messageHeadline": "Make PerplexityTask fit the framework a bit better", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "b691c44fc853aecc5f1a42a29ca4848b5fbc80c4", "messageHeadline": "Count perplexity correctly", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 10, "deletions": 5}}, {"node": {"oid": "9c4967bc857260d9ea0798a3127de552f42c9621", "messageHeadline": "Multithread bootstrapping", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 19, "deletions": 6}}, {"node": {"oid": "c77b60c1764a82593ac96a320c8e332d5eccdffc", "messageHeadline": "Merge branch 'perplexity'", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 684, "deletions": 20}}, {"node": {"oid": "eb8456d09f6b3e2ae29603747d224c429ef55046", "messageHeadline": "Merge branch 'batching'", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 82, "deletions": 32}}, {"node": {"oid": "5f42f9761753c79ce1febd7179783dbe72a0254f", "messageHeadline": "Overhaul command flags a bit", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 14, "deletions": 9}}, {"node": {"oid": "7f7673ec7ad7cb281b4af3ee7afb2bd3fa8174d6", "messageHeadline": "Add comment", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 7, "deletions": 1}}, {"node": {"oid": "59aff21d9d9dd8bd13dba9650e7732324b1e6a17", "messageHeadline": "Fix tests", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 10}}, {"node": {"oid": "0224985531bf572a5313d411e2b22d18fb95381a", "messageHeadline": "Use sample stddev", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 16, "deletions": 4}}, {"node": {"oid": "82a538d33265cb2b8dd42511d77cbaaed1f61e4c", "messageHeadline": "Make bootstrap test work", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 3}}, {"node": {"oid": "5452fddbe4ef50a1d008c13a61935aa6e888748e", "messageHeadline": "Use sample stddev", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 16, "deletions": 4}}, {"node": {"oid": "baf54a03ce7a11a6796dcbdfa022343c4d04e509", "messageHeadline": "Make bootstrap test work", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 3}}, {"node": {"oid": "00cecd5984490cf098356f4e7b2a57222cc61045", "messageHeadline": "Add task source header and versioning guide", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 27, "deletions": 2}}, {"node": {"oid": "24d50a705f0ba0215fff731835fc642789b3ddc7", "messageHeadline": "Add task implementation guide", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 226, "deletions": 0}}, {"node": {"oid": "dd573a95882dc6b5b93fbb89eb7092253c4ff7b0", "messageHeadline": "Fix Typo", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1731613e056bb5c4ac00b2d98f27eb0203a17efb", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness \u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 98, "deletions": 14}}, {"node": {"oid": "f9d87ad3eb1e8c44f0ec30ef4639b8be99885965", "messageHeadline": "Merge pull request #180 from EleutherAI/packaging", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 15, "deletions": 12}}, {"node": {"oid": "965038daef98157b743373f1a49bafb2b59fed22", "messageHeadline": "move deps to setup.py so they get installed automatically", "author": {"name": "Ben Wang", "email": "wangben3@gmail.com", "user": {"login": "kingoflolz"}}, "additions": 15, "deletions": 12}}, {"node": {"oid": "45127aa73e96d9c40ea800d24473b1b0d764629c", "messageHeadline": "Rename a ton of stuff", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 52, "deletions": 52}}, {"node": {"oid": "80a739eb37a6bfd7f0ec489ee86e35a684af1c76", "messageHeadline": "Fix some things for tests", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "f16c301e6ec690b0de1b75b6bd102a86110a217c", "messageHeadline": "Minor changes", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 7, "deletions": 8}}, {"node": {"oid": "1e7f884d0bb7caf6d912c808da677cafa8c0f8f8", "messageHeadline": "Refactor PerplexityTask", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 28, "deletions": 12}}, {"node": {"oid": "b0cf01636b0652653fcacf347c648b2c999e2bb0", "messageHeadline": "Begin refactoring perplexity code", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 57, "deletions": 26}}, {"node": {"oid": "ee5467ffcfdea8739a0ae84437e2a32c6b9a1f4f", "messageHeadline": "adding tests sortof", "author": {"name": "Jason Phang", "email": "jasonphang@nyu.edu", "user": {"login": "zphang"}}, "additions": 231, "deletions": 2}}, {"node": {"oid": "218d105d5c41c82b67565383da9d3d305dd25811", "messageHeadline": "Test bootstrapping", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 10, "deletions": 0}}, {"node": {"oid": "c1822c026dbdaed0d1a79b3654030464e039c80f", "messageHeadline": "Update metrics.py", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "0d9cc5a8c0dd6e0fcb9bf706f48ab39b0cbacc09", "messageHeadline": "Update metrics.py", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "3a490624c112dddc1f429e2d7991cf00940cdad2", "messageHeadline": "add lm_dataformat to requirements", "author": {"name": "Jason Phang", "email": "jasonphang@nyu.edu", "user": {"login": "zphang"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "8288015453ceaf4706e5519b94002794b2309318", "messageHeadline": "add pile tasks", "author": {"name": "Jason Phang", "email": "jasonphang@nyu.edu", "user": {"login": "zphang"}}, "additions": 108, "deletions": 3}}, {"node": {"oid": "fa80f7bda9792ff7a7b62d995b9cb139d57f364f", "messageHeadline": "gpt3 compatibility", "author": {"name": "Jason Phang", "email": "jasonphang@nyu.edu", "user": {"login": "zphang"}}, "additions": 49, "deletions": 1}}, {"node": {"oid": "9454c839699708fed1efd152f82059de7b28ae7d", "messageHeadline": "gpt2 perplexity", "author": {"name": "Jason Phang", "email": "jasonphang@nyu.edu", "user": {"login": "zphang"}}, "additions": 231, "deletions": 9}}, {"node": {"oid": "ba18bb4f43840a265035af64bef3b2948f0ccc12", "messageHeadline": "Merge branch 'master' into batching", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "e7cded608edb44e41d3b02a42d97ce1e07b63e89", "messageHeadline": "Merge branch 'batching' of github.com:EleutherAI/lm_evaluation_harnes\u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "45fefe9f59aa193779644fc833b1e8d18644e9fe", "messageHeadline": "implement stderr calculation", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 72, "deletions": 0}}, {"node": {"oid": "726ea95e0f28476ee3e4b3a9b3b33dbf8119404a", "messageHeadline": "implement stderr calculation", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 72, "deletions": 0}}, {"node": {"oid": "853afa366e57e481a18764c3e89777fcfb5f27a0", "messageHeadline": "Make hendrycks math tar extraction quieter", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "dd8d35295d00b2a5b65abc7864579fb8ae0ff680", "messageHeadline": "Fix first-run error", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ab0e4d262776638d194bf2a603a04400842cac4d", "messageHeadline": "Disable triviaqa and cbt tasks", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "8f4d1b494d2c50c4dc3d942d111a5340a2c6228d", "messageHeadline": "Add some more tests to test is_greedy", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 8, "deletions": 1}}, {"node": {"oid": "31c29e3b99bf6a8ca490a7937c77c6d692ff2c76", "messageHeadline": "Actually fix problems and make tests pass", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "fe0311b6ecf1c695659be7356c0467a02f4cbf24", "messageHeadline": "Fix problems", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 8, "deletions": 9}}, {"node": {"oid": "be3a6a2db4abf51e4d6a8834470d0443982c9343", "messageHeadline": "Initial implementation of gpt2 batching", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 58, "deletions": 19}}, {"node": {"oid": "8846bec0c33f4a1bdc2c38e8453412c473e37546", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "c5c09c267813749eea1cb3950141eb1ffb366700", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 115, "deletions": 0}}, {"node": {"oid": "a586a5c4f6ec1c12db935a1f3139eae2d631814e", "messageHeadline": "More refactoring of model code", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 12, "deletions": 12}}, {"node": {"oid": "ab1fdc548c5b153b317a00642a747081f973f087", "messageHeadline": "Refactor gpt3 loglikelihood", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 19, "deletions": 10}}, {"node": {"oid": "8352e6716ef418ecffb4f73775cf7bde74952a29", "messageHeadline": "Refactor gpt2 loglikelihood", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 20, "deletions": 13}}, {"node": {"oid": "edc66ecccd06d8a04be4972f5f8c5167a57e133b", "messageHeadline": "Merge pull request #169 from jon-tow/cbt-evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 115, "deletions": 0}}, {"node": {"oid": "cd1b32f92cda7f775b3dd4d15a8ecac5f7fc723f", "messageHeadline": "Merge branch 'master' into cbt-evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 254, "deletions": 183}}, {"node": {"oid": "eec180184134385074341e92707c71f6a97e4e53", "messageHeadline": "Add more gpt2 tests", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 59, "deletions": 2}}, {"node": {"oid": "5dc4692a57d2f08c314199accf0d019591a94fb4", "messageHeadline": "Fix triviaqa", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "29c1f228dbf7aaacf5774749bb1371c5bf24cdcb", "messageHeadline": "Fix typo", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 19}}, {"node": {"oid": "ec16d81d1299c60e081261c3a0c8ae2b3846bb5c", "messageHeadline": "Factor _convert_standard into MultipleChoiceTask", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 6, "deletions": 79}}, {"node": {"oid": "44f03593437c09185b645626b5f4091446495f00", "messageHeadline": "Merge pull request #176 from EleutherAI/per_char_agg", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 31, "deletions": 26}}, {"node": {"oid": "1ebf41d3f732bf2019a126f63c6b2beb9aa56edd", "messageHeadline": "Fix import", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "4462e41593d48fe2abc4b49d4c80e3d1c379bb9e", "messageHeadline": "Make piqa inherit MultipleChoiceTask", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 23, "deletions": 26}}, {"node": {"oid": "2522c27f6a3a43324816866400415f561d42092c", "messageHeadline": "also apply normalization to piqa", "author": {"name": "Ben Wang", "email": "wangben3@gmail.com", "user": {"login": "kingoflolz"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "3eaa493e4bcffcd63b3b38d513b5be4f25a320eb", "messageHeadline": "Make acc_norm a separate metric", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 8, "deletions": 5}}, {"node": {"oid": "fae5fe666df8170c988b029e8efc426c204b89f5", "messageHeadline": "do per character loss aggregation for multiple choice tasks (similar \u2026", "author": {"name": "Ben Wang", "email": "wangben3@gmail.com", "user": {"login": "kingoflolz"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "fd26ef161180e0e83a007d8d426b4b3609742043", "messageHeadline": "Merge pull request #172 from EleutherAI/lambada_cloze", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 19, "deletions": 0}}, {"node": {"oid": "7ba8c1836497b079be91c89d4bfd8ff7c0ff1c5c", "messageHeadline": "Use more inheritance", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 54}}, {"node": {"oid": "473827172b938f7549b64a2b0fb43f0d1eef5d48", "messageHeadline": "add cloze variant of lambada task for allegeldy improved few shot res\u2026", "author": {"name": "Ben Wang", "email": "wangben3@gmail.com", "user": {"login": "kingoflolz"}}, "additions": 71, "deletions": 0}}, {"node": {"oid": "e971baac7d1bc21c3cf78fcabbd0e96affa89a74", "messageHeadline": "Fix triviaqa memory consumption problem", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 11, "deletions": 5}}, {"node": {"oid": "2b8956b8ed7c919dcb2733678d7c1e6c72865ba9", "messageHeadline": "gpt2: Mask out all tokens above 50256", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "efbe6e7fb08fda40d685c8f0b4b998b5215ef9f6", "messageHeadline": "Implement partial caching", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 50, "deletions": 6}}, {"node": {"oid": "8fe59e59119a703c62325f4c7b762eb55286851e", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 59, "deletions": 39}}, {"node": {"oid": "f3fee648d0aad94b9f3f9d14a9e5e6e20ff892e5", "messageHeadline": "Roll back last token optimization", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 7, "deletions": 9}}, {"node": {"oid": "6242a00df3b95ff80f0b90caa7627e7a9b9515eb", "messageHeadline": "Merge pull request #170 from jon-tow/ethics-fix", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 59, "deletions": 39}}, {"node": {"oid": "d78917a78f43eda9354a5a1f7dfa2a61cc365aff", "messageHeadline": "Add note on accuracy reporting", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 6, "deletions": 0}}, {"node": {"oid": "48ae5457b46af121ddb1c71b04a19fea4b6fa4f1", "messageHeadline": "Clean up `Ethics` few-shot prompts", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 53, "deletions": 39}}, {"node": {"oid": "59a0104d937c95491ae79a1d32b09d945e3a90e9", "messageHeadline": "Remove unused simple_accuracy_metric", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 10}}, {"node": {"oid": "2ced79f5723b95d36690d78f1237201a6d15c975", "messageHeadline": "Add \"choices\" to logiqa, hendrycksTest", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "49ee4db245e143472ae9fb997a67de8eaffb799d", "messageHeadline": "Handle GPTNeoConfig not having n_ctx", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "44d7b2fa0e13d46b6897d4b38f6fb2ca17d1daae", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "310ca7a8f384cb91afd2404c201af06f11f23510", "messageHeadline": "Add example header", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 15, "deletions": 1}}, {"node": {"oid": "a10856dc56dcf136022b8cfbd1c63cbb041e44d4", "messageHeadline": "Implement `Childrens Book Test` dataset download and evaluation", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 114, "deletions": 0}}, {"node": {"oid": "c463265565b6ec6ed6b5245d3ebd93227bc75e35", "messageHeadline": "Merge pull request #168 from EleutherAI/translation-fix", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "3a2b7df409027d1789a0cbdd57101fea6bfe77c7", "messageHeadline": "fix translation scoring format", "author": {"name": "&", "email": "", "user": null}, "additions": 3, "deletions": 1}}, {"node": {"oid": "5aa601f3d406cbb099d5437ec3909539f79f7e34", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 73, "deletions": 20}}, {"node": {"oid": "42659c342b72b31c098be915009bfcb285479d96", "messageHeadline": "Rename hendrycks ethics and math", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 15, "deletions": 15}}, {"node": {"oid": "f984c88e500d03c5566a3819da3eec9a50f3a2d9", "messageHeadline": "Merge pull request #140 from cfoster0/master", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 73, "deletions": 20}}, {"node": {"oid": "4b133dca18252c8e10e6c13e2b6ce8e502917615", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness \u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 659, "deletions": 160}}, {"node": {"oid": "8de855340c61b1a2c8ada0a80490429139a18360", "messageHeadline": "Rename task", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "cbc5c9c8c2193af3f9d7b876272f926fa7009031", "messageHeadline": "squad: fix aggregation", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 36, "deletions": 26}}, {"node": {"oid": "caba51e18093d9d41262e2ae7e6082e89fbd3add", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 42, "deletions": 102}}, {"node": {"oid": "5643f20f6f8b2870cac820bc1f1f9fade6bcb726", "messageHeadline": "Update superglue winograd has_test_docs", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "0d663b6b4ec1f26312302fa3c207fa9f4c341d36", "messageHeadline": "Update write_out.py", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "c445e56b3b2364916ce67eebe966cc6697503e06", "messageHeadline": "Merge pull request #167 from jon-tow/write-out-random-fix", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "d96b991da932f10947c7d0bc4684f542e4d580dd", "messageHeadline": "Add `Random` argument to `write_out` `fewshot_context`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "7ff58fe17c9c212f5387a3fcfaf86d22075cb79a", "messageHeadline": "Merge pull request #166 from jon-tow/remove-unused-imports", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 38, "deletions": 100}}, {"node": {"oid": "a69ba385be3ad8d1c4bcbd7e334671a51a53ecef", "messageHeadline": "Fix translation prompting", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "fa7b9643b1dd8e3229681cd8e69129847665e304", "messageHeadline": "Mark GLUE and SuperGLUE tasks without labelled test docs as not havin\u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 11, "deletions": 11}}, {"node": {"oid": "ddc044eb56c0facb2446da60d8f053badd1b8483", "messageHeadline": "Remove import of the non-existent `NLP_TASK` module", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "9a9c8e3c82466ee41dd40108f3bbebf1678e498d", "messageHeadline": "Remove unused imports and format imports outside of `lm_eval` module", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 4, "deletions": 19}}, {"node": {"oid": "d5d1921935522a94cbe627ab92284e893a8eafb2", "messageHeadline": "Remove unused imports and format imports", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 32, "deletions": 79}}, {"node": {"oid": "1a05b8cf37eb03f9440e29d08d7d64441166c827", "messageHeadline": "Revert evaluator change", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "ca57061c7dc456285aa1692aee23b085441abeac", "messageHeadline": "Patch gpt3lm", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "bc5478a16b59c3ddf817b49a40c37db7e14de379", "messageHeadline": "Minor updates", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "c971fa82e018b94de78400cd3f09b839c826f6ce", "messageHeadline": "Fix stuff and make tests pass", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 10, "deletions": 6}}, {"node": {"oid": "0966e7b6a83e939cde0dbb07a944e2fad3304f94", "messageHeadline": "Move fewshot stuff back into the right block", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 11, "deletions": 11}}, {"node": {"oid": "3b2d5f6c055ebc79f365cd327249e098ecac8450", "messageHeadline": "Pass random state around to fewshot_examples and fewshot_context", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 21, "deletions": 19}}, {"node": {"oid": "35f58b95cff22497c3b1fb0a47f9c079cc937b72", "messageHeadline": "Change to test before val", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 5}}, {"node": {"oid": "0c402690081b9f9865f3f46f0f6214dc534d4483", "messageHeadline": "Fix fewshot for tasks without train docs", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 15, "deletions": 1}}, {"node": {"oid": "8e8d7c6de3b5d334389305f417e551807eb8d74e", "messageHeadline": "Add Reorderer and implement in gpt2 and gpt3", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 72, "deletions": 9}}, {"node": {"oid": "1b4242c146d49f3833f8aabd3467eceda466822d", "messageHeadline": "More changes to make neo work", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 10, "deletions": 5}}, {"node": {"oid": "747b851d7e139d696378b738f3a130d69452750d", "messageHeadline": "Update GPT2LM to handle neo based models as well", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 4}}, {"node": {"oid": "c84a4af4d832f55ac1270fbccdf37ca4691a3a12", "messageHeadline": "qa4mre: remove fewshot_examples that uses test docs", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 6}}, {"node": {"oid": "d7b20fe24c504db7d457f2cfd11cb84d2687caa3", "messageHeadline": "naturalqs: update random", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "d42af30df9e6c109bd15effc631c9a6fc907fe40", "messageHeadline": "wsc273: change random", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "dc1a17b1a0baa6b21e160cd4ead8075e7556b466", "messageHeadline": "PubmedQA: remove redundant fewshot_experiments", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 6}}, {"node": {"oid": "de47e6e31c1c8ab384c6931eb1e187699152879b", "messageHeadline": "Merge pull request #162 from andyzoujm/master", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 119, "deletions": 0}}, {"node": {"oid": "95daf009e1240383f28e2a2c097f0f402d7aa9b2", "messageHeadline": "Merge branch 'master' into master", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 354, "deletions": 3}}, {"node": {"oid": "e211219642ed441418b0fc21acddf679eea95aca", "messageHeadline": "Update fewshot_examples", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "6a78fdaf454bc8f3801dfe1de31121bebff16d48", "messageHeadline": "Deterministic fewshot_experiments", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "283b2c5e4da5d060069ecee98582e63b86cb822f", "messageHeadline": "Merge pull request #161 from erictang000/master", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 332, "deletions": 0}}, {"node": {"oid": "58589f396f9d7d16ca9d451a64935565ab148ac2", "messageHeadline": "Remove fewshot_examples", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 14}}, {"node": {"oid": "d22ea1f3e45d3039b493f3b9fe7e6b77bda96573", "messageHeadline": "Add `answer` key to training docs for uniformity", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 9, "deletions": 10}}, {"node": {"oid": "410ca65fbdfbb242239b410df9a5444a0c4adca1", "messageHeadline": "Split `test` and `train` doc loading and address minor fix-ups", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 60, "deletions": 73}}, {"node": {"oid": "14dd29c442552102d5bc6e4337ef31dda59a24fd", "messageHeadline": "Fixed calling of loglikelihood within SQuAD task", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 1, "deletions": 1}}, {"node": {"oid": "538be6da5d129d8970fa8585f92a4cd9df4eee02", "messageHeadline": "Merge pull request #7 from cfoster0/greedyuntil", "author": {"name": "Charles Foster", "email": "cfoster0@stanford.edu", "user": {"login": "cfoster0"}}, "additions": 2824, "deletions": 533}}, {"node": {"oid": "5be42b4ddfbb1a8010536a8cfdd541b78907a2ea", "messageHeadline": "SQuAD fixed to use loglikelihood API to calculate the probability of \u2026", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 9, "deletions": 6}}, {"node": {"oid": "d1ef37aa0088aa7f0b4e2fdef80a27cbf26ffc26", "messageHeadline": "Merge pull request #163 from jon-tow/logiqa-prompt-fix", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 19, "deletions": 2}}, {"node": {"oid": "e3769082e3d264bf69b931a59bcf3e7e92bcebdf", "messageHeadline": "Include `auxiliary_train` and all `dev` sets in `training_docs`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "364e54d42b1536ed51b551e3e3b76a933075c928", "messageHeadline": "Split `dev` and `val` sets and cleanup prompt spacing", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 8, "deletions": 12}}, {"node": {"oid": "aadf2f41c65028db0b72914710728cd19f4a083b", "messageHeadline": "Fix `LogiQA` prompts", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 19, "deletions": 2}}, {"node": {"oid": "b2bab740383a47f1ab9496fd78a10cd1523acefa", "messageHeadline": "Restore proper prompt formatting", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 16, "deletions": 2}}, {"node": {"oid": "806b022bc23c2acd2c0f2716d830ad321adb2625", "messageHeadline": "Refactor and simple formatting", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 37, "deletions": 75}}, {"node": {"oid": "884c29fb352a9fc213d1cf739482b7ac3e93c7ce", "messageHeadline": "Bring SQuAD fork up to date with EAI upstream", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 2768, "deletions": 521}}, {"node": {"oid": "13b9762678ae7a7b88321098f86fc6bbe2e03967", "messageHeadline": "minor", "author": {"name": "Andy Zou", "email": "andyzou_jiaming@berkeley.edu", "user": {"login": "andyzoujm"}}, "additions": 7, "deletions": 5}}, {"node": {"oid": "7989168ded742e9c57a15129a83aa0f727185f0b", "messageHeadline": "added training", "author": {"name": "Andy Zou", "email": "andyzou_jiaming@berkeley.edu", "user": {"login": "andyzoujm"}}, "additions": 23, "deletions": 17}}, {"node": {"oid": "8888a6beaae97babba1183fa25de89203140adcf", "messageHeadline": "adjusted naming", "author": {"name": "Andy Zou", "email": "andyzou_jiaming@berkeley.edu", "user": {"login": "andyzoujm"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "a6b014741ac1683a4df11254adc36d12e131d6bc", "messageHeadline": "add hendrycksTest", "author": {"name": "Andy Zou", "email": "andyzou_jiaming@berkeley.edu", "user": {"login": "andyzoujm"}}, "additions": 133, "deletions": 0}}, {"node": {"oid": "6d99771014033948641d8d2bd9ab02b68e0acfe4", "messageHeadline": "remove testing code again", "author": {"name": "Eric Tang", "email": "erictang000@gmail.com", "user": {"login": "erictang000"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "1e044b9244131ead255ca27242a616b5149509ea", "messageHeadline": "fixed linter issue", "author": {"name": "Eric Tang", "email": "erictang000@gmail.com", "user": {"login": "erictang000"}}, "additions": 3, "deletions": 19}}, {"node": {"oid": "52c7ec8d61f9bfac6f444ee7c95b9933595ea88d", "messageHeadline": "removed testing comments", "author": {"name": "Eric Tang", "email": "erictang000@gmail.com", "user": {"login": "erictang000"}}, "additions": 0, "deletions": 7}}, {"node": {"oid": "dea1551084d089d7752bef3cbddd3d111766d772", "messageHeadline": "Fixes to process_results string parsing", "author": {"name": "Eric Tang", "email": "erictang000@gmail.com", "user": {"login": "erictang000"}}, "additions": 7, "deletions": 2}}, {"node": {"oid": "bbac75cffa322f42fb275d76eb06c4f33732bb0d", "messageHeadline": "Update minor fixes in math.py", "author": {"name": "Eric Tang", "email": "erictang000@gmail.com", "user": {"login": "erictang000"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "72f91040aff1b8d42f15517adab4566d8eedd173", "messageHeadline": "Added math tasks", "author": {"name": "Eric Tang", "email": "erictang000@gmail.com", "user": {"login": "erictang000"}}, "additions": 379, "deletions": 1}}, {"node": {"oid": "3c7bff64584a5c618ecef5dd8921ff4dbc0c58ea", "messageHeadline": "Update README.md", "author": {"name": "Eric Tang", "email": "46737979+erictang000@users.noreply.github.com", "user": {"login": "erictang000"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "8809c5f15bc53f2543006ea9811ac576f645e30b", "messageHeadline": "Merge pull request #159 from jon-tow/logiqa-evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 72, "deletions": 0}}, {"node": {"oid": "a187cd443809c24e6a706aa5851da3feee30daa7", "messageHeadline": "Implement `LogiQA` data download and evaluation", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 72, "deletions": 0}}, {"node": {"oid": "36485d7aa844fee971204135e4fd31b9a156f323", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 91, "deletions": 84}}, {"node": {"oid": "1de3b743bac8b3d1adcbf05ad709c6914f6eed77", "messageHeadline": "Merge pull request #154 from Muennighoff/master", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 389, "deletions": 0}}, {"node": {"oid": "f1ac64357b4d41f9a4be56df3c5c6c0527156fe5", "messageHeadline": "Minor tweaks", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 4}}, {"node": {"oid": "ab4939d3cb932ee81d12be4fafbbaaf659e2f6fc", "messageHeadline": "Address the `EthicsUtilitarianism` `TODO`s", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 42, "deletions": 53}}, {"node": {"oid": "bf27b2c164e8fbb8a69d8a4b8b638e5eeb1efdfc", "messageHeadline": "Merge branch 'master' into master", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 182, "deletions": 61}}, {"node": {"oid": "94977a627dc9415527bc0bad382506d8b52e1fe2", "messageHeadline": "Merge pull request #1 from EleutherAI/mhf", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 399, "deletions": 0}}, {"node": {"oid": "b720a9cb31bfb1c02884b982a547befbc3752e21", "messageHeadline": "Merge pull request #155 from jon-tow/drop-evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 182, "deletions": 61}}, {"node": {"oid": "e48b7884562983c089af1e252261951b4df3012c", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness \u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 235, "deletions": 16}}, {"node": {"oid": "491283c59b0eb67d34a17c3fe57ae9632c44bf43", "messageHeadline": "Flag down a bunch of todos", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 8, "deletions": 3}}, {"node": {"oid": "90b56db5043b8e55c55b21d36be5332781216b66", "messageHeadline": "Cleanups", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 0, "deletions": 3}}, {"node": {"oid": "001c0a058a8fa2eaef0228f3a45efadfd7829052", "messageHeadline": "Mimic original Ethics Util procedure", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 88, "deletions": 0}}, {"node": {"oid": "d986fd3c987cafc5d490f47ea3e9d152fba80d61", "messageHeadline": "Add proper normalization, checksum, and formatting", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 270, "deletions": 42}}, {"node": {"oid": "879aabd62a15a770aca48fca5d775f06ea5b5706", "messageHeadline": "Implement fewshot description experiment", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 83, "deletions": 0}}, {"node": {"oid": "d6f6ff1d98d31ea279c2c4b467789ef9ba98ecd5", "messageHeadline": "Fix splitting", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "90f1709b909e659c87b1fe849010ab50496b8c24", "messageHeadline": "Reasonable & Unreasonable loglikelihood", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "5b06573eea3a9e72032b4206af85ef9ad543bd06", "messageHeadline": "Swap .index for .split", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "35323047d0d820bb77c15c53e06f2b6ccc31fe80", "messageHeadline": "Make some very minor changes to the prompt", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "2c41ecf6f09b48879bd86ceb7f3c6bb0720f8ce2", "messageHeadline": "Merge pull request #156 from jon-tow/anagrams-evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 110, "deletions": 1}}, {"node": {"oid": "37d4f617fa67bfdfa6ac000546b33b5dbf0f9ba0", "messageHeadline": "Remove custom fewshot examples", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 4}}, {"node": {"oid": "41a94e13a633ba61e0c7f3fa686440570fa1bdb8", "messageHeadline": "Implement all word scrambling and manipulation tasks", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 26, "deletions": 8}}, {"node": {"oid": "a5e135b6e178367d6976ffb4832d6bc188408c8d", "messageHeadline": "Implement `Anagrams` evaluations", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 96, "deletions": 1}}, {"node": {"oid": "ba9c13b2a9a1a2328a16740e4120f6809c5d8489", "messageHeadline": "Fix translation context", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 7}}, {"node": {"oid": "805b541c953ee921e8c79fbf6e17a441e79e59a0", "messageHeadline": "Add acknowledgement to official implementation", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 5, "deletions": 0}}, {"node": {"oid": "5894761aec5fe07b05ecb06928ac7d9142a7d1fb", "messageHeadline": "Remove print statements", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "41a8b58260661c72555005e802b4d011f6577357", "messageHeadline": "Undo formatting", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 5, "deletions": 7}}, {"node": {"oid": "e4d852a0f7cf739aafdfd2082ca096f2580cbf05", "messageHeadline": "Clean up", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 23, "deletions": 77}}, {"node": {"oid": "f4f7618a984346f74ab73c8c5c71e94f52f41fac", "messageHeadline": "Fixes", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "93ebce435c5b0207d7457e7a73fd1e13202d7236", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-\u2026", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 30, "deletions": 11}}, {"node": {"oid": "03f3446397fdc3b188fa32385787888caa81926f", "messageHeadline": "Fix gpt3 batching bug", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1ff4e07f32ec5c26765a8424100f5d9b6cd53158", "messageHeadline": "Add gpt3 chunking", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 30, "deletions": 11}}, {"node": {"oid": "fc7cd6304136f84361f337ef94d7858d793d0ab0", "messageHeadline": "Implement `DROP` evaluation", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 196, "deletions": 65}}, {"node": {"oid": "f3bf1c07083fcefb0d45fca81bd7757d884ec96b", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "77b444708db536af72c17c1b8059f361438950f7", "messageHeadline": "Add gpt2/3 tokenizer sanity check", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 11, "deletions": 1}}, {"node": {"oid": "86c4115dc1f1fd5fb802840d09203cd801fa3aca", "messageHeadline": "Adapt prompts to Ethics paper", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 7, "deletions": 4}}, {"node": {"oid": "a7564ff37604ced998bdb96111816fd35fc8f815", "messageHeadline": "Add Exact Match Metric", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 117, "deletions": 14}}, {"node": {"oid": "a64df04b3a798f4a4d2e9d456484d5e603044d31", "messageHeadline": "Update main.py", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "fef52be4017c56357857a91f3a3b2c4a56f1e07b", "messageHeadline": "Add Ethics", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 5, "deletions": 0}}, {"node": {"oid": "e3bdca800e89a13570f62e302a02209a32402044", "messageHeadline": "Correct Scoring", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "7c73f3c832a93593cccf2b258e2d90928d1895eb", "messageHeadline": "Correct Path", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "400f812c9900fa7467274ff6a410b3819cb92118", "messageHeadline": "Add Ethics Tasks", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 3, "deletions": 1}}, {"node": {"oid": "d4606d27b71402504b8de7f6ba3d851e6930ada5", "messageHeadline": "Add Virtue", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 28, "deletions": 5}}, {"node": {"oid": "b5c2952aa9abce680eee6dcaac8fc2da3b1b0662", "messageHeadline": "Add Utilitarianism & Justice", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 8, "deletions": 9}}, {"node": {"oid": "c9aad57fd4559d15813aef06c4c1f102ead92d63", "messageHeadline": "Label fixes", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 47, "deletions": 2}}, {"node": {"oid": "f123ce39441914cae996ed7ae29c04096014949e", "messageHeadline": "Syntax", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 11, "deletions": 11}}, {"node": {"oid": "5c92d62930a2a84106a1dc54e01b29639b93cfaf", "messageHeadline": "Add Ethics CM & Deontology", "author": {"name": "Muennighoff", "email": "62820084+Muennighoff@users.noreply.github.com", "user": {"login": "Muennighoff"}}, "additions": 130, "deletions": 0}}, {"node": {"oid": "7d5aa3f73027e97e09a9369ddfd555c7ba234585", "messageHeadline": "Typo", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "624c038e6bda1c4bd00fe57c00b6627165ddac8b", "messageHeadline": "Shuffle order of docs in evaluator", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 9, "deletions": 2}}, {"node": {"oid": "ea08242e92a3b53afa2208b3a9864121b6513d2c", "messageHeadline": "Add a bunch of scripts", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 88, "deletions": 0}}, {"node": {"oid": "7bd37779aec0896898ce821d65d1a5961a248933", "messageHeadline": "Update README.md", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 80, "deletions": 39}}, {"node": {"oid": "c155698f64563a46cdcd840850bc86415e2f8437", "messageHeadline": "Merge pull request #133 from EleutherAI/fazz/refactor-task-coqa", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 111, "deletions": 43}}, {"node": {"oid": "431e53debcb8b38249801bd0b76eea228cdd9753", "messageHeadline": "Fix CoQA for tests", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "8966289a20284be0d5adbf38beee918cf2d3151e", "messageHeadline": "Add GPT2 greedy_until truncation", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "dbdba695b2b01a0bb9d1e2b557535077b4f159b0", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness \u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 57, "deletions": 38}}, {"node": {"oid": "847747709154fbd087be4280ce8065b523327252", "messageHeadline": "Merge branch 'fazz/refactor-task-coqa' of github.com:EleutherAI/lm_ev\u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 108, "deletions": 42}}, {"node": {"oid": "1e5194a2483d882138fb175a804adbccf1f584f7", "messageHeadline": "Merge pull request #153 from EleutherAI/translation-v2", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 57, "deletions": 38}}, {"node": {"oid": "d0a301cca900b628d1e269edfd8b3696481b21bc", "messageHeadline": "fix metric iterable checks", "author": {"name": "&", "email": "", "user": null}, "additions": 9, "deletions": 4}}, {"node": {"oid": "b1619834becb055a3a72e6bfdefcadd22f7fbba0", "messageHeadline": "move task list to init file", "author": {"name": "&", "email": "", "user": null}, "additions": 39, "deletions": 34}}, {"node": {"oid": "b5bb7f6c093ee1d808205958ae8615f79a1c9675", "messageHeadline": "look at available pairs languages", "author": {"name": "&", "email": "", "user": null}, "additions": 9, "deletions": 0}}, {"node": {"oid": "758b9e3c0f024284c2880bfc51cdc2b7e38216b2", "messageHeadline": "Merge pull request #152 from zphang/record_fix", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 32, "deletions": 71}}, {"node": {"oid": "81ab341658638b5874dabf542d831ae3e79c262f", "messageHeadline": "update", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "2cfdd80acb223b6e5304f4105133e4c7b51c259d", "messageHeadline": "ReCoRD fixup", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 17, "deletions": 17}}, {"node": {"oid": "41eb4a6504999fd96ea977564a0eda3a7103c295", "messageHeadline": "Merge pull request #150 from zphang/record_update", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 57, "deletions": 22}}, {"node": {"oid": "2719a522fba83a5ac5d6d1b93b1fc954f1f95307", "messageHeadline": "tweak doc format", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 61, "deletions": 22}}, {"node": {"oid": "6a5346004cdf594e2aad2c4b76ab382ba6929b8b", "messageHeadline": "Merge remote-tracking branch 'origin/fazz/refactor-task-coqa' into fa\u2026", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "fee14a6def9c7f0c7f9b0f31ea4a40592ef499ac", "messageHeadline": "Merge remote-tracking branch 'origin/master' into fazz/refactor-task-\u2026", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 514, "deletions": 2}}, {"node": {"oid": "68122ef4afeb571ff406c3245f593c3dc21bdb2b", "messageHeadline": "Call greedy_until continuation, download fix", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 15, "deletions": 23}}, {"node": {"oid": "232c9ab6b9815c4490ac3e9511bba75d89190cdd", "messageHeadline": "Fixes SQuAD v2 metric computation.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 9, "deletions": 4}}, {"node": {"oid": "e4e9228ec052145312cb072d20222ebeac69384f", "messageHeadline": "ReCoRD fix", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 7, "deletions": 11}}, {"node": {"oid": "487b53134a15dc0a4f3335df4e2c76d8b8e81118", "messageHeadline": "Merge pull request #145 from EleutherAI/decontamination", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 514, "deletions": 2}}, {"node": {"oid": "4d90dc8ae0e8aa1cf9eafbfcdea0d908b89ecb6c", "messageHeadline": "Merge branch 'master' into decontamination", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 804, "deletions": 160}}, {"node": {"oid": "37c3139d71dd2cd13ab3cd3ebddce1315d53bd79", "messageHeadline": "Merge remote-tracking branch 'origin/master' into fazz/refactor-task-\u2026", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 1253, "deletions": 489}}, {"node": {"oid": "7ad6bf456a8542d9cd6aa2a1eaf4c97acf7f88a7", "messageHeadline": "Merge pull request #146 from EleutherAI/translation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 672, "deletions": 77}}, {"node": {"oid": "ac47d4813c05edfde0d147710a407d4852a00e69", "messageHeadline": "Merge branch 'master' into translation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 103, "deletions": 1}}, {"node": {"oid": "0601c909d8f8cc7dec7fb58715f994668060ab5b", "messageHeadline": "Merge pull request #147 from jon-tow/headqa-refactor", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 28, "deletions": 44}}, {"node": {"oid": "971742662061d7557c48b7dc453e876e131a868b", "messageHeadline": "Refactor `HeadQA` as a `MultipleChoiceTask`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 28, "deletions": 44}}, {"node": {"oid": "d194d65cb4c99fc251b75dadf0fb4ed797053a89", "messageHeadline": "Merge pull request #143 from EleutherAI/add_headqa_mathqa", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 119, "deletions": 1}}, {"node": {"oid": "cc27e38ae3171a39ee36af45b625a94db8ab43da", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness \u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 33, "deletions": 84}}, {"node": {"oid": "404530d06650a3f871b8a7af32886d904c3c6deb", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness \u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "e5a9de80ad1a6606e24bfa547e051b56e95a8e1c", "messageHeadline": "Update lambada data location", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b07ecab4e4060fe3ed49b9d503087bddde14a8fa", "messageHeadline": "small change to regex", "author": {"name": "Anthony DiPofi", "email": "anthonydipofi@gmail.com", "user": {"login": "anthony-dipofi"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "81e429323a9e567228cf1f5e71fbf0bc58062dce", "messageHeadline": "change mathqa to use numeric answer strings instead of a,b,c,d,e as c\u2026", "author": {"name": "Anthony DiPofi", "email": "anthonydipofi@gmail.com", "user": {"login": "anthony-dipofi"}}, "additions": 7, "deletions": 4}}, {"node": {"oid": "97856354bef1b6e6053df6f6b265fbeddbdd8157", "messageHeadline": "Merge branch 'master' into translation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 31, "deletions": 81}}, {"node": {"oid": "e56381f4fb9a6ee64c9fc9fd504b86761a726833", "messageHeadline": "...", "author": {"name": "&", "email": "", "user": null}, "additions": 2, "deletions": 0}}, {"node": {"oid": "1eb91d3657fff606083a1d2d2b6da515f2cf3598", "messageHeadline": "adds missing task error", "author": {"name": "&", "email": "", "user": null}, "additions": 8, "deletions": 1}}, {"node": {"oid": "61aa411b8ba9416fdbc32618088ee85a3624777a", "messageHeadline": "Fixed metrics. Translation works", "author": {"name": "&", "email": "", "user": null}, "additions": 35, "deletions": 15}}, {"node": {"oid": "1297e3423b87d470279a721b484f3f0806078974", "messageHeadline": "add tasks to registry", "author": {"name": "&", "email": "", "user": null}, "additions": 45, "deletions": 18}}, {"node": {"oid": "12ba8426485ce2f766086ff1dd969baaa5854f41", "messageHeadline": "metrics", "author": {"name": "&", "email": "", "user": null}, "additions": 31, "deletions": 5}}, {"node": {"oid": "c6c6727298f03b716378d4e6c89be9ddacc0c293", "messageHeadline": "refactor MathQA as MultipleChoiceTask, uses full words for Q. and A.", "author": {"name": "Anthony DiPofi", "email": "anthonydipofi@gmail.com", "user": {"login": "anthony-dipofi"}}, "additions": 28, "deletions": 53}}, {"node": {"oid": "bd3fcce0e2e21fe7a33a33146d3e482111284a90", "messageHeadline": "translation task draft 1. Not hooked up, no metrics", "author": {"name": "&", "email": "", "user": null}, "additions": 441, "deletions": 1}}, {"node": {"oid": "1fb90b91b602b29572af7bdeb16c38e9e03852ed", "messageHeadline": "metrics file", "author": {"name": "&", "email": "", "user": null}, "additions": 147, "deletions": 75}}, {"node": {"oid": "e26dc4d3ef820179cdf0f82e888e01295dac6183", "messageHeadline": "Merge pull request #144 from jon-tow/arc-refactor", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 32, "deletions": 83}}, {"node": {"oid": "24ac76df4ed80cbc7996b40b50b95cda8804049f", "messageHeadline": "Refactor `ARC` as a `MultipleChoiceTask`", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 32, "deletions": 83}}, {"node": {"oid": "742b5df295cc06e2b1e5c3a79fab575f842d261c", "messageHeadline": "add headqa and mathqa datasets", "author": {"name": "Anthony DiPofi", "email": "anthonydipofi@gmail.com", "user": {"login": "anthony-dipofi"}}, "additions": 141, "deletions": 1}}, {"node": {"oid": "4de29dcaa54bbe93e37cd08a6d92346e06a7b788", "messageHeadline": "Moving over training data decontamination code", "author": {"name": "&", "email": "", "user": null}, "additions": 514, "deletions": 0}}, {"node": {"oid": "4a64031cedce06de737bd75bd0a6e45464aee3ab", "messageHeadline": "Undo change to test", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "f79927898ef49d2f8c77196a13c085bd2422dab2", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 30, "deletions": 91}}, {"node": {"oid": "432bd44c5389ed42a3ecada5969713b018440e43", "messageHeadline": "Fixes to make greedy_until work", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 15, "deletions": 5}}, {"node": {"oid": "f63bc6586511e1050558852205c33df57120344d", "messageHeadline": "Merge pull request #142 from jon-tow/hellaswag-refactor", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 30, "deletions": 91}}, {"node": {"oid": "7b649ded6e0106415c1af692790b3045a3c6673a", "messageHeadline": "Fixes to make greedy_until work", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 44, "deletions": 8}}, {"node": {"oid": "e8f9dc71e9e807bbaf041de8857d97e22c5906fc", "messageHeadline": "Implement GPT2 greedy_until", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 40, "deletions": 6}}, {"node": {"oid": "a2b108b90ed3b328a0330fce21788393917bc82c", "messageHeadline": "Remove test set from `HellaSwag`", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 5}}, {"node": {"oid": "43ffe35156de4e03672022165d6c0f5986da9be7", "messageHeadline": "Refactor `HellaSwag` as a `MultipleChoiceTask`", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 31, "deletions": 88}}, {"node": {"oid": "9adf18b17a930de73589b7d8b36108d40059a863", "messageHeadline": "Merge pull request #141 from jon-tow/winograd-fixes2", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 22, "deletions": 22}}, {"node": {"oid": "155aeee93256a1109529c45bdec504dba36cf3d3", "messageHeadline": "Rename list of loglikelihoods for consistency", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "43e274f8693cf04f7f62bf768a5c605c8d388824", "messageHeadline": "Fix context orderings to match label positioning", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 22, "deletions": 22}}, {"node": {"oid": "eb4c8407af4a66649e62f72a3357c3075253a714", "messageHeadline": "Removed unnecessary import.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 0, "deletions": 2}}, {"node": {"oid": "bba6e0e9711e74f6ea17800879007e104e303d76", "messageHeadline": "Passes tests, except for NotImplementedError for request type greedy_\u2026", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 8, "deletions": 12}}, {"node": {"oid": "c6b094e0395d7499639ef397e5bfad903ac8ac01", "messageHeadline": "Merge pull request #138 from jon-tow/winograd-fixes", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 70, "deletions": 59}}, {"node": {"oid": "1a159d6bb33c789a5a924b266e3dcde5d6e7eb17", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness \u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 148, "deletions": 42}}, {"node": {"oid": "8b038c2a663ca2e8352b8d1887c02f6cc94d67e8", "messageHeadline": "Update spacing", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 4}}, {"node": {"oid": "3a90e24613faa29a61da45483953ca9c4d3eec42", "messageHeadline": "Remove incorrect context from `fewshot_context`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 27, "deletions": 47}}, {"node": {"oid": "10faacda74a0121781b24c1b73bbe9903b0dd752", "messageHeadline": "Added SQuADv2 to __init__.py. Not yet tested.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f48b119d33c81755619beed45aca218e03387811", "messageHeadline": "Skeleton of SQuADv2. Not yet tested.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 55, "deletions": 18}}, {"node": {"oid": "7614a8f35de03e4500181b897f32ffb8508e5b60", "messageHeadline": "Merge pull request #139 from jeffhsu3/pubmedqa", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 104, "deletions": 5}}, {"node": {"oid": "5960874bc89ef862d68ed42ff8068b5b082c6d99", "messageHeadline": "Update qa4mre.py", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 3}}, {"node": {"oid": "e9e5d0a2a84f887823cf41fa4ba8ebb802c803cb", "messageHeadline": "Add missing space in MultiRC", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "42df113ced8fb10febc9288d10379bad8e43affd", "messageHeadline": "Fix arithmetic", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "3d432b1a265f476fe467f38377ad5f346bb31dec", "messageHeadline": "Merge pull request #4 from EleutherAI/master", "author": {"name": "Charles Foster", "email": "cfoster0@stanford.edu", "user": {"login": "cfoster0"}}, "additions": 3123, "deletions": 1435}}, {"node": {"oid": "84f14dafeb26915b2c86c1ddf69dcdba991fbc47", "messageHeadline": "Cache by default", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "aab91285a5c8885660c080f1d4b344a2d752e656", "messageHeadline": "Update gpt2 for efficiency and allow specifying model size", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 29, "deletions": 28}}, {"node": {"oid": "a264f4492fc4486f19908f40686d9bd4f6fa80ed", "messageHeadline": "removed truncation of source", "author": {"name": "jeffhsu3", "email": "jeffhsu3@gmail.com", "user": {"login": "jeffhsu3"}}, "additions": 2, "deletions": 11}}, {"node": {"oid": "74f3307bb02b07e17364fe6e20075154fe7c4459", "messageHeadline": "Fix few-shot contexts for `Winograd Schema Challenge`-based tasks", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 92, "deletions": 61}}, {"node": {"oid": "4d8ed7d5e9ac0d141689881f3d9d83b74abeb29e", "messageHeadline": "Fix PiQA", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "627148553b9aa1f86d1952c08d499b5f6fa40a08", "messageHeadline": "Update RACE format", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 6, "deletions": 3}}, {"node": {"oid": "18e08d293ee03c5e7acc1378c886e0ca0c05577d", "messageHeadline": "space in pubmedqa for merge", "author": {"name": "jeffhsu3", "email": "jeffhsu3@gmail.com", "user": {"login": "jeffhsu3"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "c05b994068b4125c802771a813ebd3097c4d7d68", "messageHeadline": "Added qa4mre and changed to best_download for sciq", "author": {"name": "jeffhsu3", "email": "jeffhsu3@gmail.com", "user": {"login": "jeffhsu3"}}, "additions": 109, "deletions": 77}}, {"node": {"oid": "9b14235d00e8b73048f3e8f5dae46d9855615add", "messageHeadline": "More task changes", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 16, "deletions": 15}}, {"node": {"oid": "96157fc7237a5a20d27a61f65ee619d775e684cf", "messageHeadline": "Fix", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 14, "deletions": 13}}, {"node": {"oid": "c3f724cf5511f25c17e09e92317ba3ceb5389303", "messageHeadline": "Change glue and superglue prompts", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 41, "deletions": 31}}, {"node": {"oid": "1050109b414cec65f4d78e3db23f2b57fd64ec5a", "messageHeadline": "Merge pull request #136 from jon-tow/openbookqa-evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 25, "deletions": 75}}, {"node": {"oid": "aa1d72933bc723b620b66200f7e3be6c8790631d", "messageHeadline": "Implement `OpenBookQA` evaluation", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 25, "deletions": 75}}, {"node": {"oid": "359114fd23df8ec1ad35ec43e5580b01e9c2318a", "messageHeadline": "LM: handle empty context", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 17, "deletions": 3}}, {"node": {"oid": "5d56a47da7bae09fccca48d5bb4cbad58c7c73c5", "messageHeadline": "Add table making", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 17, "deletions": 1}}, {"node": {"oid": "dcd44d226ad8c8e68d936e063a7c2a07df4bfae1", "messageHeadline": "LAMBADA metric consistency change", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "442f843cf2413c2cc7d386eb55970f1eda02d34b", "messageHeadline": "Add tests for {val,test}_docs determinism", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 27, "deletions": 0}}, {"node": {"oid": "77d4b087985eda2248097b7b3dfbd7ac9b7f32b1", "messageHeadline": "Fix caching", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 9, "deletions": 10}}, {"node": {"oid": "f9750990c4f2ad03496a71f5091338b1f92e9407", "messageHeadline": "Remove stray space", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "79c9b68ae50b8d746de76c3c0ed9005bb4252582", "messageHeadline": "Merge remote-tracking branch 'origin/master' into fazz/refactor-task-\u2026", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 159, "deletions": 45}}, {"node": {"oid": "b9b3159b82ac56443138ed385678eb83a8278083", "messageHeadline": "Bugfixes, answer mapping, comments", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 59, "deletions": 31}}, {"node": {"oid": "1b467c5770c640ab03a47f38e2f759739d1261e0", "messageHeadline": "Merge pull request #112 from jeffhsu3/pubmedqa", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 144, "deletions": 1}}, {"node": {"oid": "79878d135a19846868c182aa986b6ec740e7c884", "messageHeadline": "Remove merge conflict", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 71}}, {"node": {"oid": "5dfbba05cf00c037c7af750aae10cf1fd44dd547", "messageHeadline": "Merge pull request #130 from jon-tow/piqa-prompt-fix", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 15, "deletions": 44}}, {"node": {"oid": "602d3e204130e06ff3b77c1618d3d461803792c9", "messageHeadline": "Merge remote-tracking branch 'origin/master' into fazz/refactor-task-\u2026", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 284, "deletions": 80}}, {"node": {"oid": "222027b936275fa7f568a3b2d6bcb728d02b3287", "messageHeadline": "HFTask -> Task, line break correction", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "c8e77519985eee6e311a7bf4c43d6ccd708fc7d6", "messageHeadline": "Fix `PIQA` prompt", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 15, "deletions": 44}}, {"node": {"oid": "ab41f0899c1b276a8be593db343aca5a3721561f", "messageHeadline": "merge", "author": {"name": "jeffhsu3", "email": "jeffhsu3@gmail.com", "user": {"login": "jeffhsu3"}}, "additions": 72, "deletions": 1}}, {"node": {"oid": "695d633d7424342971eaaab1715028ab8c0d9e61", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-\u2026", "author": {"name": "jeffhsu3", "email": "jeffhsu3@gmail.com", "user": {"login": "jeffhsu3"}}, "additions": 4, "deletions": 5}}, {"node": {"oid": "0971cffe83fc91ebb52812561efebab6411e252c", "messageHeadline": "separated sciq and pubmedqa to different files", "author": {"name": "jeffhsu3", "email": "jeffhsu3@gmail.com", "user": {"login": "jeffhsu3"}}, "additions": 81, "deletions": 77}}, {"node": {"oid": "016306571ef156ae3b44bada7904e257480457f3", "messageHeadline": "Merge pull request #128 from jon-tow/superglue-wsc-fix", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 5}}, {"node": {"oid": "f5f609340ece0cb53b97abdfebc52779465acdee", "messageHeadline": "Merge pull request #129 from jon-tow/write-out-fix", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "990d5e31711fad8dedb4a4d24c51ff9e5e38c466", "messageHeadline": "Fix `write_out.py` training set bug", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "715dc759cd4e1fe1d2b326b3a3f32403a7260dd9", "messageHeadline": "Fix `doc_to_text` passage bug", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 4, "deletions": 5}}, {"node": {"oid": "30c2fe23657981d8a6155da1bd8e8098487b771a", "messageHeadline": "Remove extra space", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "620c0d26160eecc2908a142b985d98e1f952bbac", "messageHeadline": "sciq", "author": {"name": "jeffhsu3", "email": "jeffhsu3@gmail.com", "user": {"login": "jeffhsu3"}}, "additions": 12, "deletions": 8}}, {"node": {"oid": "9b933d96d1842cb36953cdc88ada6256ce70ec1e", "messageHeadline": "merged changes", "author": {"name": "jeffhsu3", "email": "jeffhsu3@gmail.com", "user": {"login": "jeffhsu3"}}, "additions": 747, "deletions": 584}}, {"node": {"oid": "c71dcb9107a5e908d5392818ad4fbc4f7d57792b", "messageHeadline": "Added SciQ", "author": {"name": "jeffhsu3", "email": "jeffhsu3@gmail.com", "user": {"login": "jeffhsu3"}}, "additions": 82, "deletions": 13}}, {"node": {"oid": "c0fbf9e8b0cba0fba09d9916e2e5f71936dd04c6", "messageHeadline": "Add task table", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 70, "deletions": 5}}, {"node": {"oid": "049dfa344a4647fff6abedf8cb34f44f10859a9c", "messageHeadline": "Add test to make sure right space conventions are used", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 31, "deletions": 10}}, {"node": {"oid": "aea631629310a8ec5078f3dac8a2cd7e07784a9b", "messageHeadline": "Fix SAT", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "d7c08d5b024db6eae8573f897a17de45fa078568", "messageHeadline": "Convert SAT to use MultipleChoiceTask", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 34}}, {"node": {"oid": "2f5f42c62a1d66b8c55960a5cc6e253ae2d4425c", "messageHeadline": "Add MultipleChoiceTest target", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "7422d5937147b7a404595033a7ca03477fa4cbdf", "messageHeadline": "Add sqlitedict dependency", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "706cb53aeab1f0d59035e4135f20a1ed2de785d3", "messageHeadline": "Add MultipleChoiceTask", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 29, "deletions": 0}}, {"node": {"oid": "2a1d7d872a09205c63a4d2b4be33d0a5d9b8766b", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 50, "deletions": 15}}, {"node": {"oid": "b1f7284e06fb8024ea3e130d8eee32de14d94164", "messageHeadline": "Add retry with backoff for GPT3", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 15, "deletions": 2}}, {"node": {"oid": "a55a5c52eeabacbedb0848da9bb45c28741ad90b", "messageHeadline": "Update arithmetic.py", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "612c69c345ec8e31697704344a76a445428f58e5", "messageHeadline": "Merge pull request #124 from jon-tow/arc-evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 47, "deletions": 12}}, {"node": {"oid": "c55e823700dc881ee90f1d4dcce35574b2bffeac", "messageHeadline": "Get rid of annoying logging", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 9, "deletions": 2}}, {"node": {"oid": "d5cd9655c8ce48a5d7e04f879de76cce59a0888d", "messageHeadline": "Implement caching", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 64, "deletions": 2}}, {"node": {"oid": "a21df3552e2b1db6fd0e0ea6efd5d8205f89d1df", "messageHeadline": "Merge remote-tracking branch 'origin/master' into fazz/refactor-task-\u2026", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 471, "deletions": 503}}, {"node": {"oid": "d72ad5e249a3b9d5b774167e5b4a0bd3ed0b12e2", "messageHeadline": "Implement `ARC` evaluation", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 47, "deletions": 12}}, {"node": {"oid": "1815286cf81254effa8c0f2f533c4e3dd91b43c6", "messageHeadline": "Add missing import", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "5f4c7c50d14d63553071b05c48b30c155eb0837d", "messageHeadline": "Implement gpt3 greedy_until", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 21, "deletions": 5}}, {"node": {"oid": "38e8858f4fe130eb755c1d7c7d7cf5441049074b", "messageHeadline": "Get rid of utils_stream", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 13, "deletions": 29}}, {"node": {"oid": "88ec855df73a675aec47f84c4d397174c15a703f", "messageHeadline": "Deprecate utils_stream and remove unused code", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 135}}, {"node": {"oid": "778e0f91bb41cfbfad4bd75ae09ebd5d9330b6b7", "messageHeadline": "Massive refactor", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 110, "deletions": 85}}, {"node": {"oid": "52c1c56a82fcc18ce839b620f16c710e2ea810e5", "messageHeadline": "Implement gpt3 logprobs", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 62, "deletions": 21}}, {"node": {"oid": "b57d059ace1c99c4bd6369276a45988c5c4b4380", "messageHeadline": "Fix lambada", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 10, "deletions": 4}}, {"node": {"oid": "8eada53b051a62e867a8e6226a1c2d57a62a9530", "messageHeadline": "Add tests for GPT2", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 13, "deletions": 0}}, {"node": {"oid": "a75a9481fdfdb577c7cfa41272340dd447f13824", "messageHeadline": "Merge pull request #123 from jon-tow/winogrande-evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 44, "deletions": 36}}, {"node": {"oid": "485e45b8c640ec0d03fbef022f18bfaf49907168", "messageHeadline": "Implement `Winogrande` evaluation", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 44, "deletions": 36}}, {"node": {"oid": "a737c80269e6494c648e772f2cab3ddbb3e59c11", "messageHeadline": "Remove .download mocking", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "19b0f529326e9599909f477fdb26c5a2593ee0ea", "messageHeadline": "Merge pull request #111 from jon-tow/wsc273-evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 134, "deletions": 127}}, {"node": {"oid": "bc5495d2e5e3be284992e66dc92626af1d6278da", "messageHeadline": "Merge branch 'master' into wsc273-evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 38, "deletions": 42}}, {"node": {"oid": "e12d007849c87c800c58b3bee8da2edcd7884378", "messageHeadline": "Merge pull request #109 from EleutherAI/webqs", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 38, "deletions": 42}}, {"node": {"oid": "8444c486689f9cb367b9a8890db7b31b544ca6de", "messageHeadline": "Add todo comment", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "b5e86d3f511c1bee7bb2a51a737aaf5ed9d73f42", "messageHeadline": "Merge branch 'master' into wsc273-evaluation", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 177, "deletions": 172}}, {"node": {"oid": "269d368367f642e2dac8d0c24aa4b1d9b66b74b8", "messageHeadline": "Merge branch 'master' into webqs", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 206, "deletions": 63}}, {"node": {"oid": "a1a4a32eadcd9cd2027335f31867091d09dc08ae", "messageHeadline": "Merge pull request #119 from jon-tow/task-refactor", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 46, "deletions": 41}}, {"node": {"oid": "5cfb730829082cbb9709d62472dc6068618cdb63", "messageHeadline": "Fix naming convention to avoid `pytest` name mangling invocation", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 23, "deletions": 23}}, {"node": {"oid": "a60ef6fa774306b589488f8d2da9bf43e9c4af6c", "messageHeadline": "Refactor `Dataset` naming and `HFTask` properties", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 62, "deletions": 57}}, {"node": {"oid": "efa810f02cf5fda5774f31d644da2786d5524a66", "messageHeadline": "Score computation, use squad metrics", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 38, "deletions": 21}}, {"node": {"oid": "1b35f6b9a34fd3991c5fbad9ce06c9c75ac65516", "messageHeadline": "pubmedqa", "author": {"name": "jeffhsu3", "email": "jeffhsu3@gmail.com", "user": {"login": "jeffhsu3"}}, "additions": 87, "deletions": 8}}, {"node": {"oid": "c32a13e8deeb08accd2c28a2fd2d9478ec622a7d", "messageHeadline": "Implement `WSC273` evaluation and data processing", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 74, "deletions": 67}}, {"node": {"oid": "5552c8dc4ccda972f1a63b6bc858d0d7e1e1de5e", "messageHeadline": "Merge remote-tracking branch 'origin/master' into fazz/refactor-task-\u2026", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 161, "deletions": 23}}, {"node": {"oid": "826d90e2777ce97e80fa946a10d908a5bdfc483a", "messageHeadline": "Add badges", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "5b6182d5520aa450e54321131390a7d5e27cdbec", "messageHeadline": "Fix coverage again again again again again", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "2d6102c478b40535eaff8e06ef9d86720d221dad", "messageHeadline": "Fix coverage again again again again", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 6, "deletions": 2}}, {"node": {"oid": "2708ae4c8006a519535858820b3e68ee2ba93ecd", "messageHeadline": "Fix coverage again again again", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 23, "deletions": 1}}, {"node": {"oid": "a3a2152c3be59cee2ed52c56c437ad148d3bfe45", "messageHeadline": "Add unit tests", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 51, "deletions": 0}}, {"node": {"oid": "8bd843c01df84158f345b6ba3a568cebd5394ab3", "messageHeadline": "Fix coverage again again", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "b7da653c118321d054d3027ce3fae37422e136ec", "messageHeadline": "Fix coverage again", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1033bab6683fd70886e6785d7287a0c3eab0ee65", "messageHeadline": "Fix pytest path", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "1206753861c86a35b542b8bebc6134a277b4d3c0", "messageHeadline": "Upload to codecov", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 2}}, {"node": {"oid": "31863cde6211efdf9740950242f6a56f1e9f67d2", "messageHeadline": "Make linter happy", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 0}}, {"node": {"oid": "fe4a1efdee679e8ec72c1dcc30a90caf8ed4b2ed", "messageHeadline": "Fix linting problems", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 25, "deletions": 21}}, {"node": {"oid": "7ab3f8ca3a7371d9242e8aa05af0673758e4ef19", "messageHeadline": "Merge pull request #110 from EleutherAI/leogao2-patch-2", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 45, "deletions": 0}}, {"node": {"oid": "8dd1983b2a214957d31dd057b1e0709e34e2e265", "messageHeadline": "Add CI pipeline", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 45, "deletions": 0}}, {"node": {"oid": "c0862026659376301bef93576ad66d689abc89f1", "messageHeadline": "Text & target impl, support fns, refactoring", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 48, "deletions": 30}}, {"node": {"oid": "34eb121f8b071805e3062d572f142129085d98f8", "messageHeadline": "add webqs evaluation and fallback to test set when validation is unav\u2026", "author": {"name": "Anthony DiPofi", "email": "anthonydipofi@gmail.com", "user": {"login": "anthony-dipofi"}}, "additions": 37, "deletions": 42}}, {"node": {"oid": "6738b24177653ba8caf5c70cebbba30ab36b3c8b", "messageHeadline": "Merge branch 'master' into fazz/refactor-task-coqa", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 27, "deletions": 19}}, {"node": {"oid": "47384df731867abebd826ce94546dedecf583757", "messageHeadline": "Add CoQa to task registry", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "12880f1c8060bcdffd617e415a9f9cb0819b42db", "messageHeadline": "Initial skeleton refactoring", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 26, "deletions": 16}}, {"node": {"oid": "6598967b289637a9074f5fec44b40b1d8ed8c2cb", "messageHeadline": "Merge pull request #106 from jon-tow/anli-evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 24, "deletions": 13}}, {"node": {"oid": "ec45d0aa476532a06fe0cfb43403e6cf1f447e05", "messageHeadline": "Merge pull request #108 from EleutherAI/leogao2-patch-2", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 3, "deletions": 6}}, {"node": {"oid": "66558b359e383933dbf25afe0ed5e30b809e608f", "messageHeadline": "Merge pull request #107 from EleutherAI/triviaqa_evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 31, "deletions": 40}}, {"node": {"oid": "c8032a1a979e5abf39b91196a039f0ffccee50e8", "messageHeadline": "Merge branch 'master' into triviaqa_evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 46, "deletions": 17}}, {"node": {"oid": "b2b5a1227ef3604adb0ee39d05db7c5545fdaa46", "messageHeadline": "Handle aliases correctly in TriviaQA", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 19, "deletions": 5}}, {"node": {"oid": "0f30237a22cdc234bfc4a89984cd35381587029c", "messageHeadline": "Merge pull request #104 from jon-tow/race-evaluation", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 43, "deletions": 16}}, {"node": {"oid": "1bf97c9eaeddb3b67407e4115a1d0e5a447b6937", "messageHeadline": "Make *_docs not abstract", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 6}}, {"node": {"oid": "0601d0bbe25034dbe9716d393b5dd0078d073550", "messageHeadline": "add evaluation for TriviaQA dataset based on loglikelihood", "author": {"name": "Anthony DiPofi", "email": "anthonydipofi@gmail.com", "user": {"login": "anthony-dipofi"}}, "additions": 16, "deletions": 41}}, {"node": {"oid": "e48b60828ac3a6e4bb96541d7925e0a451dac90a", "messageHeadline": "Merge pull request #105 from jon-tow/dependency-updates", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "0a53c06e20dadae7655e9703a26cf67233525dd8", "messageHeadline": "Implement `ANLI` evaluations", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 24, "deletions": 13}}, {"node": {"oid": "483583523a56fc701ec97b54f0342348e3093e8b", "messageHeadline": "Merge branch 'master' into race-evaluation", "author": {"name": "Jonathan Tow", "email": "41410219+jon-tow@users.noreply.github.com", "user": {"login": "jon-tow"}}, "additions": 84, "deletions": 84}}, {"node": {"oid": "33ab4a29136a9681b1d3d11cf064706c6ea78130", "messageHeadline": "Fix versioning", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "f37dcab646d2d5447e947e82166b51869cbf7cd3", "messageHeadline": "Update dependencies", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "6e1a76f09fe4461f0fb61c801bd1a35de041d5a3", "messageHeadline": "Implement `RACE` evaluation", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 42, "deletions": 16}}, {"node": {"oid": "0f5368082bf196bc814e235f829f096244333a39", "messageHeadline": "Merge pull request #99 from jon-tow/hellaswag-evaluation", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 48, "deletions": 27}}, {"node": {"oid": "2d61b3ce4be2ba2a7fbc14261d936b515dc1e5b0", "messageHeadline": "Merge pull request #103 from EleutherAI/piqa", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 35, "deletions": 57}}, {"node": {"oid": "63854c10d679b0320604aec9490d3e914827a2c6", "messageHeadline": "Implement PiQA", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 35, "deletions": 57}}, {"node": {"oid": "97b88570fbe4a3ee57e8033462fa9972ac517ec0", "messageHeadline": "Remove brackets from documents", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 18, "deletions": 7}}, {"node": {"oid": "a2f5b74b383a04bab86ee6484b9b5124125c54e4", "messageHeadline": "Merge pull request #101 from EleutherAI/lambada", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 36, "deletions": 53}}, {"node": {"oid": "822fcc6fb8b9196827a188a5225987537e5b046b", "messageHeadline": "Implement LAMBADA", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 36, "deletions": 53}}, {"node": {"oid": "da016758b3bc648b59bc460e3a707d54ff270e11", "messageHeadline": "Revert \"Update `doc_to_text` to show candidate fill-ins\"", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 1, "deletions": 4}}, {"node": {"oid": "5d431ba920fad66a7bc8421530b060cd6fa5d4a9", "messageHeadline": "Update `doc_to_text` to show candidate fill-ins", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "65c46d229478102f0428c6175477ce0b18d1b020", "messageHeadline": "Add `HellaSwag` evaluation implementation", "author": {"name": "Jon Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 33, "deletions": 23}}, {"node": {"oid": "7031c324359ba49ad1c5570982a7b9739d408ba3", "messageHeadline": "Merge pull request #98 from zphang/wsc", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 19, "deletions": 8}}, {"node": {"oid": "21d527d4816d703c4b5b63f3e4733960c9b3b3ff", "messageHeadline": "Add space before target", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "63e32f9c5575c3c8677d04c7db19caafcd28e5ad", "messageHeadline": "wsc", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 19, "deletions": 8}}, {"node": {"oid": "94d782a0324b9fe6f376c3addde7934883a4be7b", "messageHeadline": "Merge pull request #97 from EleutherAI/unit-testing", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 103, "deletions": 66}}, {"node": {"oid": "60a6fd8c383ab19a3088401f8e9060b966246c2f", "messageHeadline": "Implement unit testing and fix lots of problems with tasks", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 103, "deletions": 66}}, {"node": {"oid": "693c19e25d6f5fc88daff6d51541f8aa8fbfc266", "messageHeadline": "Merge pull request #87 from VitamintK/master", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 133, "deletions": 0}}, {"node": {"oid": "baadb2da9acc8ff040ec5a247bb5e9b976b80028", "messageHeadline": "use openai's gpt3 datafiles instead of generating equations", "author": {"name": "Kevin Wang", "email": "kevinwang@kevinwang.us", "user": {"login": "VitamintK"}}, "additions": 49, "deletions": 82}}, {"node": {"oid": "38892f0a2bb54109fbf9302a4bd8f7ac7d33bd89", "messageHeadline": "Merge pull request #91 from zphang/superglue", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 171, "deletions": 108}}, {"node": {"oid": "2e707b871dfb8b83eca0c9ddf40ab91f6e1ff31a", "messageHeadline": "superglue ex wsc", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 171, "deletions": 108}}, {"node": {"oid": "183f4721a86a1ba84c6547c1edbada08084774e3", "messageHeadline": "prepending a space, because it significantly increases log-likelihood\u2026", "author": {"name": "Kevin Wang", "email": "kevinwang@kevinwang.us", "user": {"login": "VitamintK"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "a58e5e4b2548d3da6c06e143babe3eced2ce501d", "messageHeadline": "first attempt at adding arithmetic evaluations", "author": {"name": "Kevin Wang", "email": "kevinwang@kevinwang.us", "user": {"login": "VitamintK"}}, "additions": 166, "deletions": 0}}, {"node": {"oid": "8ae88962aa4985722427e71bdd662ba8c816c745", "messageHeadline": "Merge pull request #90 from EleutherAI/no_footguns", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 954, "deletions": 181}}, {"node": {"oid": "f4120e596c7a04d6275e079eb885ff9a5a0b60a2", "messageHeadline": "Remove rewrite reminder comment from everything except SuperGLUE", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 37}}, {"node": {"oid": "e31b4b3196fb06616e82b8090a9be87f5d4a5e97", "messageHeadline": "Clean up code, remove some footguns", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 946, "deletions": 143}}, {"node": {"oid": "e723d3d5af602389caa4e689234fa297b637e32c", "messageHeadline": "Implement isgreedy", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 6, "deletions": 1}}, {"node": {"oid": "27a859e2a6814634448144b4f8895368ebb27420", "messageHeadline": "Merge pull request #88 from jon-tow/glue-evaluation-refactor", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 264, "deletions": 231}}, {"node": {"oid": "36467c0e26c717f62bb7a72295e49816286a3278", "messageHeadline": "Adopt new framework for `glue`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 264, "deletions": 231}}, {"node": {"oid": "4c8d22dbcc018cb3ce3538cb4677567c959e38e2", "messageHeadline": "Merge pull request #85 from EleutherAI/sglue-eval-fazz", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 111, "deletions": 74}}, {"node": {"oid": "173b33a409867d4a0d9c32c3542b11bc26f3f33b", "messageHeadline": "Replace tab with spaces, formatting", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 4, "deletions": 3}}, {"node": {"oid": "2c05f0e4d38b97665f6c6743cdc02dd7b1139a2e", "messageHeadline": "Minor formatting correction", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "b19dff508a68cee69c9ab105103aa020f1af8450", "messageHeadline": "Refactor iteration, indentation fix", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 5, "deletions": 6}}, {"node": {"oid": "432226b0271a985d8b334bcac2dd465d067deb97", "messageHeadline": "Remove as functions are inline", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 0, "deletions": 14}}, {"node": {"oid": "64cd64bcb1a2d1aee5e08c5b13c7612bfd78f78d", "messageHeadline": "Merge branch 'master' into sglue-eval-fazz", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 188, "deletions": 201}}, {"node": {"oid": "0e4139b879f9ea0608e59ea02a25aa46ada15325", "messageHeadline": "Merge pull request #86 from jon-tow/doc-to-text-refactor", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 190, "deletions": 193}}, {"node": {"oid": "d77241eb412b92acbf76523639b3552b6ea93389", "messageHeadline": "Move `doc_to_text` target code into `doc_to_target`", "author": {"name": "Jonathan Tow", "email": "jonathantow1@gmail.com", "user": {"login": "jon-tow"}}, "additions": 190, "deletions": 193}}, {"node": {"oid": "a538a1ad06887cb0ef4dca9d31a85497134ec99c", "messageHeadline": "Add MultiRC Implementation", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 49, "deletions": 35}}, {"node": {"oid": "2c6ff0d500979a0da6019ebcb1e06fb3d0458b5c", "messageHeadline": "Add COPA implementation", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 39, "deletions": 17}}, {"node": {"oid": "5165bd38f32f1903ba56c64d445bd4e3bbbac498", "messageHeadline": "Added F1_score metric", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 15, "deletions": 5}}, {"node": {"oid": "68a8790c12f36016b32d1533e27bc5de99700e4c", "messageHeadline": "CommitmentBank framework refactor", "author": {"name": "thefazzer", "email": "pfarrington@gmail.com", "user": {"login": "thefazzer"}}, "additions": 38, "deletions": 22}}, {"node": {"oid": "e5d0229f54b97a13e529ceabf049c3291e7244b8", "messageHeadline": "Merge pull request #82 from EleutherAI/bmk_refactor2", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 127, "deletions": 362}}, {"node": {"oid": "0b3b7251a352ce256e156526b1c1981980985346", "messageHeadline": "Merge branch 'bmk_refactor2' of github.com:EleutherAI/lm_evaluation_h\u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 164, "deletions": 61}}, {"node": {"oid": "a18104a436bc7fa9ba817e3cd970aa742dc5c129", "messageHeadline": "Move higher_is_better and aggregation into their own functions", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 52, "deletions": 45}}, {"node": {"oid": "5ce42fc03bf580c8a29498d6f7f58e081eda3b58", "messageHeadline": "Merge branch 'master' into bmk_refactor2", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 164, "deletions": 61}}, {"node": {"oid": "0f9c162470822b4a0dd2d1a678a99080c82ae555", "messageHeadline": "Refactor and implement SAT evaluation", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 153, "deletions": 361}}, {"node": {"oid": "93510e3a836d936dae22a08171339275f30b2220", "messageHeadline": "Merge pull request #80 from nicholaskross/master", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 132, "deletions": 100}}, {"node": {"oid": "515e047016e21c34b22e756c6314ffdc6cb07ae7", "messageHeadline": "Merge branch 'master' into master", "author": {"name": "Nicholas Kross", "email": "45185629+nicholaskross@users.noreply.github.com", "user": {"login": "nicholaskross"}}, "additions": 295, "deletions": 191}}, {"node": {"oid": "215de04539e9a83d64152c57ee43a34003b68b1f", "messageHeadline": "rewrote based on BoolQ implementation", "author": {"name": "nicholaskross", "email": "nick@nicholaskross.com", "user": {"login": "nicholaskross"}}, "additions": 33, "deletions": 18}}, {"node": {"oid": "afc614fe28a746708ebd3c039e73ec777550f38f", "messageHeadline": "Merge pull request #81 from EleutherAI/more-comments", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 168, "deletions": 63}}, {"node": {"oid": "2d2dbf9685620e145dc54498a7f4dda034623edb", "messageHeadline": "Add comments to remind that evaluation needs to be written for the ne\u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 168, "deletions": 63}}, {"node": {"oid": "57c751fa08f4f4bd3f99077b51cabd415af8cde5", "messageHeadline": "got a start on the SAT eval", "author": {"name": "nicholaskross", "email": "nick@nicholaskross.com", "user": {"login": "nicholaskross"}}, "additions": 20, "deletions": 2}}, {"node": {"oid": "6803e64735195a4b8a120d6de17a36b6183dd5c5", "messageHeadline": "Merge pull request #79 from EleutherAI/bmk_refactor", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 256, "deletions": 275}}, {"node": {"oid": "041ea8a7a479ebcc6fec3bccf89af854fb4e8cd1", "messageHeadline": "Merge branch 'bmk_refactor' of github.com:EleutherAI/lm_evaluation_ha\u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 5}}, {"node": {"oid": "1828175888133f79455216e0e4b15d6429cad574", "messageHeadline": "Merge remote-tracking branch 'origin/master' into bmk_refactor", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "75bea6d59e0352d44f6dd50181878c1a30fb274d", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "611b1d55814fb657bcb4b596378379a1729b3605", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 3}}, {"node": {"oid": "08dc67eaf2ff6483add59fc5b8878b784c3a53b4", "messageHeadline": "Add reminder to rewrite", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 47, "deletions": 0}}, {"node": {"oid": "a9fe09e530fe988a3b13eeb3515ae136b410bb14", "messageHeadline": "Update interface", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 16, "deletions": 3}}, {"node": {"oid": "ea7c5a2c133efb50e43494d024a5f591470755d8", "messageHeadline": "Remove batch_eval", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 186}}, {"node": {"oid": "f298ca76b9b50ff06dfbee7e1c1a646222d6623f", "messageHeadline": "Fix memory problem", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "984f617b6917a849a43057655d4699779b719957", "messageHeadline": "Remove print statement", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "04e72904ecbc6f8092afa600d418106625579a57", "messageHeadline": "Fix up bugs", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 13, "deletions": 11}}, {"node": {"oid": "fb176b576c258f461361b90f71989e3257ecae9c", "messageHeadline": "Fix typo", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "011a0b0c4c8ab203bf917bf4c86570a2d58b59ff", "messageHeadline": "Finish plumbing", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 48, "deletions": 21}}, {"node": {"oid": "2e1b05d24b3919c9cbe54b3c71cd0683f8c2688f", "messageHeadline": "Merge pull request #71 from EleutherAI/uyhcire-normalize-loglikelihoods", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 39, "deletions": 31}}, {"node": {"oid": "870a247a8469557d7a2ab4edf844ae14bb01f3ec", "messageHeadline": "Merge pull request #70 from EleutherAI/uyhcire-batching-without-asyncio", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 172, "deletions": 82}}, {"node": {"oid": "e41a082cd0e7b387be4de4ae3c993cfb3f548060", "messageHeadline": "Update", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 81, "deletions": 34}}, {"node": {"oid": "76e65788d9043adc730afe0f14aa09f972febe0f", "messageHeadline": "Update interfaces", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 89, "deletions": 24}}, {"node": {"oid": "8315dce74796e64b639ba6d8c6264cfaa894b924", "messageHeadline": "Fix eval script to normalize loglikelihoods", "author": {"name": "uyhcire", "email": "27897696+uyhcire@users.noreply.github.com", "user": {"login": "uyhcire"}}, "additions": 39, "deletions": 31}}, {"node": {"oid": "599045ba59b39f78e03a2d3f847b387846a1b8c6", "messageHeadline": "Batch model inputs to speed things up", "author": {"name": "uyhcire", "email": "27897696+uyhcire@users.noreply.github.com", "user": {"login": "uyhcire"}}, "additions": 172, "deletions": 82}}, {"node": {"oid": "9edbc7c072d6d64e415081f7d0f56f07171cb980", "messageHeadline": "Add example", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 70, "deletions": 0}}, {"node": {"oid": "1e6c288500cfe1707cc2a8f5324bfe2d80c310f1", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 142, "deletions": 0}}, {"node": {"oid": "622f17ce1e4ceff2890cf3b5042e93b9c99fa908", "messageHeadline": "Merge pull request #65 from EleutherAI/StellaAthena-patch-1", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "9d975e238f8d0edc002efea6a806d0087a1c3154", "messageHeadline": "Merge pull request #67 from EleutherAI/uyhcire-new-harness-2", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 14}}, {"node": {"oid": "6b453bfde82e06a3c5cdc92b35a719f7bb10e66c", "messageHeadline": "Tweak StoryCloze script to be agnostic to tokenization", "author": {"name": "uyhcire", "email": "27897696+uyhcire@users.noreply.github.com", "user": {"login": "uyhcire"}}, "additions": 1, "deletions": 14}}, {"node": {"oid": "e4af3e77bf5be51945da9a62f1d43debb5ec3c03", "messageHeadline": "Merge pull request #66 from EleutherAI/uychire-new-harness-1", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 154, "deletions": 0}}, {"node": {"oid": "4dbde45aa93b290ae57775648d5e7d77cf075e4a", "messageHeadline": "First pass at StoryCloze evaluation script", "author": {"name": "uyhcire", "email": "27897696+uyhcire@users.noreply.github.com", "user": {"login": "uyhcire"}}, "additions": 129, "deletions": 0}}, {"node": {"oid": "90e02bcad942d8a4967eb144839828115ff92e45", "messageHeadline": "Basic setup", "author": {"name": "uyhcire", "email": "27897696+uyhcire@users.noreply.github.com", "user": {"login": "uyhcire"}}, "additions": 25, "deletions": 0}}, {"node": {"oid": "12c2ee1ed46728e9a05808358852c0d1a5e9ea8e", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 19, "deletions": 89}}, {"node": {"oid": "87b20e1d4898213ef4b1bb65cc69a7a1538cb2cf", "messageHeadline": "Create CODEOWNERS", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "61ff104e40c122d788d917e2ba7697f947cd135c", "messageHeadline": "Merge pull request #63 from EleutherAI/refactor_tokenization", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 19, "deletions": 89}}, {"node": {"oid": "75db3899dfc0e42401b577f343c809d8654aea6d", "messageHeadline": "Update docstring", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 4}}, {"node": {"oid": "e3031e847da58d5312ba50c85b433ca26b018f40", "messageHeadline": "Remove num_tokens", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 17}}, {"node": {"oid": "cf69ba9c13d8c278305f457b2cb47918d0380f28", "messageHeadline": "Add missing import", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "90e50b4c977ffb4a8f5903280cd230ed7953b4f9", "messageHeadline": "Refactor to remove generate and fix some bad tokenization", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 18, "deletions": 68}}, {"node": {"oid": "6de520aff227d0c7e491ff7c873c27d02b681e4a", "messageHeadline": "Make fewshot_examples fast", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 12, "deletions": 15}}, {"node": {"oid": "ff3adfe2f8a254f9a67b286cb1b7fdcbce80ea7d", "messageHeadline": "Undo MNLI changes", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 8}}, {"node": {"oid": "70e30a52d4f83a5dc1bdbbb3c70a298b5d2b89e7", "messageHeadline": "Merge branch 'master' of github.com:EleutherAI/lm_evaluation_harness", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 0}}, {"node": {"oid": "26d59f34775f418f65f255c8c18f1bcf04492123", "messageHeadline": "Fix MNLIMismatched", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "49cc6f5d93990f801966015892a77c18e7eaeba2", "messageHeadline": "Allow specifying sets for write_out", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 20, "deletions": 4}}, {"node": {"oid": "d4ae0c003f46280dd78a8564926430bab19317bf", "messageHeadline": "Allow specifying sets for write_out", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 21, "deletions": 4}}, {"node": {"oid": "1c9432de0a766b76aaf874db84f69bc2f91be1fc", "messageHeadline": "Fix MNLI train set", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "5076d212987bea3d2986bffb1e88e9040fd0d71d", "messageHeadline": "Remove old coqa file", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 15}}, {"node": {"oid": "a1ef56aa908b947655691ee0f47b6165a8eeecc9", "messageHeadline": "Allow num_examples to fetch all if num_examples < 0", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "15c87e9b262972337dac3ebd3e28c80154c992ee", "messageHeadline": "Merge pull request #59 from EleutherAI/dev", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 9}}, {"node": {"oid": "6c93185c78c9fcdd21e30c4ebe7755b2d5a5f054", "messageHeadline": "Merge branch 'master' into dev", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 2}}, {"node": {"oid": "37fab2009549e1a50ae604a9bfb631f2b40657b7", "messageHeadline": "Changed arg name to match master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "760788d52e9fe084acaaf2d09b6fab14b5f79ed4", "messageHeadline": "added import statement", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "3f5d4beb2f9707dc5a60b28642e9be50c6cde6b2", "messageHeadline": "Added truncation flag", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "fb5aaf5120c4b6f6533d5f17c38d5c5784f0b198", "messageHeadline": "Add --limit flag", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 9, "deletions": 1}}, {"node": {"oid": "8d7d21321bfc2f07278ab8d227c87a9040a95ee3", "messageHeadline": "Put gpt2 in eval mode", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 0}}, {"node": {"oid": "0d291df6475c2027803d2b9ace8265bb9f546554", "messageHeadline": "Merge pull request #57 from cfoster0/sat_analogies", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 98, "deletions": 0}}, {"node": {"oid": "8b83f341ef8c306bdea33a35cad36a4f31e9db4a", "messageHeadline": "Add SAT analogies dataset. Manual download needed. Checksums currentl\u2026", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 98, "deletions": 0}}, {"node": {"oid": "4a294d8afb4520ded2a340a11c4808fc5caf926f", "messageHeadline": "Merge pull request #3 from EleutherAI/master", "author": {"name": "Charles Foster", "email": "cfoster0@stanford.edu", "user": {"login": "cfoster0"}}, "additions": 381, "deletions": 37}}, {"node": {"oid": "946cb2bc1bfba0cacf6025936f2722779ef59b5f", "messageHeadline": "Merge pull request #56 from anishthite/master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 28, "deletions": 3}}, {"node": {"oid": "8e4ff678193ba620a73e0d5c06915cebe219921d", "messageHeadline": "update lambada for new dataset class", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 6, "deletions": 3}}, {"node": {"oid": "d4daf44c909fa85822dbd932d69b984f4bb6613e", "messageHeadline": "add rte text", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 22, "deletions": 0}}, {"node": {"oid": "dcadfa7c0861030847b9d683415072f1a0b7a17c", "messageHeadline": "Merge pull request #52 from cfoster0/natural_questions", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 47, "deletions": 0}}, {"node": {"oid": "9eec7ce89f4ee3f0491e5ca045e6888dbe8286c0", "messageHeadline": "Merge pull request #55 from anishthite/master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 69, "deletions": 4}}, {"node": {"oid": "171f2924a4c831b7e863bbc458c2927bc409e953", "messageHeadline": "add no target for triviaqa", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 4, "deletions": 2}}, {"node": {"oid": "c9f3b113febbf743b3603029d63530d214e20730", "messageHeadline": "add no target for storycloze", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "e3c2e69267d000462b7da25a712860cdb25e426b", "messageHeadline": "add include target for coqa", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 4, "deletions": 1}}, {"node": {"oid": "7c55e46b4f95b36322593af892efae1286d0591f", "messageHeadline": "add piqa text", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 57, "deletions": 0}}, {"node": {"oid": "3c480b61747b76a3f991d46c6f03432f508e6509", "messageHeadline": "Merge pull request #54 from zphang/mnli_mismatched", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 12, "deletions": 0}}, {"node": {"oid": "b278e42ddb8f0841b8eee6bf459f05c71947681c", "messageHeadline": "Merge pull request #53 from anishthite/master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 112, "deletions": 32}}, {"node": {"oid": "c013679d3330982958bbec892050cb79ee787664", "messageHeadline": "Fixes to natural questions.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 7, "deletions": 5}}, {"node": {"oid": "dc3560d03f8535710777bc0446540847f30750d1", "messageHeadline": "Updates to natural questions answer handling. Now uses long answers a\u2026", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 3, "deletions": 6}}, {"node": {"oid": "d51f3e7f8164f003a8b5ee3ecb931724a24eb3b5", "messageHeadline": "Small typos", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 1, "deletions": 2}}, {"node": {"oid": "9291edaa42920501de7926ab664cba190326f877", "messageHeadline": "Updates to natural questions to deal with memory issues.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 7, "deletions": 0}}, {"node": {"oid": "0dda9f6314830876bbc6299b683004efca1bde6b", "messageHeadline": "Removed apache_beam dependency.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 0, "deletions": 1}}, {"node": {"oid": "042e2926ad207250d53cc4fee4cd6717f4879807", "messageHeadline": "add storycloze", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 47, "deletions": 0}}, {"node": {"oid": "25131aa743c2f9db9674f573830dec1e3d03eb25", "messageHeadline": "mnli_mismatched", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 12, "deletions": 0}}, {"node": {"oid": "0f2b9317b109d34abd29e88ae34fafb1964beea8", "messageHeadline": "update triviaqa download", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "8bfaaa50fca9355b2fe2dd9ce08d4416ebdb63f7", "messageHeadline": "add triviaqa data", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 45, "deletions": 0}}, {"node": {"oid": "97e5a5665a21d504cbb528ee2648296b4eb791c8", "messageHeadline": "update coqa dev set", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "db3fe0d6a6ca85f0ce36fc28889c7c56dc5e55aa", "messageHeadline": "update coqa dev set", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "5d601e1494c142cb768e408aba6e7718d99bbe36", "messageHeadline": "update coqa to be consistent with gpt3 paper", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 12, "deletions": 27}}, {"node": {"oid": "01608cf4d70664efc597474633434beca0417b81", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm_evaluation_\u2026", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 87, "deletions": 2}}, {"node": {"oid": "43fc77a2227010b65028b76213e1aa96b0e93586", "messageHeadline": "make drop text consistent with gpt3 paper", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 6, "deletions": 3}}, {"node": {"oid": "9426db162c691f7afe7788c2298ab48bb1462704", "messageHeadline": "Added natural questions. Not yet validated or tested with a call to w\u2026", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 43, "deletions": 0}}, {"node": {"oid": "00abc99abc583ee16ea0cec00425d55beefec0ae", "messageHeadline": "Merge pull request #51 from anishthite/master", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 30, "deletions": 0}}, {"node": {"oid": "b0db32bc277a3ebda1e1192bb5e18bc7931b01a2", "messageHeadline": "Merge pull request #50 from cfoster0/winograd", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 87, "deletions": 2}}, {"node": {"oid": "262fe2509ffa934b6e601efc353a19f932980f9f", "messageHeadline": "add Wikitext2 + wikitext103 data", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 30, "deletions": 0}}, {"node": {"oid": "05bd05e91aa5e43256da075f0573a577544b4bf2", "messageHeadline": "Renamed WSC to make distinction between SuperGLUE Winograd Schemas (S\u2026", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 87, "deletions": 2}}, {"node": {"oid": "ca24b52f12bce9c9ddd5b6044148ef41b1ddfbfe", "messageHeadline": "Merge pull request #2 from EleutherAI/master", "author": {"name": "Charles Foster", "email": "cfoster0@stanford.edu", "user": {"login": "cfoster0"}}, "additions": 292, "deletions": 31}}, {"node": {"oid": "302ca3d6d6d37e78fc0ed9f0bfbab67ec7d1e9fb", "messageHeadline": "Merge pull request #1 from EleutherAI/master", "author": {"name": "Charles Foster", "email": "cfoster0@stanford.edu", "user": {"login": "cfoster0"}}, "additions": 292, "deletions": 31}}, {"node": {"oid": "b8a3edaf3445871ebefc10c68d8caedae716577f", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 0}}, {"node": {"oid": "8cb71ef7f2034fe90d36543c28ec42ad89786d8c", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 1, "deletions": 4}}, {"node": {"oid": "a78996bcf3d833d792b2790bc79c9dfce1d8c839", "messageHeadline": "Update README.md", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 0, "deletions": 25}}, {"node": {"oid": "eb9a522447ccbabd536622dfb9f2cb8c072ac373", "messageHeadline": "Merge pull request #45 from cfoster0/winogrande", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 52, "deletions": 0}}, {"node": {"oid": "0e0e37f48022b4c1cd20b0c568ca4a3a301a9bd6", "messageHeadline": "Merge branch 'master' into winogrande", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 238, "deletions": 3}}, {"node": {"oid": "80f5fc3be7717e38bcbd3e39e725c32afd96086e", "messageHeadline": "Merge pull request #48 from cfoster0/quac", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 71, "deletions": 0}}, {"node": {"oid": "a543cc5d2114d22c42e12076a210350d27948289", "messageHeadline": "Merge branch 'master' into quac", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 114, "deletions": 0}}, {"node": {"oid": "2b64cae6951dd3ba4d704d1371e50725dc9e79ab", "messageHeadline": "Merge pull request #43 from cfoster0/hellaswag", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 56, "deletions": 0}}, {"node": {"oid": "b14f3d1d7de497f4928f059e02b7b13f93172ff5", "messageHeadline": "Merge branch 'master' into hellaswag", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 111, "deletions": 3}}, {"node": {"oid": "bbc29a93f7da3eefb2da697b1188ae161b715d55", "messageHeadline": "Merge pull request #42 from cfoster0/openbookqa", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 58, "deletions": 0}}, {"node": {"oid": "efc4310cc03a86b46e3dd366aa295e97c11776d5", "messageHeadline": "Merge branch 'master' into openbookqa", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 53, "deletions": 3}}, {"node": {"oid": "9017bb3665f5845de72707683f581e8dd5e196a0", "messageHeadline": "Merge branch 'master' into quac", "author": {"name": "Charles Foster", "email": "cfoster0@stanford.edu", "user": {"login": "cfoster0"}}, "additions": 53, "deletions": 3}}, {"node": {"oid": "2338188f24f0dd413675ac074f2be744d377265d", "messageHeadline": "Merge pull request #41 from zphang/copa_update", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "3d5d794a83f688005aeabff2420893f648b605b1", "messageHeadline": "Merge pull request #46 from cfoster0/nlp_name_fix", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "c7e7a0503f8960c86a8bf4940ec34176cf160164", "messageHeadline": "Merge pull request #47 from cfoster0/squad", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 46, "deletions": 0}}, {"node": {"oid": "40cc4e28711a906eafe15f418bc427d22d8bd173", "messageHeadline": "Added QuAC dataset.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 71, "deletions": 0}}, {"node": {"oid": "76b081338585cda8141bebb4f355068ef01ea4bd", "messageHeadline": "Clarified that SQuaD does not include a test set.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 3, "deletions": 0}}, {"node": {"oid": "5c58b2674d802ae76c80069148fb5bd945c7e182", "messageHeadline": "Added SQuAD v2 dataset", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 43, "deletions": 0}}, {"node": {"oid": "b106e4f7092b6b37e65ce3e6944d58277ff30530", "messageHeadline": "Updated RACE to use datasets instead of nlp package.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 2, "deletions": 2}}, {"node": {"oid": "dbe43ec33e10c919b55006bc50d04c0c13cbf40c", "messageHeadline": "Add Winogrande dataset.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 52, "deletions": 0}}, {"node": {"oid": "a245d2e4d72f470fc2d6cb165d0776e9d3c5db82", "messageHeadline": "Add HellaSwag dataset.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 56, "deletions": 0}}, {"node": {"oid": "8402847e1f5deeb9bbe67950e05e1d87bf045f5e", "messageHeadline": "Added openbookqa to registry.", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 2, "deletions": 0}}, {"node": {"oid": "4c8ec8af077ac8460a055b60c4a1e01fbf4fd00d", "messageHeadline": "Added openbookqa", "author": {"name": "Charles Foster", "email": "cfoster0@ccrma.stanford.edu", "user": null}, "additions": 56, "deletions": 0}}, {"node": {"oid": "d5893cdc3fbee81daf03cdd8c47598195955753e", "messageHeadline": "Update COPA cause/effect", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 5, "deletions": 1}}, {"node": {"oid": "aa125d0a5a0395f58a003ee261f2093d84995819", "messageHeadline": "Add ANLI", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 61, "deletions": 0}}, {"node": {"oid": "988a400f61901265e0625bcb8fe10cb7c3bd93e4", "messageHeadline": "Add abstract constructor", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 0}}, {"node": {"oid": "b0585de48959dc9664c6abfc65a2efc47a18b2e6", "messageHeadline": "Don't make Dataset.download abstract", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 1}}, {"node": {"oid": "a5c48976ed8e8b866514da0398e32390c7367e8f", "messageHeadline": "More search and replace", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 21, "deletions": 21}}, {"node": {"oid": "6469a0e766c81467854177dac654be0e49df5322", "messageHeadline": "Replace HFNLPTask with HFTask everywhere", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 29, "deletions": 14}}, {"node": {"oid": "f348fa2cb7d102d7b12232f2a483361abf6f60c4", "messageHeadline": "Merge pull request #39 from EleutherAI/add_lambada", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 82, "deletions": 38}}, {"node": {"oid": "f161731cb7ea1743dc50b1fb0ee2ffc44728e820", "messageHeadline": "Merge branch 'master' into add_lambada", "author": {"name": "sdtblck", "email": "46172032+sdtblck@users.noreply.github.com", "user": {"login": "sdtblck"}}, "additions": 478, "deletions": 18}}, {"node": {"oid": "5a6c172eef990ce1c33916d1d4e23d1d360073c8", "messageHeadline": "change HF_Dataset to HFTask", "author": {"name": "sdtblck", "email": "sdtblck@gmail.com", "user": {"login": "sdtblck"}}, "additions": 11, "deletions": 11}}, {"node": {"oid": "08f2ed6e41aa9f883ee65a20555897e009c3902e", "messageHeadline": "add lambada", "author": {"name": "sdtblck", "email": "sdtblck@gmail.com", "user": {"login": "sdtblck"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "fb8c3df913ad3de91da3c148d2600495369f0954", "messageHeadline": "add lambada", "author": {"name": "sdtblck", "email": "sdtblck@gmail.com", "user": {"login": "sdtblck"}}, "additions": 45, "deletions": 0}}, {"node": {"oid": "b0e715eea33cbb61ab2887e12f711bcb665e3fc5", "messageHeadline": "name changes", "author": {"name": "sdtblck", "email": "sdtblck@gmail.com", "user": {"login": "sdtblck"}}, "additions": 34, "deletions": 43}}, {"node": {"oid": "43978e3be69fb6a91acfe2790fc3ebe907e59be9", "messageHeadline": "Merge pull request #37 from EleutherAI/download_on_demand", "author": {"name": "Anish Thite", "email": "anishthite@users.noreply.github.com", "user": {"login": "anishthite"}}, "additions": 44, "deletions": 7}}, {"node": {"oid": "4e2d14986fc4171696e46821ad01277fbafcf8c3", "messageHeadline": "add download method to base class", "author": {"name": "sdtblck", "email": "sdtblck@gmail.com", "user": {"login": "sdtblck"}}, "additions": 7, "deletions": 1}}, {"node": {"oid": "ff2d2ada71aaaf3cfdf26e8df8f83b999c994364", "messageHeadline": "add download method to base class", "author": {"name": "sdtblck", "email": "sdtblck@gmail.com", "user": {"login": "sdtblck"}}, "additions": 21, "deletions": 0}}, {"node": {"oid": "53d4f85086b658ef78aaa0bc558e9e61047e316d", "messageHeadline": "update nlp class to use HF datasets library", "author": {"name": "sdtblck", "email": "sdtblck@gmail.com", "user": {"login": "sdtblck"}}, "additions": 16, "deletions": 6}}, {"node": {"oid": "d6b9119103481aa8ad93346cc34aad879219adad", "messageHeadline": "Merge pull request #35 from zphang/superglue2", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 61, "deletions": 0}}, {"node": {"oid": "bac0a5286e4afcc48ec1e28046d04edf56418b33", "messageHeadline": "multirc", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 61, "deletions": 0}}, {"node": {"oid": "00038fea6711e14a74882db05b7fb8b711c48104", "messageHeadline": "Q/A instead of Question/Answer", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 2, "deletions": 2}}, {"node": {"oid": "6c816cb70dd482c3a419f13476b0796f5575077b", "messageHeadline": "write_out: fallover to training data if no validation", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 2}}, {"node": {"oid": "31b2a5ddb3e8b19a5babd352b86c6a5ebaed2449", "messageHeadline": "Add WebQs (data only)", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 34, "deletions": 0}}, {"node": {"oid": "6cc21f4b0c592e0364bb94372813909a85a43851", "messageHeadline": "Fix include_target", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 6, "deletions": 6}}, {"node": {"oid": "12852d5c69b3065c2d585a2f0315e14ab52794c9", "messageHeadline": "Implement ARC (data)", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 231, "deletions": 0}}, {"node": {"oid": "9e29c2a880ef23007612f9481091f4c15855faa5", "messageHeadline": "Implement ARC", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 35, "deletions": 0}}, {"node": {"oid": "edbb1a4927448b204f83fc9e9c47e755ce310cb9", "messageHeadline": "Rename NLP_TASK", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 18, "deletions": 18}}, {"node": {"oid": "d53969b5baf744a7f4220b71d7f0520962f8acbf", "messageHeadline": "Merge branch 'master' of https://github.com/EleutherAI/lm_evaluation_\u2026", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 6, "deletions": 5}}, {"node": {"oid": "ec4d361514ec0779e502460a705bd875cc11299c", "messageHeadline": "add drop data + doc2text", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 84, "deletions": 1}}, {"node": {"oid": "d9e50f87820452ba44a1523a70c54026d72c3818", "messageHeadline": "Remove residual MODEL_REGISTRY", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 0, "deletions": 2}}, {"node": {"oid": "ffea4dc55670dd31bcdb0f3264563f612e955952", "messageHeadline": "Fix GPT2 impl partially", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 6, "deletions": 3}}, {"node": {"oid": "5888a6959aec906f6e15274cc8752e778b2e73f7", "messageHeadline": "Merge pull request #33 from zphang/retrieval", "author": {"name": "Leo Gao", "email": "54557097+leogao2@users.noreply.github.com", "user": {"login": "leogao2"}}, "additions": 34, "deletions": 75}}, {"node": {"oid": "515d78b3b4f9b0c52acd972c2bd528a14abff9fa", "messageHeadline": "refactor to explicit registries", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 4, "deletions": 12}}, {"node": {"oid": "89de8d7de80ca1a21f12a6f28ae61502305fa63c", "messageHeadline": "refactor to explicit registries", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 34, "deletions": 67}}, {"node": {"oid": "635a21558ad6480f4e59b87e80ef20407d868f28", "messageHeadline": "Merge pull request #4 from zphang/superglue_part_1", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 417, "deletions": 32}}, {"node": {"oid": "f1ec7b0611d8db90b307fd7e7eacc7c6a88f8762", "messageHeadline": "Update docs", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 82, "deletions": 8}}, {"node": {"oid": "a85ad21401d8bc8329e39e7e513db6cdf5b7c263", "messageHeadline": "cleanup", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 12, "deletions": 10}}, {"node": {"oid": "8161c22e9ed0d1f8780b86467dfe4e6c94a5c7cf", "messageHeadline": "SuperGLUE, and truncation", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 328, "deletions": 19}}, {"node": {"oid": "ffbaef21e66dd1e4c7b6b6eefdf8d38b0a6887bc", "messageHeadline": "Merge pull request #3 from zphang/refactor", "author": {"name": "Stella Biderman", "email": "stellabiderman@gmail.com", "user": {"login": "StellaAthena"}}, "additions": 829, "deletions": 130}}, {"node": {"oid": "e7a87e7149f22df5ccd0d6cf9d562d5b0314e0d2", "messageHeadline": "GPT-2 fixes", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 9, "deletions": 9}}, {"node": {"oid": "9987203f93ca6e3b370a43095c459c21e7000ee8", "messageHeadline": "seed", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 8, "deletions": 1}}, {"node": {"oid": "32624a1dfd80b656ca9f012a737f1ba5990304b3", "messageHeadline": "remove seeding", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 14, "deletions": 8}}, {"node": {"oid": "f88bb82710a29057b378b747e8b9cfd33a0a80e8", "messageHeadline": "lib", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 22, "deletions": 64}}, {"node": {"oid": "cf80f3406aaa04f2541e6d3cf6945d0fd3495046", "messageHeadline": "glue tasks", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 406, "deletions": 50}}, {"node": {"oid": "c2aaa5015a3807a73a4521d7445a919add9d10b7", "messageHeadline": "combine gpt2", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 18, "deletions": 32}}, {"node": {"oid": "2d4b3a8c3997a77599c3d0a9a5921c6d108c83db", "messageHeadline": "checkin", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 221, "deletions": 7}}, {"node": {"oid": "12e12bc0212bc6110f21c67cfb0344e8ce0df148", "messageHeadline": "gpt3", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 178, "deletions": 6}}, {"node": {"oid": "7a32afeb1b70e8f07a143425c3d918bdc758e927", "messageHeadline": "Merge branch 'master' of git@github.com:EleutherAI/lm_evaluation_harn\u2026", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 73, "deletions": 159}}, {"node": {"oid": "1f3b3988c21572fe69bd895d90c0c93dae888259", "messageHeadline": "Update boolq", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 12}}, {"node": {"oid": "ba2b849cbb123d359f5ff18a59f295e61d77b3b1", "messageHeadline": "GPT2: add device parameter", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 5, "deletions": 4}}, {"node": {"oid": "5bb5e96ddabebd777c953977abed432ca8ec8b57", "messageHeadline": "Implement boolq dataset", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 51, "deletions": 0}}, {"node": {"oid": "8fffd9274ba15369f872ab17ba692c2b49af66b6", "messageHeadline": "Implement gpt2 loglikelihood", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 9, "deletions": 1}}, {"node": {"oid": "3169691007e2edfbf16a68c086052843200f7324", "messageHeadline": "Add extra options to evaluate", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 1, "deletions": 1}}, {"node": {"oid": "7592026768fa15903259b3db44b39252bbe2beeb", "messageHeadline": "Change nll to loglikelihood", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 3, "deletions": 3}}, {"node": {"oid": "a2871d9cc8c905bdb357ce590be08404156e4051", "messageHeadline": "add generate docs and generate text for coqa", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 53, "deletions": 157}}, {"node": {"oid": "345f5fefaae1fee246d25d11cf318dd009544f0d", "messageHeadline": "add gitignore for data/", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 2, "deletions": 1}}, {"node": {"oid": "bd34854288b4cee61bab443431f334a469e67b08", "messageHeadline": "merge changes, added datasets to README.md and if train dataset or va\u2026", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 18, "deletions": 1}}, {"node": {"oid": "8a5a5f74bf78aa5de291c2b7adb9d3891ab8551b", "messageHeadline": "Add evaluate method", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 4, "deletions": 0}}, {"node": {"oid": "472a67d3f758861ac0360267f7684b0fabde18ea", "messageHeadline": "Merge pull request #2 from zphang/readme_update", "author": {"name": "Anish Thite", "email": "anishthite@users.noreply.github.com", "user": {"login": "anishthite"}}, "additions": 44, "deletions": 3}}, {"node": {"oid": "185c6ade84525ea5bbd78511e225b44a8a1037dc", "messageHeadline": "Update README", "author": {"name": "Jason Phang", "email": "email@jasonphang.com", "user": {"login": "zphang"}}, "additions": 44, "deletions": 3}}, {"node": {"oid": "b21d2b6ee3248ddc553a52a4d6d05c8e5b8d8721", "messageHeadline": "Merge pull request #1 from anishthite/master", "author": {"name": "Anish Thite", "email": "anishthite@users.noreply.github.com", "user": {"login": "anishthite"}}, "additions": 163, "deletions": 0}}, {"node": {"oid": "0feef0d60e3b58a7583eb196063e0e86b9d40e2e", "messageHeadline": "add coqa extraction", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 157, "deletions": 0}}, {"node": {"oid": "38ed3ac84aa7397d12354fbe11b52ae7e8035c6d", "messageHeadline": "Add README", "author": {"name": "Anish Thite", "email": "anishthite@gmail.com", "user": {"login": "anishthite"}}, "additions": 6, "deletions": 0}}, {"node": {"oid": "1c7fb748e0055e0fa4a00dc3d24f7f129d0b3239", "messageHeadline": "Initial Commit", "author": {"name": "Leo Gao", "email": "leogao31@gmail.com", "user": {"login": "leogao2"}}, "additions": 324, "deletions": 0}}]