hubnemo/tulu3-sft-mini
This is a subset derived from tulu3 sft mixture limited to 20 samples for each source. The use case for this smaller dataset is to have a short, consistent evaluation dataset over different domains for multi-token prediction. Here's the code for how to derive this dataset: import datasets DATASET = "allenai/tulu-3-sft-mixture" OFFSETS = [ ("ai2-adapt-dev/oasst1_converted", 0, 7131), ("ai2-adapt-dev/flan_v2_converted", 7131, 97113), ("ai2-adapt-dev/tulu_hard_coded_repeated_10"… See the full description on the dataset page: https://huggingface.co/datasets/hubnemo/tulu3-sft-mini.
This is a subset derived from tulu3 sft mixture limited to 20 samples for each source.
The use case for this smaller dataset is to have a short, consistent evaluation dataset over different domains for multi-token prediction.
Here's the code for how to derive this dataset:
import datasets
DATASET = "allenai/tulu-3-sft-mixture"
OFFSETS = [
("ai2-adapt-dev/oasst1_converted", 0, 7131),
("ai2-adapt-dev/flan_v2_converted", 7131, 97113),
("ai2-adapt-dev/tulu_hard_coded_repeated_10", 97113, 97353),
("ai2-adapt-dev/no_robots_converted", 97353, 106853),
("ai2-adapt-dev/tulu_v3.9_wildchat_100k", 106853, 206853),
("ai2-adapt-dev/personahub_math_v5_regen_149960", 206853, 356813),
("allenai/tulu-3-sft-personas-math-grade", 356813, 406793),
("ai2-adapt-dev/tulu_v3.9_open_math_2_gsm8k_50k", 406793, 456793),
("ai2-adapt-dev/numinamath_tir_math_decontaminated", 456793, 521105),
("ai2-adapt-dev/tulu_v3.9_personahub_math_interm_algebra_20k", 521105, 541105),
("ai2-adapt-dev/personahub_code_v2_34999", 541105, 576104),
("ai2-adapt-dev/evol_codealpaca_heval_decontaminated", 576104, 683380),
("ai2-adapt-dev/personahub_ifdata_manual_seed_v3_29980", 683380, 713360),
("ai2-adapt-dev/coconot_converted", 713360, 724343),
("ai2-adapt-dev/tulu_v3.9_wildjailbreak_decontaminated_50k", 724343, 774343),
(
"ai2-adapt-dev/tulu_v3.9_synthetic_finalresp_wildguardmixtrain_decontaminated_50k",
774343,
824343,
),
("ai2-adapt-dev/tulu_v3.9_sciriff_10k", 824343, 834343),
("ai2-adapt-dev/tulu_v3.9_table_gpt_5k", 834343, 839343),
("ai2-adapt-dev/tulu_v3.9_aya_100k", 839343, 939343),
]
def sample(samples_per_source):
# sample equally per category; fails if n > min(category_counts)
ds = datasets.load_dataset(DATASET, split="train")
indices = {source: (start, end) for source, start, end in OFFSETS}
to_sample = {source for source, _, _ in OFFSETS}
data = []
for source in to_sample:
start_idx, end_idx = indices[source]
data.append(ds.skip(start_idx).take(samples_per_source))
assert len(data[-1]) == samples_per_source
for row in data[-1]:
assert row['source'] == source
return datasets.concatenate_datasets(data)
def main():
ds = sample(samples_per_source=20)
ds.save_to_disk('data/tulu3_mini')
if __name__ == "__main__":
main()