hoskinson-center/proof-pile

1from datasets import load_dataset2from itertools import islice3import sys4import time5from tqdm import tqdm6 7dataset = load_dataset("./proof-pile.py", "default")8 9size = dataset["train"].dataset_size / 2**3010print(f"{size} GB TRAIN TOTAL")11print(dataset)12for x in tqdm(dataset["train"]): 13    print("EXAMPLE INSTANCE (trimmed):")14    print(x["text"][:100])15    break16 17then = time.time()18for x in tqdm(dataset["train"]):19    pass20now = time.time()21print(f"{size} GB TRAIN TOTAL")22print(f"TRAVERSED IN {now-then} SECONDS")23 24size += dataset["validation"].dataset_size/2**30 + dataset["test"].dataset_size/2**3025print(f"{size} GB TOTAL (TRAIN, VAL, TEST)")26 27