pulipakav-1/translated-babylm-hindi

train/   — Training data (translated from BabyLM-2026-Strict, 100M tokens)
val/     — Validation data (translated from BabyLM-dev)
test/    — Test data (translated from BabyLM-Test)

from datasets import load_dataset

dataset = load_dataset("pulipakav-1/translated-babylm-hindi")
print(dataset)
# DatasetDict({
#     train: Dataset({features: ['text'], num_rows: 11579880}),
#     val:   Dataset({features: ['text'], num_rows: 1153113}),
#     test:  Dataset({features: ['text'], num_rows: 1097453})
# })

from datasets import load_dataset

train_data = load_dataset("pulipakav-1/translated-babylm-hindi", split="train")
val_data   = load_dataset("pulipakav-1/translated-babylm-hindi", split="val")
test_data  = load_dataset("pulipakav-1/translated-babylm-hindi", split="test")

from datasets import load_dataset

# Available sources: bnc_spoken, childes, gutenberg, open_subtitles, simple_wiki, switchboard
ds = load_dataset(
    "pulipakav-1/translated-babylm-hindi",
    data_files={"train": "train/childes.train.hi.txt"},
    split="train"
)

from datasets import load_dataset

dataset = load_dataset("pulipakav-1/translated-babylm-hindi", split="train")
for example in dataset:
    print(example["text"])
    break

from datasets import load_dataset
from transformers import AutoTokenizer

dataset = load_dataset("pulipakav-1/translated-babylm-hindi")
tokenizer = AutoTokenizer.from_pretrained("your-tokenizer")

def tokenize(example):
    return tokenizer(example["text"], truncation=True, max_length=512)

tokenized = dataset.map(tokenize, batched=True, remove_columns=["text"])

from datasets import load_dataset

dataset = load_dataset("pulipakav-1/translated-babylm-hindi", split="train")
dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])

from torch.utils.data import DataLoader
dataloader = DataLoader(dataset, batch_size=32, shuffle=True)

@misc{choshen2026babylmturns4papers,
  title  = {BabyLM Turns 4: Call for Papers for the 2026 BabyLM Workshop},
  author = {Leshem Choshen and Ryan Cotterell and Mustafa Omer Gul and Jaap Jumelet and Tal Linzen and Aaron Mueller and Suchir Salhan and Raj Sanjay Shah and Alex Warstadt and Ethan Gotlieb Wilcox},
  year   = {2026}
}

@article{gala2023indictrans2,
  title   = {IndicTrans2: Towards High-Quality and Accessible Machine Translation Models for all 22 Scheduled Indian Languages},
  author  = {Gala, Jay and Chitale, Pranjal A. and AK, Raghavan and Gumma, Varun and Doddapaneni, Sumanth and Kumar, Aswanth and Nawale, Janki and Sujatha, Anupama and Puduppully, Ratish and Raghavan, Vivek and Kumar, Pratyush and Khapra, Mitesh M. and Kunchukuttan, Anoop},
  journal = {Transactions on Machine Learning Research},
  year    = {2023},
  url     = {https://arxiv.org/abs/2305.16307}
}

@article{burnard2007british,
  title     = {Reference Guide for the British National Corpus (XML Edition)},
  author    = {Burnard, Lou},
  year      = {2007},
  publisher = {Oxford Text Archive}
}

@article{macwhinney2000childes,
  title   = {The CHILDES Project: Tools for Analyzing Talk},
  author  = {MacWhinney, Brian},
  journal = {Computational Linguistics},
  year    = {2000}
}

@misc{project_gutenberg,
  title  = {Project Gutenberg},
  author = {Hart, Michael},
  year   = {1971},
  url    = {https://www.gutenberg.org}
}

@inproceedings{lison2016opensubtitles2016,
  title     = {OpenSubtitles2016: Extracting Large Parallel Corpora from Movie and TV Subtitles},
  author    = {Lison, Pierre and Tiedemann, J{\"o}rg},
  booktitle = {Proceedings of LREC},
  year      = {2016}
}

@misc{simple_wikipedia,
  title  = {Simple English Wikipedia},
  author = {Wikimedia Foundation},
  url    = {https://simple.wikipedia.org}
}

@article{godfrey1992switchboard,
  title   = {SWITCHBOARD: Telephone Speech Corpus for Research and Development},
  author  = {Godfrey, John J and Holliman, Edward C and McDaniel, Jane},
  journal = {ICASSP},
  year    = {1992}
}