enigmare/v2-crawler

1{2  "corpusVersion": "1.0.0",3  "corpusHash": "sha256-split-1787310214153",4  "splitRatios": {5    "train": 0.95,6    "validation": 0.03,7    "test": 0.028  },9  "randomSeed": 42,10  "timestamp": "2026-08-21T11:03:34.153Z",11  "totalDocumentCount": 2321735,12  "trainCount": 2205645,13  "validationCount": 69648,14  "testCount": 46442,15  "totalTokens": 3337029721,16  "trainTokens": 3169601496,17  "validationTokens": 101401499,18  "testTokens": 66026726,19  "sources": {20    "books": {21      "source": "books",22      "candidateDocuments": 1474,23      "candidateTokens": 185262788,24      "trainDocuments": 1400,25      "trainTokens": 176292558,26      "validationDocuments": 44,27      "validationTokens": 5203232,28      "testDocuments": 30,29      "testTokens": 376699830    },31    "commoncrawl": {32      "source": "commoncrawl",33      "candidateDocuments": 372505,34      "candidateTokens": 540883137,35      "trainDocuments": 353879,36      "trainTokens": 513930403,37      "validationDocuments": 11175,38      "validationTokens": 16431670,39      "testDocuments": 7451,40      "testTokens": 1052106441    },42    "cp": {43      "source": "cp",44      "candidateDocuments": 67889,45      "candidateTokens": 49075577,46      "trainDocuments": 64494,47      "trainTokens": 46644485,48      "validationDocuments": 2036,49      "validationTokens": 1513008,50      "testDocuments": 1359,51      "testTokens": 91808452    },53    "devdocs": {54      "source": "devdocs",55      "candidateDocuments": 174646,56      "candidateTokens": 198592155,57      "trainDocuments": 165913,58      "trainTokens": 188795573,59      "validationDocuments": 5239,60      "validationTokens": 5827863,61      "testDocuments": 3494,62      "testTokens": 396871963    },64    "documentation": {65      "source": "documentation",66      "candidateDocuments": 20199,67      "candidateTokens": 26408443,68      "trainDocuments": 19189,69      "trainTokens": 24971862,70      "validationDocuments": 605,71      "validationTokens": 1006374,72      "testDocuments": 405,73      "testTokens": 43020774    },75    "github": {76      "source": "github",77      "candidateDocuments": 892854,78      "candidateTokens": 1601439497,79      "trainDocuments": 848211,80      "trainTokens": 1520703921,81      "validationDocuments": 26785,82      "validationTokens": 49005031,83      "testDocuments": 17858,84      "testTokens": 3173054585    },86    "reddit": {87      "source": "reddit",88      "candidateDocuments": 398618,89      "candidateTokens": 77913082,90      "trainDocuments": 378687,91      "trainTokens": 74015899,92      "validationDocuments": 11958,93      "validationTokens": 2331493,94      "testDocuments": 7973,95      "testTokens": 156569096    },97    "stackoverflow": {98      "source": "stackoverflow",99      "candidateDocuments": 221380,100      "candidateTokens": 321268514,101      "trainDocuments": 210311,102      "trainTokens": 305090587,103      "validationDocuments": 6641,104      "validationTokens": 9675516,105      "testDocuments": 4428,106      "testTokens": 6502411107    },108    "wikipedia_dump": {109      "source": "wikipedia_dump",110      "candidateDocuments": 172170,111      "candidateTokens": 336186528,112      "trainDocuments": 163561,113      "trainTokens": 319156208,114      "validationDocuments": 5165,115      "validationTokens": 10407312,116      "testDocuments": 3444,117      "testTokens": 6623008118    }119  },120  "generatedFiles": [121    "output/split/train/books.jsonl",122    "output/split/validation/books.jsonl",123    "output/split/test/books.jsonl",124    "output/split/train/commoncrawl.jsonl",125    "output/split/validation/commoncrawl.jsonl",126    "output/split/test/commoncrawl.jsonl",127    "output/split/train/cp.jsonl",128    "output/split/validation/cp.jsonl",129    "output/split/test/cp.jsonl",130    "output/split/train/devdocs.jsonl",131    "output/split/validation/devdocs.jsonl",132    "output/split/test/devdocs.jsonl",133    "output/split/train/documentation.jsonl",134    "output/split/validation/documentation.jsonl",135    "output/split/test/documentation.jsonl",136    "output/split/train/github.jsonl",137    "output/split/validation/github.jsonl",138    "output/split/test/github.jsonl",139    "output/split/train/reddit.jsonl",140    "output/split/validation/reddit.jsonl",141    "output/split/test/reddit.jsonl",142    "output/split/train/stackoverflow.jsonl",143    "output/split/validation/stackoverflow.jsonl",144    "output/split/test/stackoverflow.jsonl",145    "output/split/train/wikipedia_dump.jsonl",146    "output/split/validation/wikipedia_dump.jsonl",147    "output/split/test/wikipedia_dump.jsonl"148  ]149}