enigmare/v2-crawler

1{2  "documentsProcessed": 2321735,3  "trainDocuments": 2205645,4  "validationDocuments": 69648,5  "testDocuments": 46442,6  "sources": {7    "books": {8      "source": "books",9      "candidateDocuments": 1474,10      "candidateTokens": 185262788,11      "trainDocuments": 1400,12      "trainTokens": 176292558,13      "validationDocuments": 44,14      "validationTokens": 5203232,15      "testDocuments": 30,16      "testTokens": 376699817    },18    "commoncrawl": {19      "source": "commoncrawl",20      "candidateDocuments": 372505,21      "candidateTokens": 540883137,22      "trainDocuments": 353879,23      "trainTokens": 513930403,24      "validationDocuments": 11175,25      "validationTokens": 16431670,26      "testDocuments": 7451,27      "testTokens": 1052106428    },29    "cp": {30      "source": "cp",31      "candidateDocuments": 67889,32      "candidateTokens": 49075577,33      "trainDocuments": 64494,34      "trainTokens": 46644485,35      "validationDocuments": 2036,36      "validationTokens": 1513008,37      "testDocuments": 1359,38      "testTokens": 91808439    },40    "devdocs": {41      "source": "devdocs",42      "candidateDocuments": 174646,43      "candidateTokens": 198592155,44      "trainDocuments": 165913,45      "trainTokens": 188795573,46      "validationDocuments": 5239,47      "validationTokens": 5827863,48      "testDocuments": 3494,49      "testTokens": 396871950    },51    "documentation": {52      "source": "documentation",53      "candidateDocuments": 20199,54      "candidateTokens": 26408443,55      "trainDocuments": 19189,56      "trainTokens": 24971862,57      "validationDocuments": 605,58      "validationTokens": 1006374,59      "testDocuments": 405,60      "testTokens": 43020761    },62    "github": {63      "source": "github",64      "candidateDocuments": 892854,65      "candidateTokens": 1601439497,66      "trainDocuments": 848211,67      "trainTokens": 1520703921,68      "validationDocuments": 26785,69      "validationTokens": 49005031,70      "testDocuments": 17858,71      "testTokens": 3173054572    },73    "reddit": {74      "source": "reddit",75      "candidateDocuments": 398618,76      "candidateTokens": 77913082,77      "trainDocuments": 378687,78      "trainTokens": 74015899,79      "validationDocuments": 11958,80      "validationTokens": 2331493,81      "testDocuments": 7973,82      "testTokens": 156569083    },84    "stackoverflow": {85      "source": "stackoverflow",86      "candidateDocuments": 221380,87      "candidateTokens": 321268514,88      "trainDocuments": 210311,89      "trainTokens": 305090587,90      "validationDocuments": 6641,91      "validationTokens": 9675516,92      "testDocuments": 4428,93      "testTokens": 650241194    },95    "wikipedia_dump": {96      "source": "wikipedia_dump",97      "candidateDocuments": 172170,98      "candidateTokens": 336186528,99      "trainDocuments": 163561,100      "trainTokens": 319156208,101      "validationDocuments": 5165,102      "validationTokens": 10407312,103      "testDocuments": 3444,104      "testTokens": 6623008105    }106  },107  "elapsedTimeMs": 475581,108  "warnings": [],109  "errors": []110}