ruediste/codeparrot-github-code-10G

1---2dataset_info:3  - config_name: bat4    features:5      - name: code6        dtype: string7      - name: repo_name8        dtype: string9      - name: path10        dtype: string11      - name: language12        dtype: string13      - name: license14        dtype: string15      - name: size16        dtype: int3217    splits:18      - name: train19        num_bytes: 77660605620        num_examples: 23677521    download_size: 22410848922    dataset_size: 77660605623  - config_name: c24    features:25      - name: code26        dtype: string27      - name: repo_name28        dtype: string29      - name: path30        dtype: string31      - name: language32        dtype: string33      - name: license34        dtype: string35      - name: size36        dtype: int3237    splits:38      - name: train39        num_bytes: 1081134912540        num_examples: 76379741    download_size: 383835274742    dataset_size: 1081134912543  - config_name: cmake44    features:45      - name: code46        dtype: string47      - name: repo_name48        dtype: string49      - name: path50        dtype: string51      - name: language52        dtype: string53      - name: license54        dtype: string55      - name: size56        dtype: int3257    splits:58      - name: train59        num_bytes: 60114991760        num_examples: 17527561    download_size: 21778959862    dataset_size: 60114991763  - config_name: cpp64    features:65      - name: code66        dtype: string67      - name: repo_name68        dtype: string69      - name: path70        dtype: string71      - name: language72        dtype: string73      - name: license74        dtype: string75      - name: size76        dtype: int3277    splits:78      - name: train79        num_bytes: 1082031170180        num_examples: 84145981    download_size: 368698174782    dataset_size: 1082031170183  - config_name: cs84    features:85      - name: code86        dtype: string87      - name: repo_name88        dtype: string89      - name: path90        dtype: string91      - name: language92        dtype: string93      - name: license94        dtype: string95      - name: size96        dtype: int3297    splits:98      - name: train99        num_bytes: 10943819320100        num_examples: 1848161101    download_size: 3120336752102    dataset_size: 10943819320103  - config_name: css104    features:105      - name: code106        dtype: string107      - name: repo_name108        dtype: string109      - name: path110        dtype: string111      - name: language112        dtype: string113      - name: license114        dtype: string115      - name: size116        dtype: int32117    splits:118      - name: train119        num_bytes: 10810494864120        num_examples: 773883121    download_size: 2812801293122    dataset_size: 10810494864123  - config_name: dockerfile124    features:125      - name: code126        dtype: string127      - name: repo_name128        dtype: string129      - name: path130        dtype: string131      - name: language132        dtype: string133      - name: license134        dtype: string135      - name: size136        dtype: int32137    splits:138      - name: train139        num_bytes: 802019049140        num_examples: 365925141    download_size: 217263689142    dataset_size: 802019049143  - config_name: f144    features:145      - name: code146        dtype: string147      - name: repo_name148        dtype: string149      - name: path150        dtype: string151      - name: language152        dtype: string153      - name: license154        dtype: string155      - name: size156        dtype: int32157    splits:158      - name: train159        num_bytes: 1751209693160        num_examples: 141450161    download_size: 586553383162    dataset_size: 1751209693163  - config_name: go164    features:165      - name: code166        dtype: string167      - name: repo_name168        dtype: string169      - name: path170        dtype: string171      - name: language172        dtype: string173      - name: license174        dtype: string175      - name: size176        dtype: int32177    splits:178      - name: train179        num_bytes: 10844686994180        num_examples: 1177701181    download_size: 3533865959182    dataset_size: 10844686994183  - config_name: hs184    features:185      - name: code186        dtype: string187      - name: repo_name188        dtype: string189      - name: path190        dtype: string191      - name: language192        dtype: string193      - name: license194        dtype: string195      - name: size196        dtype: int32197    splits:198      - name: train199        num_bytes: 2017953331200        num_examples: 340300201    download_size: 800910590202    dataset_size: 2017953331203  - config_name: html204    features:205      - name: code206        dtype: string207      - name: repo_name208        dtype: string209      - name: path210        dtype: string211      - name: language212        dtype: string213      - name: license214        dtype: string215      - name: size216        dtype: int32217    splits:218      - name: train219        num_bytes: 10859691221220        num_examples: 950145221    download_size: 2911365108222    dataset_size: 10859691221223  - config_name: java224    features:225      - name: code226        dtype: string227      - name: repo_name228        dtype: string229      - name: path230        dtype: string231      - name: language232        dtype: string233      - name: license234        dtype: string235      - name: size236        dtype: int32237    splits:238      - name: train239        num_bytes: 10984706703240        num_examples: 1816254241    download_size: 3543707243242    dataset_size: 10984706703243  - config_name: jl244    features:245      - name: code246        dtype: string247      - name: repo_name248        dtype: string249      - name: path250        dtype: string251      - name: language252        dtype: string253      - name: license254        dtype: string255      - name: size256        dtype: int32257    splits:258      - name: train259        num_bytes: 310477024260        num_examples: 57400261    download_size: 113926926262    dataset_size: 310477024263  - config_name: js264    features:265      - name: code266        dtype: string267      - name: repo_name268        dtype: string269      - name: path270        dtype: string271      - name: language272        dtype: string273      - name: license274        dtype: string275      - name: size276        dtype: int32277    splits:278      - name: train279        num_bytes: 10859135475280        num_examples: 1321191281    download_size: 3887494730282    dataset_size: 10859135475283  - config_name: lua284    features:285      - name: code286        dtype: string287      - name: repo_name288        dtype: string289      - name: path290        dtype: string291      - name: language292        dtype: string293      - name: license294        dtype: string295      - name: size296        dtype: int32297    splits:298      - name: train299        num_bytes: 3067620267300        num_examples: 578100301    download_size: 1100472588302    dataset_size: 3067620267303  - config_name: makefile304    features:305      - name: code306        dtype: string307      - name: repo_name308        dtype: string309      - name: path310        dtype: string311      - name: language312        dtype: string313      - name: license314        dtype: string315      - name: size316        dtype: int32317    splits:318      - name: train319        num_bytes: 3187259914320        num_examples: 678550321    download_size: 1207339362322    dataset_size: 3187259914323  - config_name: md324    features:325      - name: code326        dtype: string327      - name: repo_name328        dtype: string329      - name: path330        dtype: string331      - name: language332        dtype: string333      - name: license334        dtype: string335      - name: size336        dtype: int32337    splits:338      - name: train339        num_bytes: 11106099907340        num_examples: 3654282341    download_size: 5418909097342    dataset_size: 11106099907343  - config_name: perl344    features:345      - name: code346        dtype: string347      - name: repo_name348        dtype: string349      - name: path350        dtype: string351      - name: language352        dtype: string353      - name: license354        dtype: string355      - name: size356        dtype: int32357    splits:358      - name: train359        num_bytes: 5092484681360        num_examples: 497125361    download_size: 1971626131362    dataset_size: 5092484681363  - config_name: php364    features:365      - name: code366        dtype: string367      - name: repo_name368        dtype: string369      - name: path370        dtype: string371      - name: language372        dtype: string373      - name: license374        dtype: string375      - name: size376        dtype: int32377    splits:378      - name: train379        num_bytes: 10923821552380        num_examples: 1803703381    download_size: 3855239888382    dataset_size: 10923821552383  - config_name: ps1384    features:385      - name: code386        dtype: string387      - name: repo_name388        dtype: string389      - name: path390        dtype: string391      - name: language392        dtype: string393      - name: license394        dtype: string395      - name: size396        dtype: int32397    splits:398      - name: train399        num_bytes: 751671158400        num_examples: 136325401    download_size: 266061632402    dataset_size: 751671158403  - config_name: py404    features:405      - name: code406        dtype: string407      - name: repo_name408        dtype: string409      - name: path410        dtype: string411      - name: language412        dtype: string413      - name: license414        dtype: string415      - name: size416        dtype: int32417    splits:418      - name: train419        num_bytes: 10860398080420        num_examples: 1389571421    download_size: 3880081226422    dataset_size: 10860398080423  - config_name: rb424    features:425      - name: code426        dtype: string427      - name: repo_name428        dtype: string429      - name: path430        dtype: string431      - name: language432        dtype: string433      - name: license434        dtype: string435      - name: size436        dtype: int32437    splits:438      - name: train439        num_bytes: 11107135154440        num_examples: 4084613441    download_size: 4250173287442    dataset_size: 11107135154443  - config_name: rs444    features:445      - name: code446        dtype: string447      - name: repo_name448        dtype: string449      - name: path450        dtype: string451      - name: language452        dtype: string453      - name: license454        dtype: string455      - name: size456        dtype: int32457    splits:458      - name: train459        num_bytes: 2895512507460        num_examples: 321850461    download_size: 958277948462    dataset_size: 2895512507463  - config_name: scala464    features:465      - name: code466        dtype: string467      - name: repo_name468        dtype: string469      - name: path470        dtype: string471      - name: language472        dtype: string473      - name: license474        dtype: string475      - name: size476        dtype: int32477    splits:478      - name: train479        num_bytes: 4253564514480        num_examples: 835375481    download_size: 1543815679482    dataset_size: 4253564514483  - config_name: sh484    features:485      - name: code486        dtype: string487      - name: repo_name488        dtype: string489      - name: path490        dtype: string491      - name: language492        dtype: string493      - name: license494        dtype: string495      - name: size496        dtype: int32497    splits:498      - name: train499        num_bytes: 3345439162500        num_examples: 1384775501    download_size: 1435265806502    dataset_size: 3345439162503  - config_name: sql504    features:505      - name: code506        dtype: string507      - name: repo_name508        dtype: string509      - name: path510        dtype: string511      - name: language512        dtype: string513      - name: license514        dtype: string515      - name: size516        dtype: int32517    splits:518      - name: train519        num_bytes: 6160648946520        num_examples: 656000521    download_size: 1614855561522    dataset_size: 6160648946523  - config_name: tex524    features:525      - name: code526        dtype: string527      - name: repo_name528        dtype: string529      - name: path530        dtype: string531      - name: language532        dtype: string533      - name: license534        dtype: string535      - name: size536        dtype: int32537    splits:538      - name: train539        num_bytes: 2332592780540        num_examples: 250100541    download_size: 1014672343542    dataset_size: 2332592780543  - config_name: ts544    features:545      - name: code546        dtype: string547      - name: repo_name548        dtype: string549      - name: path550        dtype: string551      - name: language552        dtype: string553      - name: license554        dtype: string555      - name: size556        dtype: int32557    splits:558      - name: train559        num_bytes: 10813959394560        num_examples: 793033561    download_size: 2913351624562    dataset_size: 10813959394563  - config_name: vb564    features:565      - name: code566        dtype: string567      - name: repo_name568        dtype: string569      - name: path570        dtype: string571      - name: language572        dtype: string573      - name: license574        dtype: string575      - name: size576        dtype: int32577    splits:578      - name: train579        num_bytes: 2055862838580        num_examples: 154775581    download_size: 507645412582    dataset_size: 2055862838583configs:584  - config_name: bat585    data_files:586      - split: train587        path: bat/train-*588  - config_name: c589    data_files:590      - split: train591        path: c/train-*592  - config_name: cmake593    data_files:594      - split: train595        path: cmake/train-*596  - config_name: cpp597    data_files:598      - split: train599        path: cpp/train-*600  - config_name: cs601    data_files:602      - split: train603        path: cs/train-*604  - config_name: css605    data_files:606      - split: train607        path: css/train-*608  - config_name: dockerfile609    data_files:610      - split: train611        path: dockerfile/train-*612  - config_name: f613    data_files:614      - split: train615        path: f/train-*616  - config_name: go617    data_files:618      - split: train619        path: go/train-*620  - config_name: hs621    data_files:622      - split: train623        path: hs/train-*624  - config_name: html625    data_files:626      - split: train627        path: html/train-*628  - config_name: java629    data_files:630      - split: train631        path: java/train-*632  - config_name: jl633    data_files:634      - split: train635        path: jl/train-*636  - config_name: js637    data_files:638      - split: train639        path: js/train-*640  - config_name: lua641    data_files:642      - split: train643        path: lua/train-*644  - config_name: makefile645    data_files:646      - split: train647        path: makefile/train-*648  - config_name: md649    data_files:650      - split: train651        path: md/train-*652  - config_name: perl653    data_files:654      - split: train655        path: perl/train-*656  - config_name: php657    data_files:658      - split: train659        path: php/train-*660  - config_name: ps1661    data_files:662      - split: train663        path: ps1/train-*664  - config_name: py665    data_files:666      - split: train667        path: py/train-*668  - config_name: rb669    data_files:670      - split: train671        path: rb/train-*672  - config_name: rs673    data_files:674      - split: train675        path: rs/train-*676  - config_name: scala677    data_files:678      - split: train679        path: scala/train-*680  - config_name: sh681    data_files:682      - split: train683        path: sh/train-*684  - config_name: sql685    data_files:686      - split: train687        path: sql/train-*688  - config_name: tex689    data_files:690      - split: train691        path: tex/train-*692  - config_name: ts693    data_files:694      - split: train695        path: ts/train-*696  - config_name: vb697    data_files:698      - split: train699        path: vb/train-*700---701 702This is data is derived from the [Codeparrot Dataset](https://huggingface.co/datasets/codeparrot/github-code) by taking the first 10GB of text from each language, and splitting it into individual configs. This results in a download size of about 3GB per language.703 704Sample usage:705 706```python707from datasets import load_dataset708dataset = load_dataset("ruediste/codeparrot-github-code-10G", "java")709```710 711List of Languages:712 713```714languages = {715    'HTML': 'html',716    'Java': 'java',717    'JavaScript': 'js',718    'CSS': 'css',719    'C#': 'cs',720    'TypeScript': 'ts',721    "Batchfile": "bat",722    "C": 'c',723    "C++": 'cpp',724    "CMake": "cmake",725    "Dockerfile": "dockerfile",726    "FORTRAN": 'f',727    "GO": "go",728    "Haskell": "hs",729    "Julia": "jl",730    "Lua": "lua",731    "Makefile": "makefile",732    "Markdown": "md",733    "PHP": "php",734    "Perl": "perl",735    "PowerShell": 'ps1',736    "Python": "py",737    "Ruby": "rb",738    "Rust": "rs",739    "SQL": "sql",740    "Scala": "scala",741    "Shell": "sh",742    "TeX": "tex",743    "Visual Basic": "vb"744}745```746 747Please note that assembly to lost in the conversion.748