ruediste/codeparrot-github-code-10G
This is data is derived from the Codeparrot Dataset by taking the first 10GB of text from each language, and splitting it into individual configs. This results in a download size of about 3GB per language. Sample usage: from datasets import load_dataset dataset = load_dataset("ruediste/codeparrot-github-code-10G", "java") List of Languages: languages = { 'HTML': 'html', 'Java': 'java', 'JavaScript': 'js', 'CSS': 'css', 'C#': 'cs', 'TypeScript': 'ts', "Batchfile":… See the full description on the dataset page: https://huggingface.co/datasets/ruediste/codeparrot-github-code-10G.
22.6k
1{2 "cells": [3 {4 "cell_type": "code",5 "execution_count": null,6 "metadata": {7 "id": "-pcSZJqCB8qn"8 },9 "outputs": [],10 "source": [11 "!pip install huggingface_hub datasets"12 ]13 },14 {15 "cell_type": "code",16 "execution_count": 2,17 "metadata": {18 "executionInfo": {19 "elapsed": 4264,20 "status": "ok",21 "timestamp": 1736020137690,22 "user": {23 "displayName": "Ruedi Steinmann",24 "userId": "06668216725139701947"25 },26 "user_tz": -6027 },28 "id": "M098ur2wEBkj"29 },30 "outputs": [],31 "source": [32 "from datasets import load_dataset\n",33 "\n",34 "ds = load_dataset(\"codeparrot/github-code\", split=\"train\", streaming=True,trust_remote_code=True)"35 ]36 },37 {38 "cell_type": "code",39 "execution_count": null,40 "metadata": {41 "colab": {42 "base_uri": "https://localhost:8080/"43 },44 "executionInfo": {45 "elapsed": 492,46 "status": "ok",47 "timestamp": 1736013861782,48 "user": {49 "displayName": "Ruedi Steinmann",50 "userId": "06668216725139701947"51 },52 "user_tz": -6053 },54 "id": "Hqn_Jn04fONN",55 "outputId": "f86724d0-e4c3-4d89-cd93-10888b2c8712"56 },57 "outputs": [],58 "source": [59 "languagesOrig={\n",60 " \"Assembly\": [\".asm\"],\n",61 " \"Batchfile\": [\".bat\", \".cmd\"],\n",62 " \"C\": [\".c\", \".h\"],\n",63 " \"C#\": [\".cs\"],\n",64 " \"C++\": [\".cpp\", \".hpp\", \".c++\", \".h++\", \".cc\", \".hh\", \".C\", \".H\"],\n",65 " \"CMake\": [\".cmake\"],\n",66 " \"CSS\": [\".css\"],\n",67 " \"Dockerfile\": [\".dockerfile\", \"Dockerfile\"],\n",68 " \"FORTRAN\": ['.f90', '.f', '.f03', '.f08', '.f77', '.f95', '.for', '.fpp'],\n",69 " \"GO\": [\".go\"],\n",70 " \"Haskell\": [\".hs\"],\n",71 " \"HTML\":[\".html\"],\n",72 " \"Java\": [\".java\"],\n",73 " \"JavaScript\": [\".js\"],\n",74 " \"Julia\": [\".jl\"],\n",75 " \"Lua\": [\".lua\"],\n",76 " \"Makefile\": [\"Makefile\"],\n",77 " \"Markdown\": [\".md\", \".markdown\"],\n",78 " \"PHP\": [\".php\", \".php3\", \".php4\", \".php5\", \".phps\", \".phpt\"],\n",79 " \"Perl\": [\".pl\", \".pm\", \".pod\", \".perl\"],\n",80 " \"PowerShell\": ['.ps1', '.psd1', '.psm1'],\n",81 " \"Python\": [\".py\"],\n",82 " \"Ruby\": [\".rb\"],\n",83 " \"Rust\": [\".rs\"],\n",84 " \"SQL\": [\".sql\"],\n",85 " \"Scala\": [\".scala\"],\n",86 " \"Shell\": [\".sh\", \".bash\", \".command\", \".zsh\"],\n",87 " \"TypeScript\": [\".ts\", \".tsx\"],\n",88 " \"TeX\": [\".tex\"],\n",89 " \"Visual Basic\": [\".vb\"]\n",90 "};"91 ]92 },93 {94 "cell_type": "code",95 "execution_count": null,96 "metadata": {97 "colab": {98 "base_uri": "https://localhost:8080/"99 },100 "executionInfo": {101 "elapsed": 385,102 "status": "ok",103 "timestamp": 1736020152638,104 "user": {105 "displayName": "Ruedi Steinmann",106 "userId": "06668216725139701947"107 },108 "user_tz": -60109 },110 "id": "kTOf2NnKkCUI",111 "outputId": "6dd96a1b-be4b-4c7e-c31c-e6ebde93a24b"112 },113 "outputs": [],114 "source": [115 "import pyarrow.parquet as pq\n",116 "import pyarrow as pa\n",117 "\n",118 "writerPerLanguage = {}\n",119 "remainingSizePerLanguage = {}\n",120 "batches = {}\n",121 "\n",122 "languages = {\n",123 " 'HTML': 'html',\n",124 " 'Java': 'java',\n",125 " 'JavaScript': 'js',\n",126 " 'CSS': 'css',\n",127 " 'C#': 'cs',\n",128 " 'TypeScript': 'ts',\n",129 " \"Assembly\": \"asm\",\n",130 " \"Batchfile\": \"bat\",\n",131 " \"C\": 'c',\n",132 " \"C++\": 'cpp',\n",133 " \"CMake\": \"cmake\",\n",134 " \"Dockerfile\": \"dockerfile\",\n",135 " \"FORTRAN\": 'f',\n",136 " \"GO\": \"go\",\n",137 " \"Haskell\": \"hs\",\n",138 " \"Julia\": \"jl\",\n",139 " \"Lua\": \"lua\",\n",140 " \"Makefile\": \"makefile\",\n",141 " \"Markdown\": \"md\",\n",142 " \"PHP\": \"php\",\n",143 " \"Perl\": \"perl\",\n",144 " \"PowerShell\": 'ps1',\n",145 " \"Python\": \"py\",\n",146 " \"Ruby\": \"rb\",\n",147 " \"Rust\": \"rs\",\n",148 " \"SQL\": \"sql\",\n",149 " \"Scala\": \"scala\",\n",150 " \"Shell\": \"sh\",\n",151 " \"TeX\": \"tex\",\n",152 " \"Visual Basic\": \"vb\"\n",153 "}\n",154 "\n",155 "batchSize=1024\n",156 "\n",157 "schema = pa.schema([\n",158 " pa.field(\"code\", pa.string()),\n",159 " pa.field(\"repo_name\", pa.string()),\n",160 " pa.field(\"path\", pa.string()),\n",161 " pa.field(\"language\", pa.string()),\n",162 " pa.field(\"license\", pa.string()),\n",163 " pa.field(\"size\", pa.int32())\n",164 "])\n",165 "for language in languages.keys():\n",166 " print('Creating writer for '+language)\n",167 " writerPerLanguage[language] = pq.ParquetWriter(languages[language]+'.parquet', schema=schema)\n",168 " remainingSizePerLanguage[language] = 10*1024*1024*1024;\n",169 " batches[language] = []"170 ]171 },172 {173 "cell_type": "code",174 "execution_count": null,175 "metadata": {176 "colab": {177 "base_uri": "https://localhost:8080/",178 "height": 391179 },180 "executionInfo": {181 "elapsed": 144963,182 "status": "error",183 "timestamp": 1736020917723,184 "user": {185 "displayName": "Ruedi Steinmann",186 "userId": "06668216725139701947"187 },188 "user_tz": -60189 },190 "id": "9MyaoY5uljr2",191 "outputId": "f977b9d5-2aac-42bc-90fa-655cbc93d26a"192 },193 "outputs": [],194 "source": [195 "for row in ds:\n",196 " language=row[\"language\"]\n",197 " if language in writerPerLanguage.keys():\n",198 " batch=batches[language]\n",199 " row[\"language\"]=languages[language]\n",200 " batch.append(row)\n",201 " remainingSizePerLanguage[language] -= row[\"size\"]\n",202 " if len(batch)>batchSize:\n",203 " writerPerLanguage[language].write_batch(pa.RecordBatch.from_pylist(batch, schema=schema))\n",204 " batches[language] = []\n",205 " batch=[]\n",206 " if remainingSizePerLanguage[language] < 0:\n",207 " print('completed '+language)\n",208 " writerPerLanguage[language].write_batch(pa.RecordBatch.from_pylist(batch, schema=schema))\n",209 " writerPerLanguage[language].close()\n",210 " del writerPerLanguage[language]\n",211 " del remainingSizePerLanguage[language]\n",212 " del batches[language]\n",213 " if len(writerPerLanguage)==0:\n",214 " break\n",215 "\n",216 "for language in writerPerLanguage.keys():\n",217 " batch=batches[language]\n",218 " writerPerLanguage[language].write_batch(pa.RecordBatch.from_pylist(batch, schema=schema))\n",219 " writerPerLanguage[language].close()"220 ]221 },222 {223 "cell_type": "code",224 "execution_count": null,225 "metadata": {226 "colab": {227 "base_uri": "https://localhost:8080/",228 "height": 130,229 "referenced_widgets": [230 "2f72ab1685684a0d91c7ea4f84fb5f15",231 "912d2bffbbf54e86bad077425f530609",232 "a8c8bdfa8caa42cb850f2c02fa861edc",233 "f737a383374843f28042865ceafb4a5e",234 "65f72fa2c0d84125be5356fe003cb9ee",235 "5c2e51261f5d418db3232b4bd6c8ef90",236 "36377f4f84e448fe9eca7f1b4e157a5d",237 "1d5915d794634d0589429c512caea30b",238 "d0ef36296eae47beb753ce63ef9dcf77",239 "58908d2094e84d64b20da8eaa22b3fee",240 "b8187001b40a41ad88819c0399106383",241 "7a98af6df3ae494fa68ae5ff44c0b83e",242 "7fee6dd431804efa9b8c4d1507367ac3",243 "497397fabc834673b4c72f2892db289e",244 "df470c6b791d4e429cd27a4b3016d737",245 "7635a39c2dc64e86b90eb5bfa930e7a8",246 "fac0fe0e91b9458593246ca8f2fa672d",247 "7b9abe4c126c47308ef872736019996c",248 "cbdb2d6998804a408d28a62680386a4e",249 "e4c92db03df34d0489a426282dfb378d",250 "da20c99456e54d16b9140000231dc050",251 "9b7fe26167a245658a13442a6a866e6f",252 "547f8629aaa3474c99d267568078c492",253 "ef6fa9adeff84619a2f226d1e5ed6530",254 "af79814543784c78b933383350092a36",255 "004d7a6197da49fe96a83d4613868dad",256 "eff2a84ca9854e5696dabf55286888c7",257 "04bc51efe2734c308109e9c84d12be0a",258 "e4b11afe53034934992cec2fdd953c0f",259 "17d56a812c7a4051a2de3d15b7ba9269",260 "a823a29929c64431b6460f3600736b23",261 "7051cb0213d7446c8aa22cace52b6067",262 "c771caf9a37e4b4a8a6618f1c1598370"263 ]264 },265 "executionInfo": {266 "elapsed": 7372,267 "status": "ok",268 "timestamp": 1736019998026,269 "user": {270 "displayName": "Ruedi Steinmann",271 "userId": "06668216725139701947"272 },273 "user_tz": -60274 },275 "id": "8Nf35EXSz1lO",276 "outputId": "205204d7-5034-4b7b-f71c-dcef59c86b06"277 },278 "outputs": [],279 "source": [280 "from datasets import load_dataset\n",281 "for language in languages.keys():\n",282 " print('Uploading '+language)\n",283 " langDs=load_dataset('parquet',data_files=[languages[language]+'.parquet'])\n",284 " langDs.push_to_hub('ruediste/codeparrot-github-code',languages[language])"285 ]286 }287 ],288 "metadata": {289 "colab": {290 "authorship_tag": "ABX9TyMcMRF4yH51aVZTSSQ4uMuD",291 "provenance": []292 },293 "kernelspec": {294 "display_name": "Python 3 (ipykernel)",295 "language": "python",296 "name": "python3"297 },298 "language_info": {299 "codemirror_mode": {300 "name": "ipython",301 "version": 3302 },303 "file_extension": ".py",304 "mimetype": "text/x-python",305 "name": "python",306 "nbconvert_exporter": "python",307 "pygments_lexer": "ipython3",308 "version": "3.12.3"309 },310 "widgets": {311 "application/vnd.jupyter.widget-state+json": {312 "004d7a6197da49fe96a83d4613868dad": {313 "model_module": "@jupyter-widgets/controls",314 "model_module_version": "1.5.0",315 "model_name": "HTMLModel",316 "state": {317 "_dom_classes": [],318 "_model_module": "@jupyter-widgets/controls",319 "_model_module_version": "1.5.0",320 "_model_name": "HTMLModel",321 "_view_count": null,322 "_view_module": "@jupyter-widgets/controls",323 "_view_module_version": "1.5.0",324 "_view_name": "HTMLView",325 "description": "",326 "description_tooltip": null,327 "layout": "IPY_MODEL_7051cb0213d7446c8aa22cace52b6067",328 "placeholder": "",329 "style": "IPY_MODEL_c771caf9a37e4b4a8a6618f1c1598370",330 "value": " 70.0/70.0 [00:00<00:00, 4.21kB/s]"331 }332 },333 "04bc51efe2734c308109e9c84d12be0a": {334 "model_module": "@jupyter-widgets/base",335 "model_module_version": "1.2.0",336 "model_name": "LayoutModel",337 "state": {338 "_model_module": "@jupyter-widgets/base",339 "_model_module_version": "1.2.0",340 "_model_name": "LayoutModel",341 "_view_count": null,342 "_view_module": "@jupyter-widgets/base",343 "_view_module_version": "1.2.0",344 "_view_name": "LayoutView",345 "align_content": null,346 "align_items": null,347 "align_self": null,348 "border": null,349 "bottom": null,350 "display": null,351 "flex": null,352 "flex_flow": null,353 "grid_area": null,354 "grid_auto_columns": null,355 "grid_auto_flow": null,356 "grid_auto_rows": null,357 "grid_column": null,358 "grid_gap": null,359 "grid_row": null,360 "grid_template_areas": null,361 "grid_template_columns": null,362 "grid_template_rows": null,363 "height": null,364 "justify_content": null,365 "justify_items": null,366 "left": null,367 "margin": null,368 "max_height": null,369 "max_width": null,370 "min_height": null,371 "min_width": null,372 "object_fit": null,373 "object_position": null,374 "order": null,375 "overflow": null,376 "overflow_x": null,377 "overflow_y": null,378 "padding": null,379 "right": null,380 "top": null,381 "visibility": null,382 "width": null383 }384 },385 "17d56a812c7a4051a2de3d15b7ba9269": {386 "model_module": "@jupyter-widgets/base",387 "model_module_version": "1.2.0",388 "model_name": "LayoutModel",389 "state": {390 "_model_module": "@jupyter-widgets/base",391 "_model_module_version": "1.2.0",392 "_model_name": "LayoutModel",393 "_view_count": null,394 "_view_module": "@jupyter-widgets/base",395 "_view_module_version": "1.2.0",396 "_view_name": "LayoutView",397 "align_content": null,398 "align_items": null,399 "align_self": null,400 "border": null,401 "bottom": null,402 "display": null,403 "flex": null,404 "flex_flow": null,405 "grid_area": null,406 "grid_auto_columns": null,407 "grid_auto_flow": null,408 "grid_auto_rows": null,409 "grid_column": null,410 "grid_gap": null,411 "grid_row": null,412 "grid_template_areas": null,413 "grid_template_columns": null,414 "grid_template_rows": null,415 "height": null,416 "justify_content": null,417 "justify_items": null,418 "left": null,419 "margin": null,420 "max_height": null,421 "max_width": null,422 "min_height": null,423 "min_width": null,424 "object_fit": null,425 "object_position": null,426 "order": null,427 "overflow": null,428 "overflow_x": null,429 "overflow_y": null,430 "padding": null,431 "right": null,432 "top": null,433 "visibility": null,434 "width": null435 }436 },437 "1d5915d794634d0589429c512caea30b": {438 "model_module": "@jupyter-widgets/base",439 "model_module_version": "1.2.0",440 "model_name": "LayoutModel",441 "state": {442 "_model_module": "@jupyter-widgets/base",443 "_model_module_version": "1.2.0",444 "_model_name": "LayoutModel",445 "_view_count": null,446 "_view_module": "@jupyter-widgets/base",447 "_view_module_version": "1.2.0",448 "_view_name": "LayoutView",449 "align_content": null,450 "align_items": null,451 "align_self": null,452 "border": null,453 "bottom": null,454 "display": null,455 "flex": null,456 "flex_flow": null,457 "grid_area": null,458 "grid_auto_columns": null,459 "grid_auto_flow": null,460 "grid_auto_rows": null,461 "grid_column": null,462 "grid_gap": null,463 "grid_row": null,464 "grid_template_areas": null,465 "grid_template_columns": null,466 "grid_template_rows": null,467 "height": null,468 "justify_content": null,469 "justify_items": null,470 "left": null,471 "margin": null,472 "max_height": null,473 "max_width": null,474 "min_height": null,475 "min_width": null,476 "object_fit": null,477 "object_position": null,478 "order": null,479 "overflow": null,480 "overflow_x": null,481 "overflow_y": null,482 "padding": null,483 "right": null,484 "top": null,485 "visibility": null,486 "width": null487 }488 },489 "2f72ab1685684a0d91c7ea4f84fb5f15": {490 "model_module": "@jupyter-widgets/controls",491 "model_module_version": "1.5.0",492 "model_name": "HBoxModel",493 "state": {494 "_dom_classes": [],495 "_model_module": "@jupyter-widgets/controls",496 "_model_module_version": "1.5.0",497 "_model_name": "HBoxModel",498 "_view_count": null,499 "_view_module": "@jupyter-widgets/controls",500 "_view_module_version": "1.5.0",501 "_view_name": "HBoxView",502 "box_style": "",503 "children": [504 "IPY_MODEL_912d2bffbbf54e86bad077425f530609",505 "IPY_MODEL_a8c8bdfa8caa42cb850f2c02fa861edc",506 "IPY_MODEL_f737a383374843f28042865ceafb4a5e"507 ],508 "layout": "IPY_MODEL_65f72fa2c0d84125be5356fe003cb9ee"509 }510 },511 "36377f4f84e448fe9eca7f1b4e157a5d": {512 "model_module": "@jupyter-widgets/controls",513 "model_module_version": "1.5.0",514 "model_name": "DescriptionStyleModel",515 "state": {516 "_model_module": "@jupyter-widgets/controls",517 "_model_module_version": "1.5.0",518 "_model_name": "DescriptionStyleModel",519 "_view_count": null,520 "_view_module": "@jupyter-widgets/base",521 "_view_module_version": "1.2.0",522 "_view_name": "StyleView",523 "description_width": ""524 }525 },526 "497397fabc834673b4c72f2892db289e": {527 "model_module": "@jupyter-widgets/controls",528 "model_module_version": "1.5.0",529 "model_name": "FloatProgressModel",530 "state": {531 "_dom_classes": [],532 "_model_module": "@jupyter-widgets/controls",533 "_model_module_version": "1.5.0",534 "_model_name": "FloatProgressModel",535 "_view_count": null,536 "_view_module": "@jupyter-widgets/controls",537 "_view_module_version": "1.5.0",538 "_view_name": "ProgressView",539 "bar_style": "success",540 "description": "",541 "description_tooltip": null,542 "layout": "IPY_MODEL_cbdb2d6998804a408d28a62680386a4e",543 "max": 1,544 "min": 0,545 "orientation": "horizontal",546 "style": "IPY_MODEL_e4c92db03df34d0489a426282dfb378d",547 "value": 1548 }549 },550 "547f8629aaa3474c99d267568078c492": {551 "model_module": "@jupyter-widgets/controls",552 "model_module_version": "1.5.0",553 "model_name": "HBoxModel",554 "state": {555 "_dom_classes": [],556 "_model_module": "@jupyter-widgets/controls",557 "_model_module_version": "1.5.0",558 "_model_name": "HBoxModel",559 "_view_count": null,560 "_view_module": "@jupyter-widgets/controls",561 "_view_module_version": "1.5.0",562 "_view_name": "HBoxView",563 "box_style": "",564 "children": [565 "IPY_MODEL_ef6fa9adeff84619a2f226d1e5ed6530",566 "IPY_MODEL_af79814543784c78b933383350092a36",567 "IPY_MODEL_004d7a6197da49fe96a83d4613868dad"568 ],569 "layout": "IPY_MODEL_eff2a84ca9854e5696dabf55286888c7"570 }571 },572 "58908d2094e84d64b20da8eaa22b3fee": {573 "model_module": "@jupyter-widgets/base",574 "model_module_version": "1.2.0",575 "model_name": "LayoutModel",576 "state": {577 "_model_module": "@jupyter-widgets/base",578 "_model_module_version": "1.2.0",579 "_model_name": "LayoutModel",580 "_view_count": null,581 "_view_module": "@jupyter-widgets/base",582 "_view_module_version": "1.2.0",583 "_view_name": "LayoutView",584 "align_content": null,585 "align_items": null,586 "align_self": null,587 "border": null,588 "bottom": null,589 "display": null,590 "flex": null,591 "flex_flow": null,592 "grid_area": null,593 "grid_auto_columns": null,594 "grid_auto_flow": null,595 "grid_auto_rows": null,596 "grid_column": null,597 "grid_gap": null,598 "grid_row": null,599 "grid_template_areas": null,600 "grid_template_columns": null,601 "grid_template_rows": null,602 "height": null,603 "justify_content": null,604 "justify_items": null,605 "left": null,606 "margin": null,607 "max_height": null,608 "max_width": null,609 "min_height": null,610 "min_width": null,611 "object_fit": null,612 "object_position": null,613 "order": null,614 "overflow": null,615 "overflow_x": null,616 "overflow_y": null,617 "padding": null,618 "right": null,619 "top": null,620 "visibility": null,621 "width": null622 }623 },624 "5c2e51261f5d418db3232b4bd6c8ef90": {625 "model_module": "@jupyter-widgets/base",626 "model_module_version": "1.2.0",627 "model_name": "LayoutModel",628 "state": {629 "_model_module": "@jupyter-widgets/base",630 "_model_module_version": "1.2.0",631 "_model_name": "LayoutModel",632 "_view_count": null,633 "_view_module": "@jupyter-widgets/base",634 "_view_module_version": "1.2.0",635 "_view_name": "LayoutView",636 "align_content": null,637 "align_items": null,638 "align_self": null,639 "border": null,640 "bottom": null,641 "display": null,642 "flex": null,643 "flex_flow": null,644 "grid_area": null,645 "grid_auto_columns": null,646 "grid_auto_flow": null,647 "grid_auto_rows": null,648 "grid_column": null,649 "grid_gap": null,650 "grid_row": null,651 "grid_template_areas": null,652 "grid_template_columns": null,653 "grid_template_rows": null,654 "height": null,655 "justify_content": null,656 "justify_items": null,657 "left": null,658 "margin": null,659 "max_height": null,660 "max_width": null,661 "min_height": null,662 "min_width": null,663 "object_fit": null,664 "object_position": null,665 "order": null,666 "overflow": null,667 "overflow_x": null,668 "overflow_y": null,669 "padding": null,670 "right": null,671 "top": null,672 "visibility": null,673 "width": null674 }675 },676 "65f72fa2c0d84125be5356fe003cb9ee": {677 "model_module": "@jupyter-widgets/base",678 "model_module_version": "1.2.0",679 "model_name": "LayoutModel",680 "state": {681 "_model_module": "@jupyter-widgets/base",682 "_model_module_version": "1.2.0",683 "_model_name": "LayoutModel",684 "_view_count": null,685 "_view_module": "@jupyter-widgets/base",686 "_view_module_version": "1.2.0",687 "_view_name": "LayoutView",688 "align_content": null,689 "align_items": null,690 "align_self": null,691 "border": null,692 "bottom": null,693 "display": null,694 "flex": null,695 "flex_flow": null,696 "grid_area": null,697 "grid_auto_columns": null,698 "grid_auto_flow": null,699 "grid_auto_rows": null,700 "grid_column": null,701 "grid_gap": null,702 "grid_row": null,703 "grid_template_areas": null,704 "grid_template_columns": null,705 "grid_template_rows": null,706 "height": null,707 "justify_content": null,708 "justify_items": null,709 "left": null,710 "margin": null,711 "max_height": null,712 "max_width": null,713 "min_height": null,714 "min_width": null,715 "object_fit": null,716 "object_position": null,717 "order": null,718 "overflow": null,719 "overflow_x": null,720 "overflow_y": null,721 "padding": null,722 "right": null,723 "top": null,724 "visibility": null,725 "width": null726 }727 },728 "7051cb0213d7446c8aa22cace52b6067": {729 "model_module": "@jupyter-widgets/base",730 "model_module_version": "1.2.0",731 "model_name": "LayoutModel",732 "state": {733 "_model_module": "@jupyter-widgets/base",734 "_model_module_version": "1.2.0",735 "_model_name": "LayoutModel",736 "_view_count": null,737 "_view_module": "@jupyter-widgets/base",738 "_view_module_version": "1.2.0",739 "_view_name": "LayoutView",740 "align_content": null,741 "align_items": null,742 "align_self": null,743 "border": null,744 "bottom": null,745 "display": null,746 "flex": null,747 "flex_flow": null,748 "grid_area": null,749 "grid_auto_columns": null,750 "grid_auto_flow": null,751 "grid_auto_rows": null,752 "grid_column": null,753 "grid_gap": null,754 "grid_row": null,755 "grid_template_areas": null,756 "grid_template_columns": null,757 "grid_template_rows": null,758 "height": null,759 "justify_content": null,760 "justify_items": null,761 "left": null,762 "margin": null,763 "max_height": null,764 "max_width": null,765 "min_height": null,766 "min_width": null,767 "object_fit": null,768 "object_position": null,769 "order": null,770 "overflow": null,771 "overflow_x": null,772 "overflow_y": null,773 "padding": null,774 "right": null,775 "top": null,776 "visibility": null,777 "width": null778 }779 },780 "7635a39c2dc64e86b90eb5bfa930e7a8": {781 "model_module": "@jupyter-widgets/base",782 "model_module_version": "1.2.0",783 "model_name": "LayoutModel",784 "state": {785 "_model_module": "@jupyter-widgets/base",786 "_model_module_version": "1.2.0",787 "_model_name": "LayoutModel",788 "_view_count": null,789 "_view_module": "@jupyter-widgets/base",790 "_view_module_version": "1.2.0",791 "_view_name": "LayoutView",792 "align_content": null,793 "align_items": null,794 "align_self": null,795 "border": null,796 "bottom": null,797 "display": null,798 "flex": null,799 "flex_flow": null,800 "grid_area": null,801 "grid_auto_columns": null,802 "grid_auto_flow": null,803 "grid_auto_rows": null,804 "grid_column": null,805 "grid_gap": null,806 "grid_row": null,807 "grid_template_areas": null,808 "grid_template_columns": null,809 "grid_template_rows": null,810 "height": null,811 "justify_content": null,812 "justify_items": null,813 "left": null,814 "margin": null,815 "max_height": null,816 "max_width": null,817 "min_height": null,818 "min_width": null,819 "object_fit": null,820 "object_position": null,821 "order": null,822 "overflow": null,823 "overflow_x": null,824 "overflow_y": null,825 "padding": null,826 "right": null,827 "top": null,828 "visibility": null,829 "width": null830 }831 },832 "7a98af6df3ae494fa68ae5ff44c0b83e": {833 "model_module": "@jupyter-widgets/controls",834 "model_module_version": "1.5.0",835 "model_name": "HBoxModel",836 "state": {837 "_dom_classes": [],838 "_model_module": "@jupyter-widgets/controls",839 "_model_module_version": "1.5.0",840 "_model_name": "HBoxModel",841 "_view_count": null,842 "_view_module": "@jupyter-widgets/controls",843 "_view_module_version": "1.5.0",844 "_view_name": "HBoxView",845 "box_style": "",846 "children": [847 "IPY_MODEL_7fee6dd431804efa9b8c4d1507367ac3",848 "IPY_MODEL_497397fabc834673b4c72f2892db289e",849 "IPY_MODEL_df470c6b791d4e429cd27a4b3016d737"850 ],851 "layout": "IPY_MODEL_7635a39c2dc64e86b90eb5bfa930e7a8"852 }853 },854 "7b9abe4c126c47308ef872736019996c": {855 "model_module": "@jupyter-widgets/controls",856 "model_module_version": "1.5.0",857 "model_name": "DescriptionStyleModel",858 "state": {859 "_model_module": "@jupyter-widgets/controls",860 "_model_module_version": "1.5.0",861 "_model_name": "DescriptionStyleModel",862 "_view_count": null,863 "_view_module": "@jupyter-widgets/base",864 "_view_module_version": "1.2.0",865 "_view_name": "StyleView",866 "description_width": ""867 }868 },869 "7fee6dd431804efa9b8c4d1507367ac3": {870 "model_module": "@jupyter-widgets/controls",871 "model_module_version": "1.5.0",872 "model_name": "HTMLModel",873 "state": {874 "_dom_classes": [],875 "_model_module": "@jupyter-widgets/controls",876 "_model_module_version": "1.5.0",877 "_model_name": "HTMLModel",878 "_view_count": null,879 "_view_module": "@jupyter-widgets/controls",880 "_view_module_version": "1.5.0",881 "_view_name": "HTMLView",882 "description": "",883 "description_tooltip": null,884 "layout": "IPY_MODEL_fac0fe0e91b9458593246ca8f2fa672d",885 "placeholder": "",886 "style": "IPY_MODEL_7b9abe4c126c47308ef872736019996c",887 "value": "Creating parquet from Arrow format: 100%"888 }889 },890 "912d2bffbbf54e86bad077425f530609": {891 "model_module": "@jupyter-widgets/controls",892 "model_module_version": "1.5.0",893 "model_name": "HTMLModel",894 "state": {895 "_dom_classes": [],896 "_model_module": "@jupyter-widgets/controls",897 "_model_module_version": "1.5.0",898 "_model_name": "HTMLModel",899 "_view_count": null,900 "_view_module": "@jupyter-widgets/controls",901 "_view_module_version": "1.5.0",902 "_view_name": "HTMLView",903 "description": "",904 "description_tooltip": null,905 "layout": "IPY_MODEL_5c2e51261f5d418db3232b4bd6c8ef90",906 "placeholder": "",907 "style": "IPY_MODEL_36377f4f84e448fe9eca7f1b4e157a5d",908 "value": "Uploading the dataset shards: 100%"909 }910 },911 "9b7fe26167a245658a13442a6a866e6f": {912 "model_module": "@jupyter-widgets/controls",913 "model_module_version": "1.5.0",914 "model_name": "DescriptionStyleModel",915 "state": {916 "_model_module": "@jupyter-widgets/controls",917 "_model_module_version": "1.5.0",918 "_model_name": "DescriptionStyleModel",919 "_view_count": null,920 "_view_module": "@jupyter-widgets/base",921 "_view_module_version": "1.2.0",922 "_view_name": "StyleView",923 "description_width": ""924 }925 },926 "a823a29929c64431b6460f3600736b23": {927 "model_module": "@jupyter-widgets/controls",928 "model_module_version": "1.5.0",929 "model_name": "ProgressStyleModel",930 "state": {931 "_model_module": "@jupyter-widgets/controls",932 "_model_module_version": "1.5.0",933 "_model_name": "ProgressStyleModel",934 "_view_count": null,935 "_view_module": "@jupyter-widgets/base",936 "_view_module_version": "1.2.0",937 "_view_name": "StyleView",938 "bar_color": null,939 "description_width": ""940 }941 },942 "a8c8bdfa8caa42cb850f2c02fa861edc": {943 "model_module": "@jupyter-widgets/controls",944 "model_module_version": "1.5.0",945 "model_name": "FloatProgressModel",946 "state": {947 "_dom_classes": [],948 "_model_module": "@jupyter-widgets/controls",949 "_model_module_version": "1.5.0",950 "_model_name": "FloatProgressModel",951 "_view_count": null,952 "_view_module": "@jupyter-widgets/controls",953 "_view_module_version": "1.5.0",954 "_view_name": "ProgressView",955 "bar_style": "success",956 "description": "",957 "description_tooltip": null,958 "layout": "IPY_MODEL_1d5915d794634d0589429c512caea30b",959 "max": 1,960 "min": 0,961 "orientation": "horizontal",962 "style": "IPY_MODEL_d0ef36296eae47beb753ce63ef9dcf77",963 "value": 1964 }965 },966 "af79814543784c78b933383350092a36": {967 "model_module": "@jupyter-widgets/controls",968 "model_module_version": "1.5.0",969 "model_name": "FloatProgressModel",970 "state": {971 "_dom_classes": [],972 "_model_module": "@jupyter-widgets/controls",973 "_model_module_version": "1.5.0",974 "_model_name": "FloatProgressModel",975 "_view_count": null,976 "_view_module": "@jupyter-widgets/controls",977 "_view_module_version": "1.5.0",978 "_view_name": "ProgressView",979 "bar_style": "success",980 "description": "",981 "description_tooltip": null,982 "layout": "IPY_MODEL_17d56a812c7a4051a2de3d15b7ba9269",983 "max": 70,984 "min": 0,985 "orientation": "horizontal",986 "style": "IPY_MODEL_a823a29929c64431b6460f3600736b23",987 "value": 70988 }989 },990 "b8187001b40a41ad88819c0399106383": {991 "model_module": "@jupyter-widgets/controls",992 "model_module_version": "1.5.0",993 "model_name": "DescriptionStyleModel",994 "state": {995 "_model_module": "@jupyter-widgets/controls",996 "_model_module_version": "1.5.0",997 "_model_name": "DescriptionStyleModel",998 "_view_count": null,999 "_view_module": "@jupyter-widgets/base",1000 "_view_module_version": "1.2.0",1001 "_view_name": "StyleView",1002 "description_width": ""1003 }1004 },1005 "c771caf9a37e4b4a8a6618f1c1598370": {1006 "model_module": "@jupyter-widgets/controls",1007 "model_module_version": "1.5.0",1008 "model_name": "DescriptionStyleModel",1009 "state": {1010 "_model_module": "@jupyter-widgets/controls",1011 "_model_module_version": "1.5.0",1012 "_model_name": "DescriptionStyleModel",1013 "_view_count": null,1014 "_view_module": "@jupyter-widgets/base",1015 "_view_module_version": "1.2.0",1016 "_view_name": "StyleView",1017 "description_width": ""1018 }1019 },1020 "cbdb2d6998804a408d28a62680386a4e": {1021 "model_module": "@jupyter-widgets/base",1022 "model_module_version": "1.2.0",1023 "model_name": "LayoutModel",1024 "state": {1025 "_model_module": "@jupyter-widgets/base",1026 "_model_module_version": "1.2.0",1027 "_model_name": "LayoutModel",1028 "_view_count": null,1029 "_view_module": "@jupyter-widgets/base",1030 "_view_module_version": "1.2.0",1031 "_view_name": "LayoutView",1032 "align_content": null,1033 "align_items": null,1034 "align_self": null,1035 "border": null,1036 "bottom": null,1037 "display": null,1038 "flex": null,1039 "flex_flow": null,1040 "grid_area": null,1041 "grid_auto_columns": null,1042 "grid_auto_flow": null,1043 "grid_auto_rows": null,1044 "grid_column": null,1045 "grid_gap": null,1046 "grid_row": null,1047 "grid_template_areas": null,1048 "grid_template_columns": null,1049 "grid_template_rows": null,1050 "height": null,1051 "justify_content": null,1052 "justify_items": null,1053 "left": null,1054 "margin": null,1055 "max_height": null,1056 "max_width": null,1057 "min_height": null,1058 "min_width": null,1059 "object_fit": null,1060 "object_position": null,1061 "order": null,1062 "overflow": null,1063 "overflow_x": null,1064 "overflow_y": null,1065 "padding": null,1066 "right": null,1067 "top": null,1068 "visibility": null,1069 "width": null1070 }1071 },1072 "d0ef36296eae47beb753ce63ef9dcf77": {1073 "model_module": "@jupyter-widgets/controls",1074 "model_module_version": "1.5.0",1075 "model_name": "ProgressStyleModel",1076 "state": {1077 "_model_module": "@jupyter-widgets/controls",1078 "_model_module_version": "1.5.0",1079 "_model_name": "ProgressStyleModel",1080 "_view_count": null,1081 "_view_module": "@jupyter-widgets/base",1082 "_view_module_version": "1.2.0",1083 "_view_name": "StyleView",1084 "bar_color": null,1085 "description_width": ""1086 }1087 },1088 "da20c99456e54d16b9140000231dc050": {1089 "model_module": "@jupyter-widgets/base",1090 "model_module_version": "1.2.0",1091 "model_name": "LayoutModel",1092 "state": {1093 "_model_module": "@jupyter-widgets/base",1094 "_model_module_version": "1.2.0",1095 "_model_name": "LayoutModel",1096 "_view_count": null,1097 "_view_module": "@jupyter-widgets/base",1098 "_view_module_version": "1.2.0",1099 "_view_name": "LayoutView",1100 "align_content": null,1101 "align_items": null,1102 "align_self": null,1103 "border": null,1104 "bottom": null,1105 "display": null,1106 "flex": null,1107 "flex_flow": null,1108 "grid_area": null,1109 "grid_auto_columns": null,1110 "grid_auto_flow": null,1111 "grid_auto_rows": null,1112 "grid_column": null,1113 "grid_gap": null,1114 "grid_row": null,1115 "grid_template_areas": null,1116 "grid_template_columns": null,1117 "grid_template_rows": null,1118 "height": null,1119 "justify_content": null,1120 "justify_items": null,1121 "left": null,1122 "margin": null,1123 "max_height": null,1124 "max_width": null,1125 "min_height": null,1126 "min_width": null,1127 "object_fit": null,1128 "object_position": null,1129 "order": null,1130 "overflow": null,1131 "overflow_x": null,1132 "overflow_y": null,1133 "padding": null,1134 "right": null,1135 "top": null,1136 "visibility": null,1137 "width": null1138 }1139 },1140 "df470c6b791d4e429cd27a4b3016d737": {1141 "model_module": "@jupyter-widgets/controls",1142 "model_module_version": "1.5.0",1143 "model_name": "HTMLModel",1144 "state": {1145 "_dom_classes": [],1146 "_model_module": "@jupyter-widgets/controls",1147 "_model_module_version": "1.5.0",1148 "_model_name": "HTMLModel",1149 "_view_count": null,1150 "_view_module": "@jupyter-widgets/controls",1151 "_view_module_version": "1.5.0",1152 "_view_name": "HTMLView",1153 "description": "",1154 "description_tooltip": null,1155 "layout": "IPY_MODEL_da20c99456e54d16b9140000231dc050",1156 "placeholder": "",1157 "style": "IPY_MODEL_9b7fe26167a245658a13442a6a866e6f",1158 "value": " 1/1 [00:00<00:00, 24.42ba/s]"1159 }1160 },1161 "e4b11afe53034934992cec2fdd953c0f": {1162 "model_module": "@jupyter-widgets/controls",1163 "model_module_version": "1.5.0",1164 "model_name": "DescriptionStyleModel",1165 "state": {1166 "_model_module": "@jupyter-widgets/controls",1167 "_model_module_version": "1.5.0",1168 "_model_name": "DescriptionStyleModel",1169 "_view_count": null,1170 "_view_module": "@jupyter-widgets/base",1171 "_view_module_version": "1.2.0",1172 "_view_name": "StyleView",1173 "description_width": ""1174 }1175 },1176 "e4c92db03df34d0489a426282dfb378d": {1177 "model_module": "@jupyter-widgets/controls",1178 "model_module_version": "1.5.0",1179 "model_name": "ProgressStyleModel",1180 "state": {1181 "_model_module": "@jupyter-widgets/controls",1182 "_model_module_version": "1.5.0",1183 "_model_name": "ProgressStyleModel",1184 "_view_count": null,1185 "_view_module": "@jupyter-widgets/base",1186 "_view_module_version": "1.2.0",1187 "_view_name": "StyleView",1188 "bar_color": null,1189 "description_width": ""1190 }1191 },1192 "ef6fa9adeff84619a2f226d1e5ed6530": {1193 "model_module": "@jupyter-widgets/controls",1194 "model_module_version": "1.5.0",1195 "model_name": "HTMLModel",1196 "state": {1197 "_dom_classes": [],1198 "_model_module": "@jupyter-widgets/controls",1199 "_model_module_version": "1.5.0",1200 "_model_name": "HTMLModel",