Hum-Works/lodestone-base-4096-v1
12143
1{2 "cells": [3 {4 "cell_type": "markdown",5 "id": "3f9ce240-fd1a-4550-83c7-8cf9658b1d3a",6 "metadata": {},7 "source": [8 "# Dataloading (1B+ Training Pairs)\n",9 "\n",10 "## This notebook collects and uploads all 50 relevant sentence embedding datasets to S3 as .json.gz files where each line contains one training record"11 ]12 },13 {14 "cell_type": "code",15 "execution_count": 62,16 "id": "d7af8d0e-b3ed-4007-abdd-5952d775e119",17 "metadata": {18 "tags": []19 },20 "outputs": [],21 "source": [22 "import os\n",23 "import pandas as pd\n",24 "\n",25 "os.chdir('/home/ec2-user/SageMaker')"26 ]27 },28 {29 "cell_type": "code",30 "execution_count": 63,31 "id": "686699fb-d2c9-4653-afca-580aef343451",32 "metadata": {33 "collapsed": true,34 "jupyter": {35 "outputs_hidden": true36 },37 "tags": []38 },39 "outputs": [40 {41 "name": "stdout",42 "output_type": "stream",43 "text": [44 "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n",45 " : update-motd, versionlock\n",46 "Cleaning repos: amzn2-core amzn2extra-docker amzn2extra-epel\n",47 " : amzn2extra-kernel-5.10 amzn2extra-python3.8 centos-extras\n",48 " : copr:copr.fedorainfracloud.org:vbatts:shadow-utils-newxidmap\n",49 " : docker-ce-stable libnvidia-container neuron\n",50 "21 metadata files removed\n",51 "15 sqlite files removed\n",52 "0 metadata files removed\n",53 "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n",54 " : update-motd, versionlock\n"55 ]56 },57 {58 "name": "stderr",59 "output_type": "stream",60 "text": [61 "https://download.docker.com/linux/centos/2/x86_64/stable/repodata/repomd.xml: [Errno 14] HTTPS Error 404 - Not Found\n",62 "Trying other mirror.\n"63 ]64 },65 {66 "name": "stdout",67 "output_type": "stream",68 "text": [69 "62 packages excluded due to repository priority protections\n",70 "Resolving Dependencies\n",71 "--> Running transaction check\n",72 "---> Package epel-release.noarch 0:7-11 will be installed\n",73 "--> Finished Dependency Resolution\n",74 "\n",75 "Dependencies Resolved\n",76 "\n",77 "================================================================================\n",78 " Package Arch Version Repository Size\n",79 "================================================================================\n",80 "Installing:\n",81 " epel-release noarch 7-11 amzn2extra-epel 15 k\n",82 "\n",83 "Transaction Summary\n",84 "================================================================================\n",85 "Install 1 Package\n",86 "\n",87 "Total download size: 15 k\n",88 "Installed size: 24 k\n",89 "Downloading packages:\n",90 "Running transaction check\n",91 "Running transaction test\n",92 "Transaction test succeeded\n",93 "Running transaction\n"94 ]95 },96 {97 "name": "stderr",98 "output_type": "stream",99 "text": [100 "Warning: RPMDB altered outside of yum.\n"101 ]102 },103 {104 "name": "stdout",105 "output_type": "stream",106 "text": [107 " Installing : epel-release-7-11.noarch 1/1 \n",108 " Verifying : epel-release-7-11.noarch 1/1 \n",109 "\n",110 "Installed:\n",111 " epel-release.noarch 0:7-11 \n",112 "\n",113 "Complete!\n",114 "Installing epel-release\n",115 " 0 ansible2 available \\\n",116 " [ =2.4.2 =2.4.6 =2.8 =stable ]\n",117 " 2 httpd_modules available [ =1.0 =stable ]\n",118 " 3 memcached1.5 available \\\n",119 " [ =1.5.1 =1.5.16 =1.5.17 ]\n",120 " 6 postgresql10 available [ =10 =stable ]\n",121 " 9 R3.4 available [ =3.4.3 =stable ]\n",122 " 10 rust1 available \\\n",123 " [ =1.22.1 =1.26.0 =1.26.1 =1.27.2 =1.31.0 =1.38.0\n",124 " =stable ]\n",125 " 18 libreoffice available \\\n",126 " [ =5.0.6.2_15 =5.3.6.1 =stable ]\n",127 " 19 gimp available [ =2.8.22 ]\n",128 " 20 docker=latest enabled \\\n",129 " [ =17.12.1 =18.03.1 =18.06.1 =18.09.9 =stable ]\n",130 " 21 mate-desktop1.x available \\\n",131 " [ =1.19.0 =1.20.0 =stable ]\n",132 " 22 GraphicsMagick1.3 available \\\n",133 " [ =1.3.29 =1.3.32 =1.3.34 =stable ]\n",134 " 23 tomcat8.5 available \\\n",135 " [ =8.5.31 =8.5.32 =8.5.38 =8.5.40 =8.5.42 =8.5.50\n",136 " =stable ]\n",137 " 24 epel=latest enabled [ =7.11 =stable ]\n",138 " 25 testing available [ =1.0 =stable ]\n",139 " 26 ecs available [ =stable ]\n",140 " 27 corretto8 available \\\n",141 " [ =1.8.0_192 =1.8.0_202 =1.8.0_212 =1.8.0_222 =1.8.0_232\n",142 " =1.8.0_242 =stable ]\n",143 " 29 golang1.11 available \\\n",144 " [ =1.11.3 =1.11.11 =1.11.13 =stable ]\n",145 " 30 squid4 available [ =4 =stable ]\n",146 " 32 lustre2.10 available \\\n",147 " [ =2.10.5 =2.10.8 =stable ]\n",148 " 33 java-openjdk11 available [ =11 =stable ]\n",149 " 34 lynis available [ =stable ]\n",150 " 36 BCC available [ =0.x =stable ]\n",151 " 37 mono available [ =5.x =stable ]\n",152 " 38 nginx1 available [ =stable ]\n",153 " 40 mock available [ =stable ]\n",154 " 41 postgresql11 available [ =11 =stable ]\n",155 " 43 livepatch available [ =stable ]\n",156 " 44 python3.8=latest enabled [ =stable ]\n",157 " 45 haproxy2 available [ =stable ]\n",158 " 46 collectd available [ =stable ]\n",159 " 47 aws-nitro-enclaves-cli available [ =stable ]\n",160 " 48 R4 available [ =stable ]\n",161 " _ kernel-5.4 available [ =stable ]\n",162 " 50 selinux-ng available [ =stable ]\n",163 " 51 php8.0 available [ =stable ]\n",164 " 52 tomcat9 available [ =stable ]\n",165 " 53 unbound1.13 available [ =stable ]\n",166 " 54 mariadb10.5 available [ =stable ]\n",167 " 55 kernel-5.10=latest enabled [ =stable ]\n",168 " 56 redis6 available [ =stable ]\n",169 " 57 ruby3.0 available [ =stable ]\n",170 " 58 postgresql12 available [ =stable ]\n",171 " 59 postgresql13 available [ =stable ]\n",172 " 60 mock2 available [ =stable ]\n",173 " 61 dnsmasq2.85 available [ =stable ]\n",174 " 62 kernel-5.15 available [ =stable ]\n",175 " 63 postgresql14 available [ =stable ]\n",176 " 64 firefox available [ =stable ]\n",177 " 65 lustre available [ =stable ]\n",178 " 66 php8.1 available [ =stable ]\n",179 " 67 awscli1 available [ =stable ]\n",180 " 68 php8.2 available [ =stable ]\n",181 " 69 dnsmasq available [ =stable ]\n",182 " 70 unbound1.17 available [ =stable ]\n",183 " 71 golang1.19 available [ =stable ]\n",184 " 72 collectd-python3 available [ =stable ]\n",185 "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n",186 " : update-motd, versionlock\n",187 "================================== repo: epel ==================================\n",188 "[epel]\n",189 "async = True\n",190 "bandwidth = 0\n",191 "base_persistdir = /var/lib/yum/repos/x86_64/2\n",192 "baseurl = \n",193 "cache = 0\n",194 "cachedir = /var/cache/yum/x86_64/2/epel\n",195 "check_config_file_age = True\n",196 "compare_providers_priority = 80\n",197 "cost = 1000\n",198 "deltarpm_metadata_percentage = 100\n",199 "deltarpm_percentage = \n",200 "enabled = True\n",201 "enablegroups = True\n",202 "exclude = \n",203 "failovermethod = priority\n",204 "ftp_disable_epsv = False\n",205 "gpgcadir = /var/lib/yum/repos/x86_64/2/epel/gpgcadir\n",206 "gpgcakey = \n",207 "gpgcheck = True\n",208 "gpgdir = /var/lib/yum/repos/x86_64/2/epel/gpgdir\n",209 "gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7\n",210 "hdrdir = /var/cache/yum/x86_64/2/epel/headers\n",211 "http_caching = all\n",212 "includepkgs = \n",213 "ip_resolve = \n",214 "keepalive = True\n",215 "keepcache = False\n",216 "mddownloadpolicy = sqlite\n",217 "mdpolicy = group:small\n",218 "mediaid = \n",219 "metadata_expire = 21600\n",220 "metadata_expire_filter = read-only:present\n",221 "metalink = https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=x86_64\n",222 "minrate = 0\n",223 "mirrorlist = \n",224 "mirrorlist_expire = 86400\n",225 "name = Extra Packages for Enterprise Linux 7 - x86_64\n",226 "old_base_cache_dir = \n",227 "password = \n",228 "persistdir = /var/lib/yum/repos/x86_64/2/epel\n",229 "pkgdir = /var/cache/yum/x86_64/2/epel/packages\n",230 "priority = 99\n",231 "proxy = False\n",232 "proxy_dict = \n",233 "proxy_password = \n",234 "proxy_username = \n",235 "repo_gpgcheck = False\n",236 "report_instanceid = False\n",237 "retries = 7\n",238 "skip_if_unavailable = False\n",239 "ssl_check_cert_permissions = True\n",240 "sslcacert = \n",241 "sslclientcert = \n",242 "sslclientkey = \n",243 "sslverify = True\n",244 "throttle = 0\n",245 "timeout = 5.0\n",246 "ui_id = epel/x86_64\n",247 "ui_repoid_vars = releasever,\n",248 " basearch\n",249 "username = \n",250 "\n",251 "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n",252 " : update-motd, versionlock\n"253 ]254 },255 {256 "name": "stderr",257 "output_type": "stream",258 "text": [259 "https://download.docker.com/linux/centos/2/x86_64/stable/repodata/repomd.xml: [Errno 14] HTTPS Error 404 - Not Found\n",260 "Trying other mirror.\n",261 "http://mirror.es.its.nyu.edu/epel/7/x86_64/repodata/repomd.xml: [Errno 12] Timeout on http://mirror.es.its.nyu.edu/epel/7/x86_64/repodata/repomd.xml: (28, 'Failed to connect to mirror.es.its.nyu.edu port 80 after 5001 ms: Timeout was reached')\n",262 "Trying other mirror.\n"263 ]264 },265 {266 "name": "stdout",267 "output_type": "stream",268 "text": [269 "286 packages excluded due to repository priority protections\n",270 "Resolving Dependencies\n",271 "--> Running transaction check\n",272 "---> Package git-lfs.x86_64 0:2.10.0-2.el7 will be installed\n",273 "--> Finished Dependency Resolution\n",274 "\n",275 "Dependencies Resolved\n",276 "\n",277 "================================================================================\n",278 " Package Arch Version Repository Size\n",279 "================================================================================\n",280 "Installing:\n",281 " git-lfs x86_64 2.10.0-2.el7 epel 3.7 M\n",282 "\n",283 "Transaction Summary\n",284 "================================================================================\n",285 "Install 1 Package\n",286 "\n",287 "Total download size: 3.7 M\n",288 "Installed size: 13 M\n",289 "Downloading packages:\n"290 ]291 },292 {293 "name": "stderr",294 "output_type": "stream",295 "text": [296 "warning: /var/cache/yum/x86_64/2/epel/packages/git-lfs-2.10.0-2.el7.x86_64.rpm: Header V4 RSA/SHA256 Signature, key ID 352c64e5: NOKEY\n",297 "Importing GPG key 0x352C64E5:\n",298 " Userid : \"Fedora EPEL (7) <epel@fedoraproject.org>\"\n",299 " Fingerprint: 91e9 7d7c 4a5e 96f1 7f3e 888f 6a2f aea2 352c 64e5\n",300 " Package : epel-release-7-11.noarch (@amzn2extra-epel)\n",301 " From : /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7\n"302 ]303 },304 {305 "name": "stdout",306 "output_type": "stream",307 "text": [308 "Public key for git-lfs-2.10.0-2.el7.x86_64.rpm is not installed\n",309 "Retrieving key from file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7\n",310 "Running transaction check\n",311 "Running transaction test\n",312 "Transaction test succeeded\n",313 "Running transaction\n",314 " Installing : git-lfs-2.10.0-2.el7.x86_64 1/1 \n",315 " Verifying : git-lfs-2.10.0-2.el7.x86_64 1/1 \n",316 "\n",317 "Installed:\n",318 " git-lfs.x86_64 0:2.10.0-2.el7 \n",319 "\n",320 "Complete!\n",321 "Git LFS initialized.\n"322 ]323 },324 {325 "data": {326 "text/plain": [327 "0"328 ]329 },330 "execution_count": 63,331 "metadata": {},332 "output_type": "execute_result"333 }334 ],335 "source": [336 "# install git-lfs\n",337 "os.system('sudo amazon-linux-extras install epel -y')\n",338 "os.system('sudo yum-config-manager --enable epel')\n",339 "os.system('sudo yum install git-lfs -y')\n",340 "os.system('git lfs install')"341 ]342 },343 {344 "cell_type": "code",345 "execution_count": 3,346 "id": "6595a6c5-9fae-4bd4-b096-53475ec98294",347 "metadata": {348 "tags": []349 },350 "outputs": [351 {352 "name": "stderr",353 "output_type": "stream",354 "text": [355 "Cloning into 'stackexchange_title_body_jsonl'...\n",356 "Cloning into 'stackexchange_titlebody_best_voted_answer_jsonl'...\n",357 "Cloning into 'stackexchange_title_best_voted_answer_jsonl'...\n",358 "Cloning into 'stackexchange_titlebody_best_and_down_voted_answer_jsonl'...\n",359 "Cloning into 'reddit-title-body'...\n",360 "Cloning into '1B_sentence_embeddings'...\n"361 ]362 },363 {364 "data": {365 "text/plain": [366 "0"367 ]368 },369 "execution_count": 3,370 "metadata": {},371 "output_type": "execute_result"372 }373 ],374 "source": [375 "# clone relevant datasets' github repositories\n",376 "stacks = ['stackexchange_title_body_jsonl', #25.3M \n",377 " 'stackexchange_titlebody_best_voted_answer_jsonl', #4.75M\n",378 " 'stackexchange_title_best_voted_answer_jsonl', #4.75M\n",379 " 'stackexchange_titlebody_best_and_down_voted_answer_jsonl'] #210K\n",380 "\n",381 "os.environ['GIT_LFS_SKIP_SMUDGE'] = \"1\"\n",382 "\n",383 "# clone stackexchange repos\n",384 "for stack in stacks:\n",385 " os.system(f'git clone https://huggingface.co/datasets/flax-sentence-embeddings/{stack}')\n",386 "# clone reddit repo\n",387 "os.system('git clone https://huggingface.co/datasets/sentence-transformers/reddit-title-body')\n",388 "# clone 1B+ sentence embeddings repo (this one is just for reference)\n",389 "os.system('git clone https://github.com/AntoineSimoulin/1B_sentence_embeddings')"390 ]391 },392 {393 "cell_type": "code",394 "execution_count": null,395 "id": "56438029-0f26-491c-b405-f92fb9caeff7",396 "metadata": {397 "collapsed": true,398 "jupyter": {399 "outputs_hidden": true400 },401 "tags": []402 },403 "outputs": [404 {405 "name": "stdout",406 "output_type": "stream",407 "text": [408 "Downloading 4 StackExchange GitHub datasets into s3://lodestone-rnd/data/\n",409 "upload: ./networkengineering.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/networkengineering.stackexchange.com.json.gz\n",410 "upload: ./emacs.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/emacs.stackexchange.com.json.gz\n",411 "upload: ./christianity.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/christianity.stackexchange.com.json.gz\n",412 "upload: ./bitcoin.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/bitcoin.stackexchange.com.json.gz\n",413 "upload: ./academia.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/academia.stackexchange.com.json.gz\n",414 "upload: ./music.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/music.stackexchange.com.json.gz\n",415 "upload: ./biology.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/biology.stackexchange.com.json.gz\n",416 "upload: ./history.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/history.stackexchange.com.json.gz\n",417 "upload: ./skeptics.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/skeptics.stackexchange.com.json.gz\n",418 "upload: ./anime.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/anime.stackexchange.com.json.gz\n",419 "upload: ./quant.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/quant.stackexchange.com.json.gz\n",420 "upload: ./boardgames.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/boardgames.stackexchange.com.json.gz\n",421 "upload: ./judaism.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/judaism.stackexchange.com.json.gz\n",422 "upload: ./travel.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/travel.stackexchange.com.json.gz\n",423 "upload: ./gaming.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/gaming.stackexchange.com.json.gz\n",424 "upload: ./webapps.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/webapps.stackexchange.com.json.gz\n",425 "upload: ./stats.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/stats.stackexchange.com.json.gz\n",426 "upload: ./law.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/law.stackexchange.com.json.gz\n",427 "upload: ./scifi.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/scifi.stackexchange.com.json.gz\n",428 "upload: ./bicycles.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/bicycles.stackexchange.com.json.gz\n",429 "upload: ./datascience.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/datascience.stackexchange.com.json.gz\n",430 "upload: ./softwareengineering.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/softwareengineering.stackexchange.com.json.gz\n",431 "upload: ./islam.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/islam.stackexchange.com.json.gz\n",432 "upload: ./craftcms.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/craftcms.stackexchange.com.json.gz\n",433 "upload: ./diy.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/diy.stackexchange.com.json.gz\n",434 "upload: ./arduino.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/arduino.stackexchange.com.json.gz\n",435 "upload: ./raspberrypi.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/raspberrypi.stackexchange.com.json.gz\n",436 "upload: ./wordpress.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/wordpress.stackexchange.com.json.gz\n",437 "upload: ./dba.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/dba.stackexchange.com.json.gz\n",438 "upload: ./apple.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/apple.stackexchange.com.json.gz\n",439 "upload: ./hinduism.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/hinduism.stackexchange.com.json.gz\n",440 "upload: ./mechanics.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/mechanics.stackexchange.com.json.gz\n",441 "upload: ./gamedev.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/gamedev.stackexchange.com.json.gz\n",442 "upload: ./writers.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/writers.stackexchange.com.json.gz\n",443 "upload: ./mathematica.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/mathematica.stackexchange.com.json.gz\n",444 "upload: ./unix.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/unix.stackexchange.com.json.gz\n",445 "upload: ./magento.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/magento.stackexchange.com.json.gz\n",446 "upload: ./ethereum.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/ethereum.stackexchange.com.json.gz\n",447 "upload: ./electronics.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/electronics.stackexchange.com.json.gz\n",448 "upload: ./cs.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/cs.stackexchange.com.json.gz\n",449 "upload: ./blender.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/blender.stackexchange.com.json.gz\n",450 "upload: ./drupal.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/drupal.stackexchange.com.json.gz\n",451 "upload: ./small_stackexchanges.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/small_stackexchanges.json.gz\n",452 "upload: ./photo.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/photo.stackexchange.com.json.gz\n",453 "upload: ./engineering.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/engineering.stackexchange.com.json.gz\n",454 "upload: ./ux.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/ux.stackexchange.com.json.gz\n",455 "upload: ./german.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/german.stackexchange.com.json.gz\n",456 "upload: ./japanese.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/japanese.stackexchange.com.json.gz\n",457 "upload: ./civicrm.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/civicrm.stackexchange.com.json.gz\n",458 "upload: ./sharepoint.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/sharepoint.stackexchange.com.json.gz\n",459 "upload: ./mathoverflow.net.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/mathoverflow.net.json.gz\n",460 "upload: ./meta.stackoverflow.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/meta.stackoverflow.com.json.gz\n",461 "upload: ./rpg.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/rpg.stackexchange.com.json.gz\n",462 "upload: ./crypto.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/crypto.stackexchange.com.json.gz\n",463 "upload: ./vi.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/vi.stackexchange.com.json.gz\n",464 "upload: ./graphicdesign.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/graphicdesign.stackexchange.com.json.gz\n",465 "upload: ./cooking.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/cooking.stackexchange.com.json.gz\n",466 "upload: ./math.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/math.stackexchange.com.json.gz\n",467 "upload: ./expressionengine.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/expressionengine.stackexchange.com.json.gz\n",468 "upload: ./movies.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/movies.stackexchange.com.json.gz\n",469 "upload: ./salesforce.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/salesforce.stackexchange.com.json.gz\n",470 "upload: ./physics.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/physics.stackexchange.com.json.gz\n",471 "upload: ./aviation.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/aviation.stackexchange.com.json.gz\n",472 "upload: ./gardening.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/gardening.stackexchange.com.json.gz\n",473 "upload: ./english.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/english.stackexchange.com.json.gz\n",474 "upload: ./askubuntu.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/askubuntu.com.json.gz\n",475 "upload: ./french.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/french.stackexchange.com.json.gz\n",476 "upload: ./codereview.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/codereview.stackexchange.com.json.gz\n",477 "upload: ./softwarerecs.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/softwarerecs.stackexchange.com.json.gz\n",478 "upload: ./rus.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/rus.stackexchange.com.json.gz\n",479 "upload: ./money.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/money.stackexchange.com.json.gz\n",480 "upload: ./philosophy.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/philosophy.stackexchange.com.json.gz\n",481 "upload: ./chemistry.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/chemistry.stackexchange.com.json.gz\n",482 "upload: ./meta.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/meta.stackexchange.com.json.gz\n",483 "upload: ./cstheory.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/cstheory.stackexchange.com.json.gz\n",484 "upload: ./space.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/space.stackexchange.com.json.gz\n",485 "upload: ./politics.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/politics.stackexchange.com.json.gz\n",486 "upload: ./ell.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/ell.stackexchange.com.json.gz\n",487 "upload: ./puzzling.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/puzzling.stackexchange.com.json.gz\n",488 "upload: ./astronomy.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/astronomy.stackexchange.com.json.gz\n",489 "upload: ./worldbuilding.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/worldbuilding.stackexchange.com.json.gz\n",490 "upload: ./economics.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/economics.stackexchange.com.json.gz\n",491 "upload: ./workplace.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/workplace.stackexchange.com.json.gz\n",492 "upload: ./tex.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/tex.stackexchange.com.json.gz\n",493 "upload: ./android.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/android.stackexchange.com.json.gz\n",494 "upload: ./gis.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/gis.stackexchange.com.json.gz\n",495 "upload: ./dsp.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/dsp.stackexchange.com.json.gz\n",496 "upload: ./superuser.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_body_jsonl/superuser.com.json.gz\n",497 "upload: ./english.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_best_voted_answer_jsonl/english.stackexchange.com.json.gz\n",498 "upload: ./meta.serverfault.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_best_voted_answer_jsonl/meta.serverfault.com.json.gz\n",499 "upload: ./scicomp.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_best_voted_answer_jsonl/scicomp.stackexchange.com.json.gz\n",500 "upload: ./askubuntu.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_best_voted_answer_jsonl/askubuntu.com.json.gz\n",501 "upload: ./french.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_best_voted_answer_jsonl/french.stackexchange.com.json.gz\n",502 "upload: ./coffee.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_best_voted_answer_jsonl/coffee.stackexchange.com.json.gz\n",503 "upload: ./codereview.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_best_voted_answer_jsonl/codereview.stackexchange.com.json.gz\n",504 "upload: ./sound.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_best_voted_answer_jsonl/sound.stackexchange.com.json.gz\n",505 "upload: ./opensource.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_best_voted_answer_jsonl/opensource.stackexchange.com.json.gz\n",506 "upload: ./woodworking.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_best_voted_answer_jsonl/woodworking.stackexchange.com.json.gz\n",507 "upload: ./outdoors.stackexchange.com.jsonl.gz to s3://lodestone-rnd/data/stackexchange_title_best_voted_answer_jsonl/outdoors.stackexchange.com.json.gz\n",508 "upload: ./reddit_title_text_2018.jsonl.gz to s3://lodestone-rnd/data/reddit-title-body/reddit_title_text_2018.json.gz\n",509 "upload: ./reddit_title_text_2011.jsonl.gz to s3://lodestone-rnd/data/reddit-title-body/reddit_title_text_2011.json.gz\n",510 "upload: ./reddit_title_text_2020.jsonl.gz to s3://lodestone-rnd/data/reddit-title-body/reddit_title_text_2020.json.gz\n",511 "upload: ./reddit_title_text_2012.jsonl.gz to s3://lodestone-rnd/data/reddit-title-body/reddit_title_text_2012.json.gz\n",512 "upload: ./reddit_title_text_2021.jsonl.gz to s3://lodestone-rnd/data/reddit-title-body/reddit_title_text_2021.json.gz\n",513 "upload: ./reddit_title_text_2019.jsonl.gz to s3://lodestone-rnd/data/reddit-title-body/reddit_title_text_2019.json.gz\n",514 "upload: ./reddit_title_text_2010.jsonl.gz to s3://lodestone-rnd/data/reddit-title-body/reddit_title_text_2010.json.gz\n",515 "upload: ./reddit_title_text_2014.jsonl.gz to s3://lodestone-rnd/data/reddit-title-body/reddit_title_text_2014.json.gz\n",516 "\u001b[32mDone\u001b[0m\n",517 "Total files uploaded: 12\n",518 "\n",519 "\n",520 "Downloading 9 HuggingFace datasets into s3://lodestone-rnd/data/\n",521 "Downloading dataset S2ORC_citations_abstracts (39,567,485 pairs) ... "522 ]523 }524 ],525 "source": [526 "# DOWNLOAD GITHUB DATASETS (STACKEXCHANGE (https://huggingface.co/flax-sentence-embeddings) & REDDIT (https://huggingface.co/datasets/sentence-transformers/reddit-title-body))\n",527 "\n",528 "# these are the files marked as unsafe by HuggingFace when viewing the each of the datasets' pages\n",529 "unsafe = [[\"serverfault.com.jsonl.gz\", \"security.stackexchange.com.jsonl.gz\"],\n",530 " [\"monero.stackexchange.com.jsonl.gz\", \"serverfault.com.jsonl.gz\", \"security.stackexchange.com.jsonl.gz\"],\n",531 " [\"elementaryos.stackexchange.com.jsonl.gz\", \"monero.stackexchange.com.jsonl.gz\", \"security.stackexchange.com.jsonl.gz\"],\n",532 " [\"\"]]\n",533 "\n",534 "file_counts = []\n",535 "print('Downloading {:,} StackExchange GitHub datasets into s3://lodestone-rnd/data/'.format(len(stacks)))\n",536 "for i, stack in enumerate(stacks):\n",537 " # get the names of all the files in the repository that are not unsafe\n",538 " files = [file for file in os.listdir(f'/home/ec2-user/SageMaker/{stack}') if file.endswith(\".jsonl.gz\")==True if file not in unsafe[i]]\n",539 " file_counts.append(len(files))\n",540 " os.chdir(f'/home/ec2-user/SageMaker/{stack}')\n",541 " print('Downloading dataset {} ({} files) ... '.format(stack, len(files)), end='', flush=True)\n",542 " # sequentially pull each dataset from git lfs, stream it to S3, and then delete the local copy to free up disk memory\n",543 " for file_name in files:\n",544 " os.system(f'git lfs pull --include={file_name}')\n",545 " os.system(f'aws s3 cp {file_name} s3://lodestone-rnd/data/{stack}/{file_name[:-9] + \".json.gz\"}')\n",546 " os.remove(file_name)\n",547 " os.system('rm -r .git/lfs/objects/*')\n",548 " if len(os.listdir('.git/objects/pack')) == 4:\n",549 " os.system('ls -t .git/objects/pack/* | head -2 | xargs rm --')\n",550 " print('\\033[32m' + 'Done' + '\\033[0m')\n",551 "print(f'Total files uploaded: {sum(file_counts)}')\n",552 "\n",553 "print(\"\\n\")\n",554 "\n",555 "print('Downloading {:,} Reddit GitHub dataset into s3://lodestone-rnd/data/'.format(1))\n",556 "# get the names of all the files in the repository\n",557 "files = [file for file in os.listdir(f'/home/ec2-user/SageMaker/reddit-title-body') if file.endswith(\".jsonl.gz\")==True]\n",558 "os.chdir(f'/home/ec2-user/SageMaker/reddit-title-body')\n",559 "print('Downloading dataset {} ({} files) ... '.format(\"reddit-title-body\", len(files)), end='', flush=True)\n",560 "# sequentially pull each dataset from git lfs, stream it to S3, and then delete the local copy to free up disk memory\n",561 "for file_name in files:\n",562 " os.system(f'git lfs pull --include={file_name}')\n",563 " os.system(f'aws s3 cp {file_name} s3://lodestone-rnd/data/reddit-title-body/{file_name[:-9] + \".json.gz\"}')\n",564 " os.remove(file_name)\n",565 " os.system('rm -r .git/lfs/objects/*')\n",566 " if len(os.listdir('.git/objects/pack')) == 4:\n",567 " os.system('ls -t .git/objects/pack/* | head -2 | xargs rm --')\n",568 "print('\\033[32m' + 'Done' + '\\033[0m')\n",569 "print(f'Total files uploaded: {len(files)}')\n",570 "\n",571 "os.chdir('/home/ec2-user/SageMaker')\n",572 "\n",573 "print(\"\\n\")\n",574 "\n",575 "# DOWNLOAD HUGGINGFACE DATASETS (https://huggingface.co/datasets/sentence-transformers/embedding-training-data)\n",576 "\n",577 "# read dataset information from HuggingFace_datasets.tsv\n",578 "datasets = pd.read_csv(\n",579 " 'HuggingFace_datasets.tsv',\n",580 " index_col=0,\n",581 " sep='\\t',\n",582 " dtype={\n",583 " 'Description': str,\n",584 " 'Size (#Pairs)': str,\n",585 " 'Performance': float,\n",586 " 'Download link': str,\n",587 " 'Source': str})\n",588 "datasets['Size (#Pairs)'] = datasets['Size (#Pairs)'].str.replace(',', '').astype(int)\n",589 "datasets = datasets.to_dict(orient='index')\n",590 "\n",591 "print('Downloading {:,} HuggingFace datasets into s3://lodestone-rnd/data/'.format(len(datasets)))\n",592 "\n",593 "# stream each of the datasets from the URL provided into S3\n",594 "# (note that S2ORC_citations_abstracts is larger than 50GB and therefore requires the expected size to be passed into the command line as well)\n",595 "for d in datasets.keys():\n",596 " print('Downloading dataset {} ({:,} pairs) ... '.format(d, datasets[d]['Size (#Pairs)']), end='', flush=True)\n",597 " if d == \"S2ORC_citations_abstracts\":\n",598 " os.system(f'wget -qO- {datasets[d][\"Download link\"]} | aws s3 cp - s3://lodestone-rnd/data/{d + \".json.gz\"} --expected-size 120259084288')\n",599 " else:\n",600 " os.system(f'wget -qO- {datasets[d][\"Download link\"]} | aws s3 cp - s3://lodestone-rnd/data/{d + \".json.gz\"}')\n",601 " print('\\033[32m' + 'Done' + '\\033[0m')\n",602 "\n",603 "print(\"\\n\")\n",604 "\n",605 "# DOWNLOAD GOOGLE SHEETS DATASETS (https://docs.google.com/spreadsheets/d/1vXJrIg38cEaKjOG5y4I4PQwAQFUmCkohbViJ9zj_Emg/edit#gid=0)\n",606 "\n",607 "# read dataset information from GoogleSheets_datasets.tsv\n",608 "datasets = pd.read_csv(\n",609 " 'GoogleSheets_datasets.tsv',\n",610 " index_col=0,\n",611 " sep='\\t',\n",612 " dtype={\n",613 " 'Description': str,\n",614 " 'Size (#Pairs)': str,\n",615 " 'Performance': float,\n",616 " 'Download link': str,\n",617 " 'Source': str})\n",618 "datasets['Size (#Pairs)'] = datasets['Size (#Pairs)'].str.replace(',', '').astype(int)\n",619 "datasets = datasets.to_dict(orient='index')\n",620 "\n",621 "print('Downloading {:,} 1B+ Google Sheets datasets into s3://lodestone-rnd/data/'.format(len(datasets)))\n",622 "\n",623 "# stream each of the datasets from the URL provided into S3\n",624 "for d in datasets.keys():\n",625 " print('Downloading dataset {} ({:,} pairs) ... '.format(d, datasets[d]['Size (#Pairs)']), end='', flush=True)\n",626 " os.system(f'wget -qO- {datasets[d][\"Download link\"]} | aws s3 cp - s3://lodestone-rnd/data/{d + \".json.gz\"}')\n",627 " print('\\033[32m' + 'Done' + '\\033[0m')\n",628 "\n",629 "print(\"\\n\")\n",630 " \n",631 "print(f'Successfully downloaded 50 datasets and {621+12+9+36} files into s3://lodestone-rnd/data/')"632 ]633 },634 {635 "cell_type": "code",636 "execution_count": 1,637 "id": "216d4564-6b69-4688-94ec-a67287134a2d",638 "metadata": {639 "tags": []640 },641 "outputs": [],642 "source": [643 "# clean up (remove the cloned repositories)\n",644 "import shutil\n",645 "shutil.rmtree(\"/home/ec2-user/SageMaker/stackexchange_title_body_jsonl\")\n",646 "shutil.rmtree(\"/home/ec2-user/SageMaker/stackexchange_titlebody_best_voted_answer_jsonl\")\n",647 "shutil.rmtree(\"/home/ec2-user/SageMaker/stackexchange_title_best_voted_answer_jsonl\")\n",648 "shutil.rmtree(\"/home/ec2-user/SageMaker/stackexchange_titlebody_best_and_down_voted_answer_jsonl\")\n",649 "shutil.rmtree(\"/home/ec2-user/SageMaker/reddit-title-body\")\n",650 "shutil.rmtree(\"/home/ec2-user/SageMaker/1B_sentence_embeddings\")"651 ]652 }653 ],654 "metadata": {655 "kernelspec": {656 "display_name": "conda_pytorch_p310",657 "language": "python",658 "name": "conda_pytorch_p310"659 },660 "language_info": {661 "codemirror_mode": {662 "name": "ipython",663 "version": 3664 },665 "file_extension": ".py",666 "mimetype": "text/x-python",667 "name": "python",668 "nbconvert_exporter": "python",669 "pygments_lexer": "ipython3",670 "version": "3.10.10"671 }672 },673 "nbformat": 4,674 "nbformat_minor": 5675}676