sluzala/the-algorithm-rag
0
1{2 "cells": [3 {4 "cell_type": "code",5 "execution_count": 1,6 "id": "a3ae6327-2c81-4e46-9bc4-0e2c9bc7e334",7 "metadata": {8 "scrolled": true9 },10 "outputs": [11 {12 "name": "stdout",13 "output_type": "stream",14 "text": [15 "Requirement already satisfied: huggingface_hub==0.22.2 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (0.22.2)\n",16 "Requirement already satisfied: sentence-transformers==2.7.0 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (2.7.0)\n",17 "Requirement already satisfied: filelock in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from huggingface_hub==0.22.2) (3.13.4)\n",18 "Requirement already satisfied: fsspec>=2023.5.0 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from huggingface_hub==0.22.2) (2024.3.1)\n",19 "Requirement already satisfied: packaging>=20.9 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from huggingface_hub==0.22.2) (23.2)\n",20 "Requirement already satisfied: pyyaml>=5.1 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from huggingface_hub==0.22.2) (6.0.1)\n",21 "Requirement already satisfied: requests in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from huggingface_hub==0.22.2) (2.31.0)\n",22 "Requirement already satisfied: tqdm>=4.42.1 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from huggingface_hub==0.22.2) (4.66.2)\n",23 "Requirement already satisfied: typing-extensions>=3.7.4.3 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from huggingface_hub==0.22.2) (4.11.0)\n",24 "Requirement already satisfied: transformers<5.0.0,>=4.34.0 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from sentence-transformers==2.7.0) (4.40.1)\n",25 "Requirement already satisfied: torch>=1.11.0 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from sentence-transformers==2.7.0) (2.3.0)\n",26 "Requirement already satisfied: numpy in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from sentence-transformers==2.7.0) (1.26.4)\n",27 "Requirement already satisfied: scikit-learn in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from sentence-transformers==2.7.0) (1.4.2)\n",28 "Requirement already satisfied: scipy in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from sentence-transformers==2.7.0) (1.13.0)\n",29 "Requirement already satisfied: Pillow in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from sentence-transformers==2.7.0) (10.3.0)\n",30 "Requirement already satisfied: sympy in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers==2.7.0) (1.12)\n",31 "Requirement already satisfied: networkx in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers==2.7.0) (3.3)\n",32 "Requirement already satisfied: jinja2 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers==2.7.0) (3.1.3)\n",33 "Requirement already satisfied: regex!=2019.12.17 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from transformers<5.0.0,>=4.34.0->sentence-transformers==2.7.0) (2024.4.16)\n",34 "Requirement already satisfied: tokenizers<0.20,>=0.19 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from transformers<5.0.0,>=4.34.0->sentence-transformers==2.7.0) (0.19.1)\n",35 "Requirement already satisfied: safetensors>=0.4.1 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from transformers<5.0.0,>=4.34.0->sentence-transformers==2.7.0) (0.4.3)\n",36 "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from requests->huggingface_hub==0.22.2) (3.3.2)\n",37 "Requirement already satisfied: idna<4,>=2.5 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from requests->huggingface_hub==0.22.2) (3.7)\n",38 "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from requests->huggingface_hub==0.22.2) (2.2.1)\n",39 "Requirement already satisfied: certifi>=2017.4.17 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from requests->huggingface_hub==0.22.2) (2024.2.2)\n",40 "Requirement already satisfied: joblib>=1.2.0 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from scikit-learn->sentence-transformers==2.7.0) (1.4.0)\n",41 "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from scikit-learn->sentence-transformers==2.7.0) (3.4.0)\n",42 "Requirement already satisfied: MarkupSafe>=2.0 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from jinja2->torch>=1.11.0->sentence-transformers==2.7.0) (2.1.5)\n",43 "Requirement already satisfied: mpmath>=0.19 in /Users/gabrielfritzsluzala/miniconda3/envs/rag/lib/python3.12/site-packages (from sympy->torch>=1.11.0->sentence-transformers==2.7.0) (1.3.0)\n",44 "Note: you may need to restart the kernel to use updated packages.\n"45 ]46 }47 ],48 "source": [49 "pip install huggingface_hub==0.22.2 sentence-transformers==2.7.0"50 ]51 },52 {53 "cell_type": "code",54 "execution_count": 2,55 "id": "d7c23447-5252-438d-b1a8-81bc62a7388a",56 "metadata": {},57 "outputs": [58 {59 "data": {60 "application/vnd.jupyter.widget-view+json": {61 "model_id": "1de14534027e4e5b8e6c1df8bd109332",62 "version_major": 2,63 "version_minor": 064 },65 "text/plain": [66 "Downloading readme: 0%| | 0.00/312 [00:00<?, ?B/s]"67 ]68 },69 "metadata": {},70 "output_type": "display_data"71 },72 {73 "data": {74 "application/vnd.jupyter.widget-view+json": {75 "model_id": "d7469d01d3d24c85a65a51b97f9bbc55",76 "version_major": 2,77 "version_minor": 078 },79 "text/plain": [80 "Downloading data: 0%| | 0.00/7.42M [00:00<?, ?B/s]"81 ]82 },83 "metadata": {},84 "output_type": "display_data"85 },86 {87 "data": {88 "application/vnd.jupyter.widget-view+json": {89 "model_id": "b76bf10fccda4599bca11ddba60cb6b1",90 "version_major": 2,91 "version_minor": 092 },93 "text/plain": [94 "Generating train split: 0%| | 0/6889 [00:00<?, ? examples/s]"95 ]96 },97 "metadata": {},98 "output_type": "display_data"99 }100 ],101 "source": [102 "from datasets import load_dataset\n",103 "import pandas as pd\n",104 "\n",105 "dataset = load_dataset(\"sluzala/the-algorithm-main\", split=\"train\")\n",106 "df = pd.DataFrame(dataset)"107 ]108 },109 {110 "cell_type": "code",111 "execution_count": 3,112 "id": "ee697ed3-c26f-41aa-99b7-00f4ce98ee62",113 "metadata": {},114 "outputs": [115 {116 "data": {117 "text/html": [118 "<div>\n",119 "<style scoped>\n",120 " .dataframe tbody tr th:only-of-type {\n",121 " vertical-align: middle;\n",122 " }\n",123 "\n",124 " .dataframe tbody tr th {\n",125 " vertical-align: top;\n",126 " }\n",127 "\n",128 " .dataframe thead th {\n",129 " text-align: right;\n",130 " }\n",131 "</style>\n",132 "<table border=\"1\" class=\"dataframe\">\n",133 " <thead>\n",134 " <tr style=\"text-align: right;\">\n",135 " <th></th>\n",136 " <th>path</th>\n",137 " <th>content</th>\n",138 " </tr>\n",139 " </thead>\n",140 " <tbody>\n",141 " <tr>\n",142 " <th>0</th>\n",143 " <td>the-algorithm-main/COPYING</td>\n",144 " <td>GNU AFFERO GENERAL PUBLIC ...</td>\n",145 " </tr>\n",146 " <tr>\n",147 " <th>1</th>\n",148 " <td>the-algorithm-main/README.md</td>\n",149 " <td># Twitter's Recommendation Algorithm\\n\\nTwitte...</td>\n",150 " </tr>\n",151 " <tr>\n",152 " <th>2</th>\n",153 " <td>the-algorithm-main/RETREIVAL_SIGNALS.md</td>\n",154 " <td># Signals for Candidate Sources\\n\\n## Overview...</td>\n",155 " </tr>\n",156 " <tr>\n",157 " <th>3</th>\n",158 " <td>the-algorithm-main/ann/src/main/java/com/twitt...</td>\n",159 " <td>target(\\n name = \"faiss\",\\n dependencies...</td>\n",160 " </tr>\n",161 " <tr>\n",162 " <th>4</th>\n",163 " <td>the-algorithm-main/ann/src/main/java/com/twitt...</td>\n",164 " <td>package com.twitter.ann.faiss;\\n\\nimport java....</td>\n",165 " </tr>\n",166 " </tbody>\n",167 "</table>\n",168 "</div>"169 ],170 "text/plain": [171 " path \\\n",172 "0 the-algorithm-main/COPYING \n",173 "1 the-algorithm-main/README.md \n",174 "2 the-algorithm-main/RETREIVAL_SIGNALS.md \n",175 "3 the-algorithm-main/ann/src/main/java/com/twitt... \n",176 "4 the-algorithm-main/ann/src/main/java/com/twitt... \n",177 "\n",178 " content \n",179 "0 GNU AFFERO GENERAL PUBLIC ... \n",180 "1 # Twitter's Recommendation Algorithm\\n\\nTwitte... \n",181 "2 # Signals for Candidate Sources\\n\\n## Overview... \n",182 "3 target(\\n name = \"faiss\",\\n dependencies... \n",183 "4 package com.twitter.ann.faiss;\\n\\nimport java.... "184 ]185 },186 "execution_count": 3,187 "metadata": {},188 "output_type": "execute_result"189 }190 ],191 "source": [192 "df.head()"193 ]194 },195 {196 "cell_type": "code",197 "execution_count": 4,198 "id": "108abe9e-fe20-4844-bae4-dda26572bd47",199 "metadata": {},200 "outputs": [],201 "source": [202 "from transformers import AutoTokenizer\n",203 "\n",204 "model_id = 'mixedbread-ai/mxbai-embed-large-v1'\n",205 "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",206 "\n",207 "def count_tokens(text):\n",208 " return tokenizer(text, return_length=True)[\"length\"][0]"209 ]210 },211 {212 "cell_type": "code",213 "execution_count": 5,214 "id": "507e695a-e62d-43ee-9666-2542082b9e59",215 "metadata": {},216 "outputs": [217 {218 "name": "stderr",219 "output_type": "stream",220 "text": [221 "Token indices sequence length is longer than the specified maximum sequence length for this model (6675 > 512). Running this sequence through the model will result in indexing errors\n"222 ]223 },224 {225 "data": {226 "text/plain": [227 "0 6675\n",228 "1 2000\n",229 "2 1159\n",230 "3 112\n",231 "4 1386\n",232 "Name: content, dtype: int64"233 ]234 },235 "execution_count": 5,236 "metadata": {},237 "output_type": "execute_result"238 }239 ],240 "source": [241 "df.head().content.apply(count_tokens)"242 ]243 },244 {245 "cell_type": "code",246 "execution_count": 6,247 "id": "dbc96561-ea30-47d6-9814-59371e10ddd1",248 "metadata": {},249 "outputs": [],250 "source": [251 "import re\n",252 "\n",253 "def split(text, separators):\n",254 "\n",255 " if count_tokens(text) <= n_tokens:\n",256 " return [text]\n",257 "\n",258 " if len(separators) == 0:\n",259 " raise Exception(\"No separators left and splits are still bigger than n_tokens\")\n",260 "\n",261 " separator = separators.pop(0)\n",262 "\n",263 " splits_by_separator = [s for s in re.split(re.escape(separator), text) if s != \"\"]\n",264 "\n",265 " result_splits = []\n",266 " \n",267 " for s in splits_by_separator:\n",268 " splits_by_next_separators = split(s, separators.copy())\n",269 " result_splits.extend(splits_by_next_separators)\n",270 "\n",271 " return result_splits"272 ]273 },274 {275 "cell_type": "code",276 "execution_count": 7,277 "id": "948ca446-2935-428c-b0a6-44102076cbfe",278 "metadata": {},279 "outputs": [],280 "source": [281 "separators = [\"\\n\\n\", \"\\n\", \" \", \"\"]\n",282 "n_tokens = 512\n",283 "\n",284 "df[\"splits\"] = df[\"content\"].apply(lambda x: split(x, separators.copy()))"285 ]286 },287 {288 "cell_type": "code",289 "execution_count": 8,290 "id": "f050b4c0-5b6d-41ab-b78b-2b8f2f5cba6b",291 "metadata": {},292 "outputs": [293 {294 "data": {295 "text/html": [296 "<div>\n",297 "<style scoped>\n",298 " .dataframe tbody tr th:only-of-type {\n",299 " vertical-align: middle;\n",300 " }\n",301 "\n",302 " .dataframe tbody tr th {\n",303 " vertical-align: top;\n",304 " }\n",305 "\n",306 " .dataframe thead th {\n",307 " text-align: right;\n",308 " }\n",309 "</style>\n",310 "<table border=\"1\" class=\"dataframe\">\n",311 " <thead>\n",312 " <tr style=\"text-align: right;\">\n",313 " <th></th>\n",314 " <th>path</th>\n",315 " <th>content</th>\n",316 " <th>splits</th>\n",317 " </tr>\n",318 " </thead>\n",319 " <tbody>\n",320 " <tr>\n",321 " <th>0</th>\n",322 " <td>the-algorithm-main/COPYING</td>\n",323 " <td>GNU AFFERO GENERAL PUBLIC ...</td>\n",324 " <td>[ GNU AFFERO GENERAL PUBLIC...</td>\n",325 " </tr>\n",326 " <tr>\n",327 " <th>1</th>\n",328 " <td>the-algorithm-main/README.md</td>\n",329 " <td># Twitter's Recommendation Algorithm\\n\\nTwitte...</td>\n",330 " <td>[# Twitter's Recommendation Algorithm, Twitter...</td>\n",331 " </tr>\n",332 " <tr>\n",333 " <th>2</th>\n",334 " <td>the-algorithm-main/RETREIVAL_SIGNALS.md</td>\n",335 " <td># Signals for Candidate Sources\\n\\n## Overview...</td>\n",336 " <td>[# Signals for Candidate Sources, ## Overview,...</td>\n",337 " </tr>\n",338 " <tr>\n",339 " <th>3</th>\n",340 " <td>the-algorithm-main/ann/src/main/java/com/twitt...</td>\n",341 " <td>target(\\n name = \"faiss\",\\n dependencies...</td>\n",342 " <td>[target(\\n name = \"faiss\",\\n dependencie...</td>\n",343 " </tr>\n",344 " <tr>\n",345 " <th>4</th>\n",346 " <td>the-algorithm-main/ann/src/main/java/com/twitt...</td>\n",347 " <td>package com.twitter.ann.faiss;\\n\\nimport java....</td>\n",348 " <td>[package com.twitter.ann.faiss;, import java.i...</td>\n",349 " </tr>\n",350 " </tbody>\n",351 "</table>\n",352 "</div>"353 ],354 "text/plain": [355 " path \\\n",356 "0 the-algorithm-main/COPYING \n",357 "1 the-algorithm-main/README.md \n",358 "2 the-algorithm-main/RETREIVAL_SIGNALS.md \n",359 "3 the-algorithm-main/ann/src/main/java/com/twitt... \n",360 "4 the-algorithm-main/ann/src/main/java/com/twitt... \n",361 "\n",362 " content \\\n",363 "0 GNU AFFERO GENERAL PUBLIC ... \n",364 "1 # Twitter's Recommendation Algorithm\\n\\nTwitte... \n",365 "2 # Signals for Candidate Sources\\n\\n## Overview... \n",366 "3 target(\\n name = \"faiss\",\\n dependencies... \n",367 "4 package com.twitter.ann.faiss;\\n\\nimport java.... \n",368 "\n",369 " splits \n",370 "0 [ GNU AFFERO GENERAL PUBLIC... \n",371 "1 [# Twitter's Recommendation Algorithm, Twitter... \n",372 "2 [# Signals for Candidate Sources, ## Overview,... \n",373 "3 [target(\\n name = \"faiss\",\\n dependencie... \n",374 "4 [package com.twitter.ann.faiss;, import java.i... "375 ]376 },377 "execution_count": 8,378 "metadata": {},379 "output_type": "execute_result"380 }381 ],382 "source": [383 "df.head()"384 ]385 },386 {387 "cell_type": "code",388 "execution_count": 9,389 "id": "5fdd6ae3-0104-4b1e-a031-87fa0c6bc548",390 "metadata": {},391 "outputs": [392 {393 "data": {394 "text/plain": [395 "[' GNU AFFERO... Version 3, 19 November 2007',\n",396 " ' Copyright (C) 2007 Free Softw...ut changing it is not allowed.',\n",397 " ' Pr... Preamble',\n",398 " ' The GNU Affero General Publi...se of network server software.',\n",399 " ' The licenses for most softwa...ee\\nsoftware for all its users.',\n",400 " ' When we speak of free softwa... know you can do these things.',\n",401 " ' Developers that use our Gene...te\\nand/or modify the software.',\n",402 " ' A secondary benefit of defen...its\\nsource code to the public.',\n",403 " ' The GNU Affero General Publi...\\ncode of the modified version.',\n",404 " ' An older license, called the...elicensing under\\nthis license.']"405 ]406 },407 "execution_count": 9,408 "metadata": {},409 "output_type": "execute_result"410 }411 ],412 "source": [413 "[f\"{s[:30]}...{s[-30:]}\" for s in df.iloc[0].splits][:10]"414 ]415 },416 {417 "cell_type": "code",418 "execution_count": 10,419 "id": "2dbdd8f2-150f-4148-b313-3ef7bf0bc9fc",420 "metadata": {},421 "outputs": [422 {423 "data": {424 "text/plain": [425 "0 15\n",426 "0 47\n",427 "0 5\n",428 "0 42\n",429 "0 61\n",430 " ..\n",431 "0 50\n",432 "0 41\n",433 "0 16\n",434 "0 89\n",435 "0 71\n",436 "Name: splits, Length: 118, dtype: int64"437 ]438 },439 "execution_count": 10,440 "metadata": {},441 "output_type": "execute_result"442 }443 ],444 "source": [445 "df.iloc[:1].explode(\"splits\").splits.apply(count_tokens)"446 ]447 },448 {449 "cell_type": "code",450 "execution_count": 11,451 "id": "8939fd5e-f733-4f75-ab86-bed8de195749",452 "metadata": {},453 "outputs": [],454 "source": [455 "def get_chunks(splits):\n",456 " \n",457 " chunks = []\n",458 " \n",459 " for split in splits:\n",460 " if len(chunks) == 0:\n",461 " chunks.append(split)\n",462 " elif count_tokens(f\"{chunks[-1]}\\n{split}\") <= n_tokens:\n",463 " chunks[-1] = f\"{chunks[-1]}\\n{split}\"\n",464 " else:\n",465 " chunks.append(split)\n",466 " \n",467 " return chunks"468 ]469 },470 {471 "cell_type": "code",472 "execution_count": 12,473 "id": "038ff867-5c38-4a25-a0e3-9206399eac6d",474 "metadata": {},475 "outputs": [],476 "source": [477 "df[\"chunks\"] = df[\"splits\"].apply(get_chunks)"478 ]479 },480 {481 "cell_type": "code",482 "execution_count": 13,483 "id": "3b275446-25dd-4b65-83f6-0973faadd370",484 "metadata": {},485 "outputs": [486 {487 "data": {488 "text/html": [489 "<div>\n",490 "<style scoped>\n",491 " .dataframe tbody tr th:only-of-type {\n",492 " vertical-align: middle;\n",493 " }\n",494 "\n",495 " .dataframe tbody tr th {\n",496 " vertical-align: top;\n",497 " }\n",498 "\n",499 " .dataframe thead th {\n",500 " text-align: right;\n",501 " }\n",502 "</style>\n",503 "<table border=\"1\" class=\"dataframe\">\n",504 " <thead>\n",505 " <tr style=\"text-align: right;\">\n",506 " <th></th>\n",507 " <th>path</th>\n",508 " <th>content</th>\n",509 " <th>splits</th>\n",510 " <th>chunks</th>\n",511 " </tr>\n",512 " </thead>\n",513 " <tbody>\n",514 " <tr>\n",515 " <th>0</th>\n",516 " <td>the-algorithm-main/COPYING</td>\n",517 " <td>GNU AFFERO GENERAL PUBLIC ...</td>\n",518 " <td>[ GNU AFFERO GENERAL PUBLIC...</td>\n",519 " <td>[ GNU AFFERO GENERAL PUBLIC...</td>\n",520 " </tr>\n",521 " <tr>\n",522 " <th>1</th>\n",523 " <td>the-algorithm-main/README.md</td>\n",524 " <td># Twitter's Recommendation Algorithm\\n\\nTwitte...</td>\n",525 " <td>[# Twitter's Recommendation Algorithm, Twitter...</td>\n",526 " <td>[# Twitter's Recommendation Algorithm\\nTwitter...</td>\n",527 " </tr>\n",528 " <tr>\n",529 " <th>2</th>\n",530 " <td>the-algorithm-main/RETREIVAL_SIGNALS.md</td>\n",531 " <td># Signals for Candidate Sources\\n\\n## Overview...</td>\n",532 " <td>[# Signals for Candidate Sources, ## Overview,...</td>\n",533 " <td>[# Signals for Candidate Sources\\n## Overview\\...</td>\n",534 " </tr>\n",535 " <tr>\n",536 " <th>3</th>\n",537 " <td>the-algorithm-main/ann/src/main/java/com/twitt...</td>\n",538 " <td>target(\\n name = \"faiss\",\\n dependencies...</td>\n",539 " <td>[target(\\n name = \"faiss\",\\n dependencie...</td>\n",540 " <td>[target(\\n name = \"faiss\",\\n dependencie...</td>\n",541 " </tr>\n",542 " <tr>\n",543 " <th>4</th>\n",544 " <td>the-algorithm-main/ann/src/main/java/com/twitt...</td>\n",545 " <td>package com.twitter.ann.faiss;\\n\\nimport java....</td>\n",546 " <td>[package com.twitter.ann.faiss;, import java.i...</td>\n",547 " <td>[package com.twitter.ann.faiss;\\nimport java.i...</td>\n",548 " </tr>\n",549 " </tbody>\n",550 "</table>\n",551 "</div>"552 ],553 "text/plain": [554 " path \\\n",555 "0 the-algorithm-main/COPYING \n",556 "1 the-algorithm-main/README.md \n",557 "2 the-algorithm-main/RETREIVAL_SIGNALS.md \n",558 "3 the-algorithm-main/ann/src/main/java/com/twitt... \n",559 "4 the-algorithm-main/ann/src/main/java/com/twitt... \n",560 "\n",561 " content \\\n",562 "0 GNU AFFERO GENERAL PUBLIC ... \n",563 "1 # Twitter's Recommendation Algorithm\\n\\nTwitte... \n",564 "2 # Signals for Candidate Sources\\n\\n## Overview... \n",565 "3 target(\\n name = \"faiss\",\\n dependencies... \n",566 "4 package com.twitter.ann.faiss;\\n\\nimport java.... \n",567 "\n",568 " splits \\\n",569 "0 [ GNU AFFERO GENERAL PUBLIC... \n",570 "1 [# Twitter's Recommendation Algorithm, Twitter... \n",571 "2 [# Signals for Candidate Sources, ## Overview,... \n",572 "3 [target(\\n name = \"faiss\",\\n dependencie... \n",573 "4 [package com.twitter.ann.faiss;, import java.i... \n",574 "\n",575 " chunks \n",576 "0 [ GNU AFFERO GENERAL PUBLIC... \n",577 "1 [# Twitter's Recommendation Algorithm\\nTwitter... \n",578 "2 [# Signals for Candidate Sources\\n## Overview\\... \n",579 "3 [target(\\n name = \"faiss\",\\n dependencie... \n",580 "4 [package com.twitter.ann.faiss;\\nimport java.i... "581 ]582 },583 "execution_count": 13,584 "metadata": {},585 "output_type": "execute_result"586 }587 ],588 "source": [589 "df.head()"590 ]591 },592 {593 "cell_type": "code",594 "execution_count": 14,595 "id": "6326a558-5e02-46bc-9da5-2f2d97315550",596 "metadata": {},597 "outputs": [],598 "source": [599 "df = df.explode(\"chunks\")\n",600 "df[\"id\"] = df[\"path\"] + \"-\" + (df.groupby(\"path\").cumcount()+1).astype(str).values\n",601 "df = df.reset_index(drop=True)\n",602 "df = df[[\"id\", \"chunks\"]]"603 ]604 },605 {606 "cell_type": "code",607 "execution_count": 15,608 "id": "131a609f-3c6e-489d-a20b-c9b609cbaa1e",609 "metadata": {},610 "outputs": [611 {612 "data": {613 "text/html": [614 "<div>\n",615 "<style scoped>\n",616 " .dataframe tbody tr th:only-of-type {\n",617 " vertical-align: middle;\n",618 " }\n",619 "\n",620 " .dataframe tbody tr th {\n",621 " vertical-align: top;\n",622 " }\n",623 "\n",624 " .dataframe thead th {\n",625 " text-align: right;\n",626 " }\n",627 "</style>\n",628 "<table border=\"1\" class=\"dataframe\">\n",629 " <thead>\n",630 " <tr style=\"text-align: right;\">\n",631 " <th></th>\n",632 " <th>id</th>\n",633 " <th>chunks</th>\n",634 " </tr>\n",635 " </thead>\n",636 " <tbody>\n",637 " <tr>\n",638 " <th>0</th>\n",639 " <td>the-algorithm-main/COPYING-1</td>\n",640 " <td>GNU AFFERO GENERAL PUBLIC ...</td>\n",641 " </tr>\n",642 " <tr>\n",643 " <th>1</th>\n",644 " <td>the-algorithm-main/COPYING-2</td>\n",645 " <td>An older license, called the Affero General ...</td>\n",646 " </tr>\n",647 " <tr>\n",648 " <th>2</th>\n",649 " <td>the-algorithm-main/COPYING-3</td>\n",650 " <td>The \"source code\" for a work means the prefe...</td>\n",651 " </tr>\n",652 " <tr>\n",653 " <th>3</th>\n",654 " <td>the-algorithm-main/COPYING-4</td>\n",655 " <td>All rights granted under this License are gr...</td>\n",656 " </tr>\n",657 " <tr>\n",658 " <th>4</th>\n",659 " <td>the-algorithm-main/COPYING-5</td>\n",660 " <td>You may charge any price or no price for eac...</td>\n",661 " </tr>\n",662 " </tbody>\n",663 "</table>\n",664 "</div>"665 ],666 "text/plain": [667 " id \\\n",668 "0 the-algorithm-main/COPYING-1 \n",669 "1 the-algorithm-main/COPYING-2 \n",670 "2 the-algorithm-main/COPYING-3 \n",671 "3 the-algorithm-main/COPYING-4 \n",672 "4 the-algorithm-main/COPYING-5 \n",673 "\n",674 " chunks \n",675 "0 GNU AFFERO GENERAL PUBLIC ... \n",676 "1 An older license, called the Affero General ... \n",677 "2 The \"source code\" for a work means the prefe... \n",678 "3 All rights granted under this License are gr... \n",679 "4 You may charge any price or no price for eac... "680 ]681 },682 "execution_count": 15,683 "metadata": {},684 "output_type": "execute_result"685 }686 ],687 "source": [688 "df.head()"689 ]690 },691 {692 "cell_type": "code",693 "execution_count": 16,694 "id": "9f53f1fc-e84e-4717-b5c8-63716cf9112a",695 "metadata": {},696 "outputs": [697 {698 "data": {699 "application/vnd.jupyter.widget-view+json": {700 "model_id": "af08609956304c0ea335b17f34bb8c47",701 "version_major": 2,702 "version_minor": 0703 },704 "text/plain": [705 "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"706 ]707 },708 "metadata": {},709 "output_type": "display_data"710 }711 ],712 "source": [713 "from huggingface_hub import notebook_login\n",714 "\n",715 "notebook_login()"716 ]717 },718 {719 "cell_type": "code",720 "execution_count": 17,721 "id": "f301992e-f451-42ff-99d6-d8519b597c54",722 "metadata": {},723 "outputs": [724 {725 "data": {726 "application/vnd.jupyter.widget-view+json": {727 "model_id": "5b42ad52225e44b2a983ba297ec61a96",728 "version_major": 2,729 "version_minor": 0730 },731 "text/plain": [732 "Uploading the dataset shards: 0%| | 0/1 [00:00<?, ?it/s]"733 ]734 },735 "metadata": {},736 "output_type": "display_data"737 },738 {739 "data": {740 "application/vnd.jupyter.widget-view+json": {741 "model_id": "d8360635776a4247b10dbeb555c083fd",742 "version_major": 2,743 "version_minor": 0744 },745 "text/plain": [746 "Creating parquet from Arrow format: 0%| | 0/20 [00:00<?, ?ba/s]"747 ]748 },749 "metadata": {},750 "output_type": "display_data"751 },752 {753 "data": {754 "application/vnd.jupyter.widget-view+json": {755 "model_id": "e3ce42d21a4d4248a060c9fb9ea197a4",756 "version_major": 2,757 "version_minor": 0758 },759 "text/plain": [760 "README.md: 0%| | 0.00/310 [00:00<?, ?B/s]"761 ]762 },763 "metadata": {},764 "output_type": "display_data"765 },766 {767 "data": {768 "text/plain": [769 "CommitInfo(commit_url='https://huggingface.co/datasets/sluzala/the-algorithm-main/commit/5772286def455a8040870d884357a242edcb907e', commit_message='Upload dataset', commit_description='', oid='5772286def455a8040870d884357a242edcb907e', pr_url=None, pr_revision=None, pr_num=None)"770 ]771 },772 "execution_count": 17,773 "metadata": {},774 "output_type": "execute_result"775 }776 ],777 "source": [778 "from datasets import Dataset\n",779 "new_dataset = Dataset.from_pandas(df, split=\"train\")\n",780 "new_dataset.push_to_hub(\"sluzala/the-algorithm-main\", revision=\"chunks\")"781 ]782 },783 {784 "cell_type": "code",785 "execution_count": null,786 "id": "885ef7c5-fd7b-414b-950f-87e2d45b6546",787 "metadata": {},788 "outputs": [],789 "source": []790 }791 ],792 "metadata": {793 "kernelspec": {794 "display_name": "Python 3 (ipykernel)",795 "language": "python",796 "name": "python3"797 },798 "language_info": {799 "codemirror_mode": {800 "name": "ipython",801 "version": 3802 },803 "file_extension": ".py",804 "mimetype": "text/x-python",805 "name": "python",806 "nbconvert_exporter": "python",807 "pygments_lexer": "ipython3",808 "version": "3.12.3"809 }810 },811 "nbformat": 4,812 "nbformat_minor": 5813}814