Melk/DataAnalytics
0
1{2 "cells": [3 {4 "cell_type": "markdown",5 "id": "a679eb6f",6 "metadata": {7 "execution": {8 "iopub.execute_input": "2021-06-29T16:53:03.143789Z",9 "iopub.status.busy": "2021-06-29T16:53:03.142088Z",10 "iopub.status.idle": "2021-06-29T16:53:03.144605Z",11 "shell.execute_reply": "2021-06-29T16:53:03.145080Z",12 "shell.execute_reply.started": "2021-06-29T15:57:06.131907Z"13 },14 "papermill": {15 "duration": 0.065755,16 "end_time": "2021-06-29T16:53:03.145384",17 "exception": false,18 "start_time": "2021-06-29T16:53:03.079629",19 "status": "completed"20 },21 "tags": []22 },23 "source": [24 "# ORB-facebook-comments-sentiment-analysis"25 ]26 },27 {28 "cell_type": "code",29 "execution_count": 89,30 "id": "iraqi-vocabulary",31 "metadata": {32 "execution": {33 "iopub.execute_input": "2021-06-29T16:53:03.262309Z",34 "iopub.status.busy": "2021-06-29T16:53:03.261454Z",35 "iopub.status.idle": "2021-06-29T16:53:03.264983Z",36 "shell.execute_reply": "2021-06-29T16:53:03.264487Z",37 "shell.execute_reply.started": "2021-06-29T15:57:06.143507Z"38 },39 "papermill": {40 "duration": 0.066249,41 "end_time": "2021-06-29T16:53:03.265123",42 "exception": false,43 "start_time": "2021-06-29T16:53:03.198874",44 "status": "completed"45 },46 "tags": []47 },48 "outputs": [],49 "source": [50 "import warnings\n",51 "warnings.filterwarnings(\"ignore\")\n",52 "import numpy as np\n",53 "import pandas as pd\n",54 "import nltk"55 ]56 },57 {58 "cell_type": "code",59 "execution_count": 2,60 "id": "prerequisite-smell",61 "metadata": {62 "execution": {63 "iopub.execute_input": "2021-06-29T16:53:03.389393Z",64 "iopub.status.busy": "2021-06-29T16:53:03.388530Z",65 "iopub.status.idle": "2021-06-29T16:53:03.560950Z",66 "shell.execute_reply": "2021-06-29T16:53:03.561457Z",67 "shell.execute_reply.started": "2021-06-29T15:57:06.158431Z"68 },69 "papermill": {70 "duration": 0.235778,71 "end_time": "2021-06-29T16:53:03.561640",72 "exception": false,73 "start_time": "2021-06-29T16:53:03.325862",74 "status": "completed"75 },76 "tags": []77 },78 "outputs": [79 {80 "data": {81 "text/html": [82 "<div>\n",83 "<style scoped>\n",84 " .dataframe tbody tr th:only-of-type {\n",85 " vertical-align: middle;\n",86 " }\n",87 "\n",88 " .dataframe tbody tr th {\n",89 " vertical-align: top;\n",90 " }\n",91 "\n",92 " .dataframe thead th {\n",93 " text-align: right;\n",94 " }\n",95 "</style>\n",96 "<table border=\"1\" class=\"dataframe\">\n",97 " <thead>\n",98 " <tr style=\"text-align: right;\">\n",99 " <th></th>\n",100 " <th>text</th>\n",101 " <th>label</th>\n",102 " </tr>\n",103 " </thead>\n",104 " <tbody>\n",105 " <tr>\n",106 " <th>0</th>\n",107 " <td>bayyeessa itti nu dabaala Mee</td>\n",108 " <td>1</td>\n",109 " </tr>\n",110 " <tr>\n",111 " <th>1</th>\n",112 " <td>Gaariidha halumaa kanaan itti fufaa</td>\n",113 " <td>1</td>\n",114 " </tr>\n",115 " <tr>\n",116 " <th>2</th>\n",117 " <td>Guddoo dansaa</td>\n",118 " <td>1</td>\n",119 " </tr>\n",120 " <tr>\n",121 " <th>3</th>\n",122 " <td>osoo addaa hin cittiin yoo barreesitaan gaarii...</td>\n",123 " <td>1</td>\n",124 " </tr>\n",125 " <tr>\n",126 " <th>4</th>\n",127 " <td>Gaaridha itti fufaa</td>\n",128 " <td>1</td>\n",129 " </tr>\n",130 " </tbody>\n",131 "</table>\n",132 "</div>"133 ],134 "text/plain": [135 " text label\n",136 "0 bayyeessa itti nu dabaala Mee 1\n",137 "1 Gaariidha halumaa kanaan itti fufaa 1\n",138 "2 Guddoo dansaa 1\n",139 "3 osoo addaa hin cittiin yoo barreesitaan gaarii... 1\n",140 "4 Gaaridha itti fufaa 1"141 ]142 },143 "execution_count": 2,144 "metadata": {},145 "output_type": "execute_result"146 }147 ],148 "source": [149 "#load csv file\n",150 "Train = pd.read_csv(\"Posts_comment2.csv\",names=['text','label'])\n",151 "data=Train\n",152 "data.head()"153 ]154 },155 {156 "cell_type": "code",157 "execution_count": 3,158 "id": "8a7ac055",159 "metadata": {},160 "outputs": [161 {162 "data": {163 "text/html": [164 "<div>\n",165 "<style scoped>\n",166 " .dataframe tbody tr th:only-of-type {\n",167 " vertical-align: middle;\n",168 " }\n",169 "\n",170 " .dataframe tbody tr th {\n",171 " vertical-align: top;\n",172 " }\n",173 "\n",174 " .dataframe thead th {\n",175 " text-align: right;\n",176 " }\n",177 "</style>\n",178 "<table border=\"1\" class=\"dataframe\">\n",179 " <thead>\n",180 " <tr style=\"text-align: right;\">\n",181 " <th></th>\n",182 " <th>stopwords</th>\n",183 " </tr>\n",184 " </thead>\n",185 " <tbody>\n",186 " <tr>\n",187 " <th>0</th>\n",188 " <td>akka</td>\n",189 " </tr>\n",190 " <tr>\n",191 " <th>1</th>\n",192 " <td>akkam</td>\n",193 " </tr>\n",194 " <tr>\n",195 " <th>2</th>\n",196 " <td>Akkasumas</td>\n",197 " </tr>\n",198 " <tr>\n",199 " <th>3</th>\n",200 " <td>Akkuma</td>\n",201 " </tr>\n",202 " <tr>\n",203 " <th>4</th>\n",204 " <td>Ammo</td>\n",205 " </tr>\n",206 " </tbody>\n",207 "</table>\n",208 "</div>"209 ],210 "text/plain": [211 " stopwords\n",212 "0 akka\n",213 "1 akkam\n",214 "2 Akkasumas\n",215 "3 Akkuma\n",216 "4 Ammo"217 ]218 },219 "execution_count": 3,220 "metadata": {},221 "output_type": "execute_result"222 }223 ],224 "source": [225 "aforstopwords = pd.read_csv(\"aforstopwords.txt\")\n",226 "aforstopwords.head()"227 ]228 },229 {230 "cell_type": "code",231 "execution_count": 4,232 "id": "29a5f209",233 "metadata": {},234 "outputs": [235 {236 "data": {237 "text/html": [238 "<div>\n",239 "<style scoped>\n",240 " .dataframe tbody tr th:only-of-type {\n",241 " vertical-align: middle;\n",242 " }\n",243 "\n",244 " .dataframe tbody tr th {\n",245 " vertical-align: top;\n",246 " }\n",247 "\n",248 " .dataframe thead th {\n",249 " text-align: right;\n",250 " }\n",251 "</style>\n",252 "<table border=\"1\" class=\"dataframe\">\n",253 " <thead>\n",254 " <tr style=\"text-align: right;\">\n",255 " <th></th>\n",256 " <th>text</th>\n",257 " </tr>\n",258 " </thead>\n",259 " <tbody>\n",260 " <tr>\n",261 " <th>0</th>\n",262 " <td>Jabaadha.</td>\n",263 " </tr>\n",264 " <tr>\n",265 " <th>1</th>\n",266 " <td>Odeeffannoo fi Hubannoo nuf uumeera Galaatoma</td>\n",267 " </tr>\n",268 " <tr>\n",269 " <th>2</th>\n",270 " <td>Odeeffannoo gaariidha!</td>\n",271 " </tr>\n",272 " <tr>\n",273 " <th>3</th>\n",274 " <td>Odeeffannoo gaariidha.</td>\n",275 " </tr>\n",276 " <tr>\n",277 " <th>4</th>\n",278 " <td>B/gaariidha Itti Nuuf Fufaa</td>\n",279 " </tr>\n",280 " </tbody>\n",281 "</table>\n",282 "</div>"283 ],284 "text/plain": [285 " text\n",286 "0 Jabaadha.\n",287 "1 Odeeffannoo fi Hubannoo nuf uumeera Galaatoma\n",288 "2 Odeeffannoo gaariidha!\n",289 "3 Odeeffannoo gaariidha.\n",290 "4 B/gaariidha Itti Nuuf Fufaa"291 ]292 },293 "execution_count": 4,294 "metadata": {},295 "output_type": "execute_result"296 }297 ],298 "source": [299 "test_data = pd.read_csv(\"Posts_comment_test.csv\",names=['text'])\n",300 "test_data.head()"301 ]302 },303 {304 "cell_type": "code",305 "execution_count": 5,306 "id": "7f699878",307 "metadata": {},308 "outputs": [309 {310 "data": {311 "text/plain": [312 "274"313 ]314 },315 "execution_count": 5,316 "metadata": {},317 "output_type": "execute_result"318 }319 ],320 "source": [321 "len(test_data)"322 ]323 },324 {325 "cell_type": "code",326 "execution_count": 6,327 "id": "arbitrary-front",328 "metadata": {329 "execution": {330 "iopub.execute_input": "2021-06-29T16:53:03.885521Z",331 "iopub.status.busy": "2021-06-29T16:53:03.884553Z",332 "iopub.status.idle": "2021-06-29T16:53:03.896060Z",333 "shell.execute_reply": "2021-06-29T16:53:03.896605Z",334 "shell.execute_reply.started": "2021-06-29T15:57:06.429211Z"335 },336 "papermill": {337 "duration": 0.069046,338 "end_time": "2021-06-29T16:53:03.896797",339 "exception": false,340 "start_time": "2021-06-29T16:53:03.827751",341 "status": "completed"342 },343 "tags": []344 },345 "outputs": [346 {347 "data": {348 "text/plain": [349 "1 377\n",350 "0 272\n",351 "Name: label, dtype: int64"352 ]353 },354 "execution_count": 6,355 "metadata": {},356 "output_type": "execute_result"357 }358 ],359 "source": [360 "data.label.value_counts()"361 ]362 },363 {364 "cell_type": "code",365 "execution_count": 7,366 "id": "cubic-chile",367 "metadata": {368 "execution": {369 "iopub.execute_input": "2021-06-29T16:53:04.011469Z",370 "iopub.status.busy": "2021-06-29T16:53:04.010477Z",371 "iopub.status.idle": "2021-06-29T16:53:04.016260Z",372 "shell.execute_reply": "2021-06-29T16:53:04.016760Z",373 "shell.execute_reply.started": "2021-06-29T15:57:06.440478Z"374 },375 "papermill": {376 "duration": 0.064911,377 "end_time": "2021-06-29T16:53:04.016942",378 "exception": false,379 "start_time": "2021-06-29T16:53:03.952031",380 "status": "completed"381 },382 "tags": []383 },384 "outputs": [385 {386 "data": {387 "text/plain": [388 "(649, 2)"389 ]390 },391 "execution_count": 7,392 "metadata": {},393 "output_type": "execute_result"394 }395 ],396 "source": [397 "data.shape"398 ]399 },400 {401 "cell_type": "code",402 "execution_count": 8,403 "id": "regulated-wisconsin",404 "metadata": {405 "execution": {406 "iopub.execute_input": "2021-06-29T16:53:04.134525Z",407 "iopub.status.busy": "2021-06-29T16:53:04.133842Z",408 "iopub.status.idle": "2021-06-29T16:53:04.139826Z",409 "shell.execute_reply": "2021-06-29T16:53:04.140258Z",410 "shell.execute_reply.started": "2021-06-29T15:57:06.453852Z"411 },412 "papermill": {413 "duration": 0.067293,414 "end_time": "2021-06-29T16:53:04.140444",415 "exception": false,416 "start_time": "2021-06-29T16:53:04.073151",417 "status": "completed"418 },419 "tags": []420 },421 "outputs": [422 {423 "data": {424 "text/plain": [425 "1 377\n",426 "0 272\n",427 "Name: label, dtype: int64"428 ]429 },430 "execution_count": 8,431 "metadata": {},432 "output_type": "execute_result"433 }434 ],435 "source": [436 "data['label'].value_counts()"437 ]438 },439 {440 "cell_type": "code",441 "execution_count": 9,442 "id": "drawn-wagon",443 "metadata": {444 "execution": {445 "iopub.execute_input": "2021-06-29T16:53:04.256512Z",446 "iopub.status.busy": "2021-06-29T16:53:04.255695Z",447 "iopub.status.idle": "2021-06-29T16:53:05.218496Z",448 "shell.execute_reply": "2021-06-29T16:53:05.219162Z",449 "shell.execute_reply.started": "2021-06-29T15:57:06.469938Z"450 },451 "papermill": {452 "duration": 1.023265,453 "end_time": "2021-06-29T16:53:05.219340",454 "exception": false,455 "start_time": "2021-06-29T16:53:04.196075",456 "status": "completed"457 },458 "tags": []459 },460 "outputs": [461 {462 "name": "stderr",463 "output_type": "stream",464 "text": [465 "C:\\Users\\ENVY\\.conda\\envs\\tensorflow\\lib\\site-packages\\seaborn\\_decorators.py:43: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.\n",466 " FutureWarning\n"467 ]468 },469 {470 "data": {471 "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAQs0lEQVR4nO3df6ieZ33H8fenadfq7GZLTruYpEspUUidxnnINoWhVtZO2KLSSsp02VaIsLopyLD1j6kbGY5ZRUQLEWujc3bBH2smTheDP3BzxpMS26YxGGzXHpMlx1/U/mG2xO/+OHeuPiYnp0/T3s9zkvN+wcNz39d9XffzDRzy4f513akqJEkCOG/cBUiSFg5DQZLUGAqSpMZQkCQ1hoIkqTl/3AU8FUuXLq1Vq1aNuwxJOqvs3r37B1U1Mde2szoUVq1axdTU1LjLkKSzSpL/Pt02Tx9JkhpDQZLUGAqSpMZQkCQ1hoIkqTEUJEmNoSBJagwFSVJjKEiSmrP6iWbpXPbw3/zGuEvQAnTFX9/X6/49UpAkNYaCJKkxFCRJjaEgSWoMBUlSYyhIkhpDQZLUGAqSpMZQkCQ1hoIkqektFJJclGRXkm8n2ZvkXV37O5N8P8me7vOqgTG3JjmQZH+Sa/uqTZI0tz7nPjoKvKKqHktyAfD1JP/WbXtfVb1nsHOSNcAG4GrgOcCXkjy3qo73WKMkaUBvRwo167Fu9YLuU/MMWQ/cVVVHq+pB4ACwrq/6JEmn6vWaQpIlSfYAR4AdVfXNbtObktyb5I4kl3Rty4FHBoZPd22SpBHpNRSq6nhVrQVWAOuSPB+4HbgKWAscAm7rumeuXZzckGRTkqkkUzMzM73ULUmL1UjuPqqqnwBfAa6rqsNdWPwc+DCPnyKaBlYODFsBHJxjX1uqarKqJicmJvotXJIWmT7vPppI8uxu+RnAK4HvJFk20O01wP3d8nZgQ5ILk1wJrAZ29VWfJOlUfd59tAzYmmQJs+Gzrao+l+TjSdYye2roIeCNAFW1N8k24AHgGHCzdx5J0mj1FgpVdS/wojna3zDPmM3A5r5qkiTNzyeaJUmNoSBJagwFSVJjKEiSGkNBktQYCpKkxlCQJDWGgiSpMRQkSY2hIElqDAVJUmMoSJIaQ0GS1BgKkqTGUJAkNYaCJKkxFCRJjaEgSWoMBUlS01soJLkoya4k306yN8m7uvZLk+xI8t3u+5KBMbcmOZBkf5Jr+6pNkjS3Po8UjgKvqKoXAmuB65L8NnALsLOqVgM7u3WSrAE2AFcD1wEfSrKkx/okSSfpLRRq1mPd6gXdp4D1wNaufSvw6m55PXBXVR2tqgeBA8C6vuqTJJ2q12sKSZYk2QMcAXZU1TeBy6vqEED3fVnXfTnwyMDw6a7t5H1uSjKVZGpmZqbP8iVp0ek1FKrqeFWtBVYA65I8f57umWsXc+xzS1VNVtXkxMTE01SpJAlGdPdRVf0E+Aqz1woOJ1kG0H0f6bpNAysHhq0ADo6iPknSrD7vPppI8uxu+RnAK4HvANuBjV23jcDd3fJ2YEOSC5NcCawGdvVVnyTpVOf3uO9lwNbuDqLzgG1V9bkk3wC2JbkJeBi4AaCq9ibZBjwAHANurqrjPdYnSTpJb6FQVfcCL5qj/YfANacZsxnY3FdNkqT5+USzJKkxFCRJjaEgSWoMBUlSYyhIkhpDQZLUGAqSpMZQkCQ1hoIkqelzmouzwov/6mPjLkEL0O5/+ONxlyCNhUcKkqTGUJAkNYaCJKkxFCRJjaEgSWoMBUlSYyhIkhpDQZLU9BYKSVYm+XKSfUn2Jnlz1/7OJN9Psqf7vGpgzK1JDiTZn+TavmqTJM2tzyeajwFvrap7klwM7E6yo9v2vqp6z2DnJGuADcDVwHOALyV5blUd77FGSdKA3o4UqupQVd3TLf8U2Acsn2fIeuCuqjpaVQ8CB4B1fdUnSTrVSK4pJFkFvAj4Ztf0piT3JrkjySVd23LgkYFh08wRIkk2JZlKMjUzM9Nn2ZK06PQeCkmeBXwaeEtVPQrcDlwFrAUOAbed6DrH8DqloWpLVU1W1eTExEQ/RUvSItVrKCS5gNlA+ERVfQagqg5X1fGq+jnwYR4/RTQNrBwYvgI42Gd9kqRf1OfdRwE+AuyrqvcOtC8b6PYa4P5ueTuwIcmFSa4EVgO7+qpPknSqPu8+einwBuC+JHu6trcDNyZZy+ypoYeANwJU1d4k24AHmL1z6WbvPJKk0eotFKrq68x9neDz84zZDGzuqyZJ0vx8olmS1BgKkqTGUJAkNYaCJKkxFCRJjaEgSWoMBUlSYyhIkhpDQZLUGAqSpMZQkCQ1hoIkqRkqFJLsHKZNknR2m3eW1CQXAc8ElnavzTwx6+mvAM/puTZJ0og90dTZbwTewmwA7ObxUHgU+GB/ZUmSxmHeUKiq9wPvT/IXVfWBEdUkSRqToV6yU1UfSPISYNXgmKr6WE91SZLGYKhQSPJx4CpgD3DiFZkFGAqSdA4Z9nWck8Caqqo+i5EkjdewzyncD/zak9lxkpVJvpxkX5K9Sd7ctV+aZEeS73bflwyMuTXJgST7k1z7ZH5PkvTUDXuksBR4IMku4OiJxqr6w3nGHAPeWlX3JLkY2J1kB/AnwM6qeneSW4BbgLclWQNsAK5m9m6nLyV5blUdP83+JUlPs2FD4Z1PdsdVdQg41C3/NMk+YDmwHnhZ120r8BXgbV37XVV1FHgwyQFgHfCNJ/vbkqQzM+zdR199Kj+SZBXwIuCbwOVdYFBVh5Jc1nVbDvzXwLDpru3kfW0CNgFcccUVT6UsSdJJhp3m4qdJHu0+P0tyPMmjQ459FvBp4C1VNd+YzNF2yoXtqtpSVZNVNTkxMTFMCZKkIQ17pHDx4HqSVzN7amdeSS5gNhA+UVWf6ZoPJ1nWHSUsA4507dPAyoHhK4CDw9QnSXp6nNEsqVX1L8Ar5uuTJMBHgH1V9d6BTduBjd3yRuDugfYNSS5MciWwGth1JvVJks7MsA+vvXZg9Txmn1t4omcWXgq8AbgvyZ6u7e3Au4FtSW4CHgZuAKiqvUm2AQ8we+fSzd55JEmjNezdR38wsHwMeIjZu4VOq6q+ztzXCQCuOc2YzcDmIWuSJD3Nhr2m8Kd9FyJJGr9h7z5akeSzSY4kOZzk00lW9F2cJGm0hr3Q/FFmLwQ/h9lnB/61a5MknUOGDYWJqvpoVR3rPncCPiQgSeeYYUPhB0len2RJ93k98MM+C5Mkjd6wofBnwOuA/2F2PqPrAS8+S9I5ZthbUv8W2FhVP4bZ6a+B9zAbFpKkc8SwRwovOBEIAFX1I2YnuJMknUOGDYXzTnoZzqUMf5QhSTpLDPsf+23Afyb5FLPTW7wOnzyWpHPOsE80fyzJFLOT4AV4bVU90GtlkqSRG/oUUBcCBoEkncPOaOpsSdK5yVCQJDWGgiSpMRQkSY2hIElqDAVJUtNbKCS5o3spz/0Dbe9M8v0ke7rPqwa23ZrkQJL9Sa7tqy5J0un1eaRwJ3DdHO3vq6q13efzAEnWABuAq7sxH0qypMfaJElz6C0UquprwI+G7L4euKuqjlbVg8ABYF1ftUmS5jaOawpvSnJvd3rpxCR7y4FHBvpMd22nSLIpyVSSqZmZmb5rlaRFZdShcDtwFbCW2Zf13Na1Z46+NdcOqmpLVU1W1eTEhG8ElaSn00hDoaoOV9Xxqvo58GEeP0U0Dawc6LoCODjK2iRJIw6FJMsGVl8DnLgzaTuwIcmFSa4EVgO7RlmbJKnHF+Uk+STwMmBpkmngHcDLkqxl9tTQQ8AbAapqb5JtzM7Cegy4uaqO91WbJGluvYVCVd04R/NH5um/GV/cI0lj5RPNkqTGUJAkNYaCJKkxFCRJjaEgSWoMBUlSYyhIkhpDQZLUGAqSpMZQkCQ1hoIkqTEUJEmNoSBJagwFSVJjKEiSGkNBktQYCpKkxlCQJDWGgiSp6S0UktyR5EiS+wfaLk2yI8l3u+9LBrbdmuRAkv1Jru2rLknS6fV5pHAncN1JbbcAO6tqNbCzWyfJGmADcHU35kNJlvRYmyRpDr2FQlV9DfjRSc3rga3d8lbg1QPtd1XV0ap6EDgArOurNknS3EZ9TeHyqjoE0H1f1rUvBx4Z6DfdtZ0iyaYkU0mmZmZmei1WkhabhXKhOXO01Vwdq2pLVU1W1eTExETPZUnS4jLqUDicZBlA932ka58GVg70WwEcHHFtkrTojToUtgMbu+WNwN0D7RuSXJjkSmA1sGvEtUnSond+XztO8kngZcDSJNPAO4B3A9uS3AQ8DNwAUFV7k2wDHgCOATdX1fG+apMkza23UKiqG0+z6ZrT9N8MbO6rHknSE1soF5olSQuAoSBJagwFSVJjKEiSGkNBktQYCpKkxlCQJDWGgiSpMRQkSY2hIElqDAVJUmMoSJIaQ0GS1BgKkqTGUJAkNYaCJKkxFCRJjaEgSWp6ex3nfJI8BPwUOA4cq6rJJJcC/wysAh4CXldVPx5HfZK0WI3zSOHlVbW2qia79VuAnVW1GtjZrUuSRmghnT5aD2ztlrcCrx5fKZK0OI0rFAr49yS7k2zq2i6vqkMA3fdlcw1MsinJVJKpmZmZEZUrSYvDWK4pAC+tqoNJLgN2JPnOsAOraguwBWBycrL6KlCSFqOxHClU1cHu+wjwWWAdcDjJMoDu+8g4apOkxWzkoZDkl5NcfGIZ+D3gfmA7sLHrthG4e9S1SdJiN47TR5cDn01y4vf/qaq+kORbwLYkNwEPAzeMoTZJWtRGHgpV9T3ghXO0/xC4ZtT1SJIet5BuSZUkjZmhIElqDAVJUmMoSJIaQ0GS1BgKkqTGUJAkNYaCJKkxFCRJjaEgSWoMBUlSYyhIkhpDQZLUGAqSpMZQkCQ1hoIkqTEUJEmNoSBJagwFSVKz4EIhyXVJ9ic5kOSWcdcjSYvJggqFJEuADwK/D6wBbkyyZrxVSdLisaBCAVgHHKiq71XV/wJ3AevHXJMkLRrnj7uAkywHHhlYnwZ+a7BDkk3Apm71sST7R1TbYrAU+MG4i1gI8p6N4y5Bv8i/zRPekadjL79+ug0LLRTm+tfWL6xUbQG2jKacxSXJVFVNjrsO6WT+bY7OQjt9NA2sHFhfARwcUy2StOgstFD4FrA6yZVJfgnYAGwfc02StGgsqNNHVXUsyZuALwJLgDuqau+Yy1pMPC2nhcq/zRFJVT1xL0nSorDQTh9JksbIUJAkNYaCnFpEC1aSO5IcSXL/uGtZLAyFRc6pRbTA3QlcN+4iFhNDQU4togWrqr4G/GjcdSwmhoLmmlpk+ZhqkTRmhoKecGoRSYuHoSCnFpHUGApyahFJjaGwyFXVMeDE1CL7gG1OLaKFIskngW8Az0syneSmcdd0rnOaC0lS45GCJKkxFCRJjaEgSWoMBUlSYyhIkhpDQXoSkjz2BNtXPdkZPZPcmeT6p1aZ9PQwFCRJjaEgnYEkz0qyM8k9Se5LMjiz7PlJtia5N8mnkjyzG/PiJF9NsjvJF5MsG1P50mkZCtKZ+Rnwmqr6TeDlwG1JTkwu+DxgS1W9AHgU+PMkFwAfAK6vqhcDdwCbx1C3NK/zx12AdJYK8HdJfhf4ObPTjV/ebXukqv6jW/5H4C+BLwDPB3Z02bEEODTSiqUhGArSmfkjYAJ4cVX9X5KHgIu6bSfPHVPMhsjeqvqd0ZUoPXmePpLOzK8CR7pAeDnw6wPbrkhy4j//G4GvA/uBiRPtSS5IcvVIK5aGYChIZ+YTwGSSKWaPGr4zsG0fsDHJvcClwO3dq06vB/4+ybeBPcBLRluy9MScJVWS1HikIElqDAVJUmMoSJIaQ0GS1BgKkqTGUJAkNYaCJKn5fyRrTNYjzxGZAAAAAElFTkSuQmCC\n",472 "text/plain": [473 "<Figure size 432x288 with 1 Axes>"474 ]475 },476 "metadata": {477 "needs_background": "light"478 },479 "output_type": "display_data"480 }481 ],482 "source": [483 "import seaborn as sns\n",484 "ax=sns.countplot(data.label)"485 ]486 },487 {488 "cell_type": "code",489 "execution_count": 10,490 "id": "regulation-naples",491 "metadata": {492 "execution": {493 "iopub.execute_input": "2021-06-29T16:53:05.366022Z",494 "iopub.status.busy": "2021-06-29T16:53:05.360777Z",495 "iopub.status.idle": "2021-06-29T16:53:05.385266Z",496 "shell.execute_reply": "2021-06-29T16:53:05.384717Z",497 "shell.execute_reply.started": "2021-06-29T15:57:07.639082Z"498 },499 "papermill": {500 "duration": 0.108781,501 "end_time": "2021-06-29T16:53:05.385427",502 "exception": false,503 "start_time": "2021-06-29T16:53:05.276646",504 "status": "completed"505 },506 "tags": []507 },508 "outputs": [509 {510 "data": {511 "text/html": [512 "<div>\n",513 "<style scoped>\n",514 " .dataframe tbody tr th:only-of-type {\n",515 " vertical-align: middle;\n",516 " }\n",517 "\n",518 " .dataframe tbody tr th {\n",519 " vertical-align: top;\n",520 " }\n",521 "\n",522 " .dataframe thead th {\n",523 " text-align: right;\n",524 " }\n",525 "</style>\n",526 "<table border=\"1\" class=\"dataframe\">\n",527 " <thead>\n",528 " <tr style=\"text-align: right;\">\n",529 " <th></th>\n",530 " <th>text</th>\n",531 " <th>label</th>\n",532 " </tr>\n",533 " </thead>\n",534 " <tbody>\n",535 " <tr>\n",536 " <th>0</th>\n",537 " <td>bayyeessa itti nu dabaala Mee</td>\n",538 " <td>1</td>\n",539 " </tr>\n",540 " <tr>\n",541 " <th>1</th>\n",542 " <td>Gaariidha halumaa kanaan itti fufaa</td>\n",543 " <td>1</td>\n",544 " </tr>\n",545 " <tr>\n",546 " <th>2</th>\n",547 " <td>Guddoo dansaa</td>\n",548 " <td>1</td>\n",549 " </tr>\n",550 " <tr>\n",551 " <th>3</th>\n",552 " <td>osoo addaa hin cittiin yoo barreesitaan gaarii...</td>\n",553 " <td>1</td>\n",554 " </tr>\n",555 " <tr>\n",556 " <th>4</th>\n",557 " <td>Gaaridha itti fufaa</td>\n",558 " <td>1</td>\n",559 " </tr>\n",560 " </tbody>\n",561 "</table>\n",562 "</div>"563 ],564 "text/plain": [565 " text label\n",566 "0 bayyeessa itti nu dabaala Mee 1\n",567 "1 Gaariidha halumaa kanaan itti fufaa 1\n",568 "2 Guddoo dansaa 1\n",569 "3 osoo addaa hin cittiin yoo barreesitaan gaarii... 1\n",570 "4 Gaaridha itti fufaa 1"571 ]572 },573 "execution_count": 10,574 "metadata": {},575 "output_type": "execute_result"576 }577 ],578 "source": [579 "# removing usernames from tweet\n",580 "\n",581 "data['text'] = data.text.str.replace('@user', '')\n",582 "data.head()"583 ]584 },585 {586 "cell_type": "code",587 "execution_count": 11,588 "id": "industrial-palestine",589 "metadata": {590 "execution": {591 "iopub.execute_input": "2021-06-29T16:53:05.654996Z",592 "iopub.status.busy": "2021-06-29T16:53:05.648080Z",593 "iopub.status.idle": "2021-06-29T16:53:05.982089Z",594 "shell.execute_reply": "2021-06-29T16:53:05.981557Z",595 "shell.execute_reply.started": "2021-06-29T15:57:07.718486Z"596 },597 "papermill": {598 "duration": 0.39935,599 "end_time": "2021-06-29T16:53:05.982223",600 "exception": false,601 "start_time": "2021-06-29T16:53:05.582873",602 "status": "completed"603 },604 "tags": []605 },606 "outputs": [607 {608 "name": "stderr",609 "output_type": "stream",610 "text": [611 "C:\\Users\\ENVY\\.conda\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:6: FutureWarning: The default value of regex will change from True to False in a future version.\n",612 " \n"613 ]614 },615 {616 "data": {617 "text/html": [618 "<div>\n",619 "<style scoped>\n",620 " .dataframe tbody tr th:only-of-type {\n",621 " vertical-align: middle;\n",622 " }\n",623 "\n",624 " .dataframe tbody tr th {\n",625 " vertical-align: top;\n",626 " }\n",627 "\n",628 " .dataframe thead th {\n",629 " text-align: right;\n",630 " }\n",631 "</style>\n",632 "<table border=\"1\" class=\"dataframe\">\n",633 " <thead>\n",634 " <tr style=\"text-align: right;\">\n",635 " <th></th>\n",636 " <th>text</th>\n",637 " <th>label</th>\n",638 " </tr>\n",639 " </thead>\n",640 " <tbody>\n",641 " <tr>\n",642 " <th>0</th>\n",643 " <td>bayyeessa itti nu dabaala Mee</td>\n",644 " <td>1</td>\n",645 " </tr>\n",646 " <tr>\n",647 " <th>1</th>\n",648 " <td>Gaariidha halumaa kanaan itti fufaa</td>\n",649 " <td>1</td>\n",650 " </tr>\n",651 " <tr>\n",652 " <th>2</th>\n",653 " <td>Guddoo dansaa</td>\n",654 " <td>1</td>\n",655 " </tr>\n",656 " <tr>\n",657 " <th>3</th>\n",658 " <td>osoo addaa hin cittiin yoo barreesitaan gaarii...</td>\n",659 " <td>1</td>\n",660 " </tr>\n",661 " <tr>\n",662 " <th>4</th>\n",663 " <td>Gaaridha itti fufaa</td>\n",664 " <td>1</td>\n",665 " </tr>\n",666 " </tbody>\n",667 "</table>\n",668 "</div>"669 ],670 "text/plain": [671 " text label\n",672 "0 bayyeessa itti nu dabaala Mee 1\n",673 "1 Gaariidha halumaa kanaan itti fufaa 1\n",674 "2 Guddoo dansaa 1\n",675 "3 osoo addaa hin cittiin yoo barreesitaan gaarii... 1\n",676 "4 Gaaridha itti fufaa 1"677 ]678 },679 "execution_count": 11,680 "metadata": {},681 "output_type": "execute_result"682 }683 ],684 "source": [685 "#Removing Punctuations, Numbers, and Special Characters\n",686 "#[a-zA-Z] = Any single character in the range a-z or A-Z\n",687 "# ^ = Start of line \n",688 "# $ = End of line \n",689 "\n",690 "data['text'] = data['text'].str.replace(\"[^a-zA-Z#]\", \" \")\n",691 "data['text'] = data['text'].str.replace(\"#\", \"\")\n",692 "data.head()\n"693 ]694 },695 {696 "cell_type": "code",697 "execution_count": 12,698 "id": "neutral-bargain",699 "metadata": {700 "execution": {701 "iopub.execute_input": "2021-06-29T16:53:06.122857Z",702 "iopub.status.busy": "2021-06-29T16:53:06.107267Z",703 "iopub.status.idle": "2021-06-29T16:53:06.301142Z",704 "shell.execute_reply": "2021-06-29T16:53:06.301953Z",705 "shell.execute_reply.started": "2021-06-29T15:57:08.084397Z"706 },707 "papermill": {708 "duration": 0.261657,709 "end_time": "2021-06-29T16:53:06.302153",710 "exception": false,711 "start_time": "2021-06-29T16:53:06.040496",712 "status": "completed"713 },714 "tags": []715 },716 "outputs": [717 {718 "name": "stderr",719 "output_type": "stream",720 "text": [721 "C:\\Users\\ENVY\\.conda\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: The default value of regex will change from True to False in a future version.\n",722 " \"\"\"Entry point for launching an IPython kernel.\n"723 ]724 },725 {726 "data": {727 "text/html": [728 "<div>\n",729 "<style scoped>\n",730 " .dataframe tbody tr th:only-of-type {\n",731 " vertical-align: middle;\n",732 " }\n",733 "\n",734 " .dataframe tbody tr th {\n",735 " vertical-align: top;\n",736 " }\n",737 "\n",738 " .dataframe thead th {\n",739 " text-align: right;\n",740 " }\n",741 "</style>\n",742 "<table border=\"1\" class=\"dataframe\">\n",743 " <thead>\n",744 " <tr style=\"text-align: right;\">\n",745 " <th></th>\n",746 " <th>text</th>\n",747 " </tr>\n",748 " </thead>\n",749 " <tbody>\n",750 " <tr>\n",751 " <th>0</th>\n",752 " <td>Jabaadha</td>\n",753 " </tr>\n",754 " <tr>\n",755 " <th>1</th>\n",756 " <td>Odeeffannoo fi Hubannoo nuf uumeera Galaatoma</td>\n",757 " </tr>\n",758 " <tr>\n",759 " <th>2</th>\n",760 " <td>Odeeffannoo gaariidha</td>\n",761 " </tr>\n",762 " <tr>\n",763 " <th>3</th>\n",764 " <td>Odeeffannoo gaariidha</td>\n",765 " </tr>\n",766 " <tr>\n",767 " <th>4</th>\n",768 " <td>B gaariidha Itti Nuuf Fufaa</td>\n",769 " </tr>\n",770 " </tbody>\n",771 "</table>\n",772 "</div>"773 ],774 "text/plain": [775 " text\n",776 "0 Jabaadha \n",777 "1 Odeeffannoo fi Hubannoo nuf uumeera Galaatoma\n",778 "2 Odeeffannoo gaariidha \n",779 "3 Odeeffannoo gaariidha \n",780 "4 B gaariidha Itti Nuuf Fufaa"781 ]782 },783 "execution_count": 12,784 "metadata": {},785 "output_type": "execute_result"786 }787 ],788 "source": [789 "test_data['text'] = test_data['text'].str.replace(\"[^a-zA-Z#]\", \" \")\n",790 "test_data['text'] = test_data['text'].str.replace(\"#\", \"\")\n",791 "test_data.head()"792 ]793 },794 {795 "cell_type": "code",796 "execution_count": 13,797 "id": "expected-freeze",798 "metadata": {799 "execution": {800 "iopub.execute_input": "2021-06-29T16:53:06.443572Z",801 "iopub.status.busy": "2021-06-29T16:53:06.442784Z",802 "iopub.status.idle": "2021-06-29T16:53:06.661997Z",803 "shell.execute_reply": "2021-06-29T16:53:06.660921Z",804 "shell.execute_reply.started": "2021-06-29T15:57:08.288388Z"805 },806 "papermill": {807 "duration": 0.299334,808 "end_time": "2021-06-29T16:53:06.662200",809 "exception": false,810 "start_time": "2021-06-29T16:53:06.362866",811 "status": "completed"812 },813 "tags": []814 },815 "outputs": [816 {817 "data": {818 "text/plain": [819 "[('hin', 88),\n",820 " ('gaariidha', 69),\n",821 " ('itti', 59),\n",822 " ('kun', 58),\n",823 " ('ta', 53),\n",824 " ('waan', 52),\n",825 " ('fi', 52),\n",826 " ('ee', 50),\n",827 " ('kan', 41),\n",828 " ('dha', 40)]"829 ]830 },831 "execution_count": 13,832 "metadata": {},833 "output_type": "execute_result"834 }835 ],836 "source": [837 "# get most common words in training dataset\n",838 "from collections import Counter \n",839 "all_words = []\n",840 "for line in list(data['text']):\n",841 " words = line.split()\n",842 " for word in words:\n",843 " all_words.append(word.lower())\n",844 " \n",845 " \n",846 "a=Counter(all_words).most_common(10)\n",847 "a"848 ]849 },850 {851 "cell_type": "code",852 "execution_count": 14,853 "id": "contained-round",854 "metadata": {855 "execution": {856 "iopub.execute_input": "2021-06-29T16:53:06.785618Z",857 "iopub.status.busy": "2021-06-29T16:53:06.784907Z",858 "iopub.status.idle": "2021-06-29T16:53:06.986588Z",859 "shell.execute_reply": "2021-06-29T16:53:06.987028Z",860 "shell.execute_reply.started": "2021-06-29T15:57:08.514314Z"861 },862 "papermill": {863 "duration": 0.266643,864 "end_time": "2021-06-29T16:53:06.987210",865 "exception": false,866 "start_time": "2021-06-29T16:53:06.720567",867 "status": "completed"868 },869 "tags": []870 },871 "outputs": [872 {873 "data": {874 "text/html": [875 "<div>\n",876 "<style scoped>\n",877 " .dataframe tbody tr th:only-of-type {\n",878 " vertical-align: middle;\n",879 " }\n",880 "\n",881 " .dataframe tbody tr th {\n",882 " vertical-align: top;\n",883 " }\n",884 "\n",885 " .dataframe thead th {\n",886 " text-align: right;\n",887 " }\n",888 "</style>\n",889 "<table border=\"1\" class=\"dataframe\">\n",890 " <thead>\n",891 " <tr style=\"text-align: right;\">\n",892 " <th></th>\n",893 " <th>text</th>\n",894 " <th>label</th>\n",895 " </tr>\n",896 " </thead>\n",897 " <tbody>\n",898 " <tr>\n",899 " <th>0</th>\n",900 " <td>[bayyeessa, itti, nu, dabaala, Mee]</td>\n",901 " <td>1</td>\n",902 " </tr>\n",903 " <tr>\n",904 " <th>1</th>\n",905 " <td>[Gaariidha, halumaa, kanaan, itti, fufaa]</td>\n",906 " <td>1</td>\n",907 " </tr>\n",908 " <tr>\n",909 " <th>2</th>\n",910 " <td>[Guddoo, dansaa]</td>\n",911 " <td>1</td>\n",912 " </tr>\n",913 " <tr>\n",914 " <th>3</th>\n",915 " <td>[osoo, addaa, hin, cittiin, yoo, barreesitaan,...</td>\n",916 " <td>1</td>\n",917 " </tr>\n",918 " <tr>\n",919 " <th>4</th>\n",920 " <td>[Gaaridha, itti, fufaa]</td>\n",921 " <td>1</td>\n",922 " </tr>\n",923 " </tbody>\n",924 "</table>\n",925 "</div>"926 ],927 "text/plain": [928 " text label\n",929 "0 [bayyeessa, itti, nu, dabaala, Mee] 1\n",930 "1 [Gaariidha, halumaa, kanaan, itti, fufaa] 1\n",931 "2 [Guddoo, dansaa] 1\n",932 "3 [osoo, addaa, hin, cittiin, yoo, barreesitaan,... 1\n",933 "4 [Gaaridha, itti, fufaa] 1"934 ]935 },936 "execution_count": 14,937 "metadata": {},938 "output_type": "execute_result"939 }940 ],941 "source": [942 "#tokenization\n",943 "data['text'] = data['text'].apply(lambda x: x.split())\n",944 "data.head()"945 ]946 },947 {948 "cell_type": "code",949 "execution_count": 15,950 "id": "desperate-worship",951 "metadata": {952 "execution": {953 "iopub.execute_input": "2021-06-29T16:53:07.110449Z",954 "iopub.status.busy": "2021-06-29T16:53:07.109768Z",955 "iopub.status.idle": "2021-06-29T16:53:07.160474Z",956 "shell.execute_reply": "2021-06-29T16:53:07.161052Z",957 "shell.execute_reply.started": "2021-06-29T15:57:08.720726Z"958 },959 "papermill": {960 "duration": 0.114147,961 "end_time": "2021-06-29T16:53:07.161231",962 "exception": false,963 "start_time": "2021-06-29T16:53:07.047084",964 "status": "completed"965 },966 "tags": []967 },968 "outputs": [969 {970 "data": {971 "text/html": [972 "<div>\n",973 "<style scoped>\n",974 " .dataframe tbody tr th:only-of-type {\n",975 " vertical-align: middle;\n",976 " }\n",977 "\n",978 " .dataframe tbody tr th {\n",979 " vertical-align: top;\n",980 " }\n",981 "\n",982 " .dataframe thead th {\n",983 " text-align: right;\n",984 " }\n",985 "</style>\n",986 "<table border=\"1\" class=\"dataframe\">\n",987 " <thead>\n",988 " <tr style=\"text-align: right;\">\n",989 " <th></th>\n",990 " <th>text</th>\n",991 " </tr>\n",992 " </thead>\n",993 " <tbody>\n",994 " <tr>\n",995 " <th>0</th>\n",996 " <td>[Jabaadha]</td>\n",997 " </tr>\n",998 " <tr>\n",999 " <th>1</th>\n",1000 " <td>[Odeeffannoo, fi, Hubannoo, nuf, uumeera, Gala...</td>\n",1001 " </tr>\n",1002 " <tr>\n",1003 " <th>2</th>\n",1004 " <td>[Odeeffannoo, gaariidha]</td>\n",1005 " </tr>\n",1006 " <tr>\n",1007 " <th>3</th>\n",1008 " <td>[Odeeffannoo, gaariidha]</td>\n",1009 " </tr>\n",1010 " <tr>\n",1011 " <th>4</th>\n",1012 " <td>[B, gaariidha, Itti, Nuuf, Fufaa]</td>\n",1013 " </tr>\n",1014 " </tbody>\n",1015 "</table>\n",1016 "</div>"1017 ],1018 "text/plain": [1019 " text\n",1020 "0 [Jabaadha]\n",1021 "1 [Odeeffannoo, fi, Hubannoo, nuf, uumeera, Gala...\n",1022 "2 [Odeeffannoo, gaariidha]\n",1023 "3 [Odeeffannoo, gaariidha]\n",1024 "4 [B, gaariidha, Itti, Nuuf, Fufaa]"1025 ]1026 },1027 "execution_count": 15,1028 "metadata": {},1029 "output_type": "execute_result"1030 }1031 ],1032 "source": [1033 "test_data['text'] = test_data['text'].apply(lambda x: x.split())\n",1034 "test_data.head()"1035 ]1036 },1037 {1038 "cell_type": "code",1039 "execution_count": 16,1040 "id": "broken-original",1041 "metadata": {1042 "execution": {1043 "iopub.execute_input": "2021-06-29T16:53:07.283146Z",1044 "iopub.status.busy": "2021-06-29T16:53:07.282456Z",1045 "iopub.status.idle": "2021-06-29T16:53:17.697490Z",1046 "shell.execute_reply": "2021-06-29T16:53:17.698010Z",1047 "shell.execute_reply.started": "2021-06-29T15:57:08.777047Z"1048 },1049 "papermill": {1050 "duration": 10.477415,1051 "end_time": "2021-06-29T16:53:17.698230",1052 "exception": false,1053 "start_time": "2021-06-29T16:53:07.220815",1054 "status": "completed"1055 },1056 "tags": []1057 },1058 "outputs": [1059 {1060 "data": {1061 "text/html": [1062 "<div>\n",1063 "<style scoped>\n",1064 " .dataframe tbody tr th:only-of-type {\n",1065 " vertical-align: middle;\n",1066 " }\n",1067 "\n",1068 " .dataframe tbody tr th {\n",1069 " vertical-align: top;\n",1070 " }\n",1071 "\n",1072 " .dataframe thead th {\n",1073 " text-align: right;\n",1074 " }\n",1075 "</style>\n",1076 "<table border=\"1\" class=\"dataframe\">\n",1077 " <thead>\n",1078 " <tr style=\"text-align: right;\">\n",1079 " <th></th>\n",1080 " <th>text</th>\n",1081 " <th>label</th>\n",1082 " </tr>\n",1083 " </thead>\n",1084 " <tbody>\n",1085 " <tr>\n",1086 " <th>0</th>\n",1087 " <td>[bayyeessa, itti, nu, dabaala, mee]</td>\n",1088 " <td>1</td>\n",1089 " </tr>\n",1090 " <tr>\n",1091 " <th>1</th>\n",1092 " <td>[gaariidha, halumaa, kanaan, itti, fufaa]</td>\n",1093 " <td>1</td>\n",1094 " </tr>\n",1095 " <tr>\n",1096 " <th>2</th>\n",1097 " <td>[guddoo, dansaa]</td>\n",1098 " <td>1</td>\n",1099 " </tr>\n",1100 " <tr>\n",1101 " <th>3</th>\n",1102 " <td>[osoo, addaa, hin, cittiin, yoo, barreesitaan,...</td>\n",1103 " <td>1</td>\n",1104 " </tr>\n",1105 " <tr>\n",1106 " <th>4</th>\n",1107 " <td>[gaaridha, itti, fufaa]</td>\n",1108 " <td>1</td>\n",1109 " </tr>\n",1110 " <tr>\n",1111 " <th>5</th>\n",1112 " <td>[waan, salphaa, fakkaata, garuu, hojii, guddaa...</td>\n",1113 " <td>1</td>\n",1114 " </tr>\n",1115 " <tr>\n",1116 " <th>6</th>\n",1117 " <td>[ka, umsa, bareedaadha, itti, fufaa]</td>\n",1118 " <td>1</td>\n",1119 " </tr>\n",1120 " <tr>\n",1121 " <th>7</th>\n",1122 " <td>[bayyeessa]</td>\n",1123 " <td>1</td>\n",1124 " </tr>\n",1125 " <tr>\n",1126 " <th>8</th>\n",1127 " <td>[hojii, sirrii, hojjechuuf, jalqabii, gaarii]</td>\n",1128 " <td>1</td>\n",1129 " </tr>\n",1130 " <tr>\n",1131 " <th>9</th>\n",1132 " <td>[baayye, gaaridha]</td>\n",1133 " <td>1</td>\n",1134 " </tr>\n",1135 " </tbody>\n",1136 "</table>\n",1137 "</div>"1138 ],1139 "text/plain": [1140 " text label\n",1141 "0 [bayyeessa, itti, nu, dabaala, mee] 1\n",1142 "1 [gaariidha, halumaa, kanaan, itti, fufaa] 1\n",1143 "2 [guddoo, dansaa] 1\n",1144 "3 [osoo, addaa, hin, cittiin, yoo, barreesitaan,... 1\n",1145 "4 [gaaridha, itti, fufaa] 1\n",1146 "5 [waan, salphaa, fakkaata, garuu, hojii, guddaa... 1\n",1147 "6 [ka, umsa, bareedaadha, itti, fufaa] 1\n",1148 "7 [bayyeessa] 1\n",1149 "8 [hojii, sirrii, hojjechuuf, jalqabii, gaarii] 1\n",1150 "9 [baayye, gaaridha] 1"1151 ]1152 },1153 "execution_count": 16,1154 "metadata": {},1155 "output_type": "execute_result"1156 }1157 ],1158 "source": [1159 "#stemmer\n",1160 "from nltk.stem import PorterStemmer\n",1161 "stemmer = PorterStemmer()\n",1162 "\n",1163 "data['text']= data['text'].apply(lambda x: [stemmer.stem(i) for i in x])\n",1164 "data.head(10)"1165 ]1166 },1167 {1168 "cell_type": "code",1169 "execution_count": 17,1170 "id": "affiliated-therapist",1171 "metadata": {1172 "execution": {1173 "iopub.execute_input": "2021-06-29T16:53:17.822464Z",1174 "iopub.status.busy": "2021-06-29T16:53:17.821764Z",1175 "iopub.status.idle": "2021-06-29T16:53:22.850129Z",1176 "shell.execute_reply": "2021-06-29T16:53:22.850669Z",1177 "shell.execute_reply.started": "2021-06-29T15:57:19.610466Z"1178 },1179 "papermill": {1180 "duration": 5.091172,1181 "end_time": "2021-06-29T16:53:22.850844",1182 "exception": false,1183 "start_time": "2021-06-29T16:53:17.759672",1184 "status": "completed"1185 },1186 "tags": []1187 },1188 "outputs": [1189 {1190 "data": {1191 "text/html": [1192 "<div>\n",1193 "<style scoped>\n",1194 " .dataframe tbody tr th:only-of-type {\n",1195 " vertical-align: middle;\n",1196 " }\n",1197 "\n",1198 " .dataframe tbody tr th {\n",1199 " vertical-align: top;\n",1200 " }\n",