EMPATHY-S12-GROUP-7/dataset_processing
014
1{2 "cells": [3 {4 "cell_type": "code",5 "execution_count": 40,6 "id": "ccd4f6a4-aab8-4bec-9e9e-1c003f9fa117",7 "metadata": {8 "tags": []9 },10 "outputs": [],11 "source": [12 "import pandas as pd\n",13 "import numpy as np\n",14 "import random"15 ]16 },17 {18 "cell_type": "code",19 "execution_count": 28,20 "id": "c31131b7-5697-4d8d-ae31-e21033fa19c2",21 "metadata": {22 "tags": []23 },24 "outputs": [25 {26 "data": {27 "text/plain": [28 "(788, 7)"29 ]30 },31 "execution_count": 28,32 "metadata": {},33 "output_type": "execute_result"34 }35 ],36 "source": [37 "docDf = pd.read_csv('datasets/cleaned_doctors_data_with_city_province.csv')\n",38 "docDf.shape"39 ]40 },41 {42 "cell_type": "code",43 "execution_count": 29,44 "id": "efc0967f-46f4-4029-a436-87091411ee8e",45 "metadata": {46 "tags": []47 },48 "outputs": [49 {50 "name": "stdout",51 "output_type": "stream",52 "text": [53 "1\n"54 ]55 }56 ],57 "source": [58 "print(docDf['ADDRESS'].isna().sum())"59 ]60 },61 {62 "cell_type": "markdown",63 "id": "372fa39b-bbf3-4d65-8425-8f24318daa2f",64 "metadata": {},65 "source": [66 "Drop entries with null address:"67 ]68 },69 {70 "cell_type": "code",71 "execution_count": 30,72 "id": "1be7df67-ab3a-4edb-8a9a-0f65364f412c",73 "metadata": {74 "tags": []75 },76 "outputs": [],77 "source": [78 "docDf = docDf.dropna(subset=[\"ADDRESS\"])"79 ]80 },81 {82 "cell_type": "markdown",83 "id": "b2f821b5-88dd-4b1a-bf96-6b59d6485534",84 "metadata": {},85 "source": [86 "Drop all doctors specializing in Radiology (does not cater to illnesses):"87 ]88 },89 {90 "cell_type": "code",91 "execution_count": 31,92 "id": "bd322e8d-7f7a-43ee-834c-0d4cc7ea7129",93 "metadata": {94 "tags": []95 },96 "outputs": [],97 "source": [98 "docDf = docDf.drop(docDf[docDf['SPECIALIZATION'] == 'Radiology'].index)"99 ]100 },101 {102 "cell_type": "markdown",103 "id": "e94a0a42-0822-47cb-ab00-53e5372c0298",104 "metadata": {},105 "source": [106 "Drop \"province\" column:"107 ]108 },109 {110 "cell_type": "code",111 "execution_count": 32,112 "id": "67bd02a3-8645-49d0-bc44-6f73a5b90e1d",113 "metadata": {114 "tags": []115 },116 "outputs": [],117 "source": [118 "docDf = docDf.drop(columns=['PROVINCE'])"119 ]120 },121 {122 "cell_type": "code",123 "execution_count": 33,124 "id": "6ad9f775-8bf5-4366-9384-dff80df0e4e5",125 "metadata": {126 "tags": []127 },128 "outputs": [129 {130 "data": {131 "text/html": [132 "<div>\n",133 "<style scoped>\n",134 " .dataframe tbody tr th:only-of-type {\n",135 " vertical-align: middle;\n",136 " }\n",137 "\n",138 " .dataframe tbody tr th {\n",139 " vertical-align: top;\n",140 " }\n",141 "\n",142 " .dataframe thead th {\n",143 " text-align: right;\n",144 " }\n",145 "</style>\n",146 "<table border=\"1\" class=\"dataframe\">\n",147 " <thead>\n",148 " <tr style=\"text-align: right;\">\n",149 " <th></th>\n",150 " <th>NAME</th>\n",151 " <th>SPECIALIZATION</th>\n",152 " <th>PHONE</th>\n",153 " <th>ADDRESS</th>\n",154 " <th>SCHEDULE</th>\n",155 " <th>CITY</th>\n",156 " </tr>\n",157 " </thead>\n",158 " <tbody>\n",159 " <tr>\n",160 " <th>0</th>\n",161 " <td>Miko Angelo De Leon, MD</td>\n",162 " <td>Anesthesia</td>\n",163 " <td>NaN</td>\n",164 " <td>Chong Hua Hospital\\nDon Mariano Cui St., Fuent...</td>\n",165 " <td>on call</td>\n",166 " <td>Maria</td>\n",167 " </tr>\n",168 " <tr>\n",169 " <th>1</th>\n",170 " <td>Cristina V. Formoso-Avendaño, MD</td>\n",171 " <td>Anesthesia</td>\n",172 " <td>NaN</td>\n",173 " <td>St. Luke’s Medical Center – Quezon City\\nPain ...</td>\n",174 " <td>Mon: 8:00AM – 5:00PM\\nTue: 8:00AM – 5:00PM\\nWe...</td>\n",175 " <td>Rodriguez</td>\n",176 " </tr>\n",177 " <tr>\n",178 " <th>2</th>\n",179 " <td>Sherwin Tayag, MD</td>\n",180 " <td>Anesthesia</td>\n",181 " <td>(0922) 876 8435</td>\n",182 " <td>Manila Medical Center\\nUN Avenue, Taft Ave, Er...</td>\n",183 " <td>NaN</td>\n",184 " <td>Taft</td>\n",185 " </tr>\n",186 " <tr>\n",187 " <th>3</th>\n",188 " <td>Maritess Gonzales, MD</td>\n",189 " <td>Anesthesia</td>\n",190 " <td>NaN</td>\n",191 " <td>Metronorth Medical Center\\nMindanao Avenue Que...</td>\n",192 " <td>NaN</td>\n",193 " <td>Quezon</td>\n",194 " </tr>\n",195 " <tr>\n",196 " <th>4</th>\n",197 " <td>Aimee Flor Barrion, MD</td>\n",198 " <td>Anesthesia</td>\n",199 " <td>(02) 771 9000</td>\n",200 " <td>Asian Hospital And Medical Center\\n2205 Civic ...</td>\n",201 " <td>NaN</td>\n",202 " <td>Muntinlupa</td>\n",203 " </tr>\n",204 " </tbody>\n",205 "</table>\n",206 "</div>"207 ],208 "text/plain": [209 " NAME SPECIALIZATION PHONE \\\n",210 "0 Miko Angelo De Leon, MD Anesthesia NaN \n",211 "1 Cristina V. Formoso-Avendaño, MD Anesthesia NaN \n",212 "2 Sherwin Tayag, MD Anesthesia (0922) 876 8435 \n",213 "3 Maritess Gonzales, MD Anesthesia NaN \n",214 "4 Aimee Flor Barrion, MD Anesthesia (02) 771 9000 \n",215 "\n",216 " ADDRESS \\\n",217 "0 Chong Hua Hospital\\nDon Mariano Cui St., Fuent... \n",218 "1 St. Luke’s Medical Center – Quezon City\\nPain ... \n",219 "2 Manila Medical Center\\nUN Avenue, Taft Ave, Er... \n",220 "3 Metronorth Medical Center\\nMindanao Avenue Que... \n",221 "4 Asian Hospital And Medical Center\\n2205 Civic ... \n",222 "\n",223 " SCHEDULE CITY \n",224 "0 on call Maria \n",225 "1 Mon: 8:00AM – 5:00PM\\nTue: 8:00AM – 5:00PM\\nWe... Rodriguez \n",226 "2 NaN Taft \n",227 "3 NaN Quezon \n",228 "4 NaN Muntinlupa "229 ]230 },231 "execution_count": 33,232 "metadata": {},233 "output_type": "execute_result"234 }235 ],236 "source": [237 "docDf.head()"238 ]239 },240 {241 "cell_type": "code",242 "execution_count": 26,243 "id": "8f6f664a-ea7f-41b4-91cf-8d1156140661",244 "metadata": {245 "tags": []246 },247 "outputs": [248 {249 "data": {250 "text/plain": [251 "array(['Maria', 'Rodriguez', 'Taft', 'Quezon', 'Muntinlupa', 'Rizal',\n",252 " 'Manila', 'Santo Tomas', nan, 'Pio Duran', 'Labo', 'Makati',\n",253 " 'Mandaluyong', 'Valenzuela', 'San Juan', 'Ramon', 'Pasay',\n",254 " 'Llorente', 'Pasig', 'Santiago', 'San Roque', 'San Miguel',\n",255 " 'Taguig', 'Asturias', 'Roxas', 'Rosario', 'Lopez Jaena',\n",256 " 'Catarman', 'Tagaytay', 'Sikatuna', 'Balasan', 'Sampaloc',\n",257 " 'Libmanan', 'Magsaysay', 'Marilao', 'Los Baños', 'San Pedro',\n",258 " 'Oton', 'San Vicente', 'Jones', 'Lapu-Lapu', 'Calamba',\n",259 " 'MacArthur', 'Batan', 'San Fernando', 'San Nicolas', 'San Rafael',\n",260 " 'Trinidad', 'Aguilar', 'Isabela', 'Valencia', 'San Dionisio',\n",261 " 'Quirino', 'Batangas City', 'Caba', 'San Jose', 'Bulan', 'Luna',\n",262 " 'Pamplona', 'Subic', 'Kauswagan', 'Mandaue', 'San Marcelino',\n",263 " 'San Carlos', 'Burgos', 'Taytay', 'Pateros', 'Ramos', 'Biñan',\n",264 " 'Las Piñas', 'Marikina', 'Naga', 'Pilar', 'Malolos', 'Jaro',\n",265 " 'Bacolod', 'Lopez', 'Tamparan', 'Iba', 'Mabini', 'Bula', 'Laurel',\n",266 " 'Balanga', 'Daraga', 'Teresa', 'Santa Cruz', 'Dagupan', 'Leyte',\n",267 " 'Agoo', 'Lemery', 'Koronadal', 'Tayum', 'Palanan', 'San Andres',\n",268 " 'San Francisco', 'Victoria', 'San Agustin', 'Carmen', 'Manaoag',\n",269 " 'Balingasag', 'Bacoor', 'Jose Abad Santos', 'San Isidro', 'Ocampo',\n",270 " 'Digos', 'Parañaque', 'Real', 'Baler', 'Malabon', 'Santo Domingo',\n",271 " 'Leon', 'Guiguinto', 'Medina', 'Buguias', 'San Gabriel'],\n",272 " dtype=object)"273 ]274 },275 "execution_count": 26,276 "metadata": {},277 "output_type": "execute_result"278 }279 ],280 "source": [281 "pd.unique(docDf['CITY'])"282 ]283 },284 {285 "cell_type": "code",286 "execution_count": 8,287 "id": "6e507cea-a7fc-43d5-9596-6a0d389a83e4",288 "metadata": {289 "tags": []290 },291 "outputs": [292 {293 "data": {294 "text/html": [295 "<div>\n",296 "<style scoped>\n",297 " .dataframe tbody tr th:only-of-type {\n",298 " vertical-align: middle;\n",299 " }\n",300 "\n",301 " .dataframe tbody tr th {\n",302 " vertical-align: top;\n",303 " }\n",304 "\n",305 " .dataframe thead th {\n",306 " text-align: right;\n",307 " }\n",308 "</style>\n",309 "<table border=\"1\" class=\"dataframe\">\n",310 " <thead>\n",311 " <tr style=\"text-align: right;\">\n",312 " <th></th>\n",313 " <th>SPECIALIZATION</th>\n",314 " <th>ILLNESS</th>\n",315 " <th>SYMPTOMS</th>\n",316 " </tr>\n",317 " </thead>\n",318 " <tbody>\n",319 " <tr>\n",320 " <th>0</th>\n",321 " <td>Anesthesia</td>\n",322 " <td>Local anesthetic toxicity</td>\n",323 " <td>tingling</td>\n",324 " </tr>\n",325 " <tr>\n",326 " <th>1</th>\n",327 " <td>Anesthesia</td>\n",328 " <td>Local anesthetic toxicity</td>\n",329 " <td>seizures</td>\n",330 " </tr>\n",331 " <tr>\n",332 " <th>2</th>\n",333 " <td>Anesthesia</td>\n",334 " <td>Local anesthetic toxicity</td>\n",335 " <td>cardiac arrest</td>\n",336 " </tr>\n",337 " <tr>\n",338 " <th>3</th>\n",339 " <td>Anesthesia</td>\n",340 " <td>Local anesthetic toxicity</td>\n",341 " <td>confusion</td>\n",342 " </tr>\n",343 " <tr>\n",344 " <th>4</th>\n",345 " <td>Anesthesia</td>\n",346 " <td>Local anesthetic toxicity</td>\n",347 " <td>dizziness</td>\n",348 " </tr>\n",349 " </tbody>\n",350 "</table>\n",351 "</div>"352 ],353 "text/plain": [354 " SPECIALIZATION ILLNESS SYMPTOMS\n",355 "0 Anesthesia Local anesthetic toxicity tingling\n",356 "1 Anesthesia Local anesthetic toxicity seizures\n",357 "2 Anesthesia Local anesthetic toxicity cardiac arrest\n",358 "3 Anesthesia Local anesthetic toxicity confusion\n",359 "4 Anesthesia Local anesthetic toxicity dizziness"360 ]361 },362 "execution_count": 8,363 "metadata": {},364 "output_type": "execute_result"365 }366 ],367 "source": [368 "illnessDf = pd.read_csv('datasets/illness_symptom.csv')\n",369 "illnessDf.head()"370 ]371 },372 {373 "cell_type": "code",374 "execution_count": 9,375 "id": "3acb4df1-eb88-4744-9fed-182d22c5f7df",376 "metadata": {377 "tags": []378 },379 "outputs": [380 {381 "name": "stdout",382 "output_type": "stream",383 "text": [384 "1\n"385 ]386 }387 ],388 "source": [389 "print(illnessDf['ILLNESS'].isna().sum())"390 ]391 },392 {393 "cell_type": "markdown",394 "id": "7006dff1-0b34-42d1-a5c0-585f864b58eb",395 "metadata": {},396 "source": [397 "Drop entries with null illness:"398 ]399 },400 {401 "cell_type": "code",402 "execution_count": 10,403 "id": "a6cd7b76-46c1-483a-8ec4-476ae40cf11a",404 "metadata": {405 "tags": []406 },407 "outputs": [],408 "source": [409 "illnessDf = illnessDf.dropna(subset=[\"ILLNESS\"])"410 ]411 },412 {413 "cell_type": "markdown",414 "id": "2d194e46-43aa-4d20-858b-24cce9622370",415 "metadata": {},416 "source": [417 "Drop entries with Radiology specialization"418 ]419 },420 {421 "cell_type": "code",422 "execution_count": 11,423 "id": "4ea60490-2b02-4c0b-afed-0b08fb45061b",424 "metadata": {425 "tags": []426 },427 "outputs": [],428 "source": [429 "illnessDf = illnessDf.drop(illnessDf[illnessDf['SPECIALIZATION'] == 'Radiology'].index)"430 ]431 },432 {433 "cell_type": "code",434 "execution_count": 12,435 "id": "c1551c0b-05f0-4651-933e-18f956297c10",436 "metadata": {437 "tags": []438 },439 "outputs": [440 {441 "name": "stdout",442 "output_type": "stream",443 "text": [444 "435\n"445 ]446 }447 ],448 "source": [449 "print(len(pd.unique(illnessDf['ILLNESS'])))"450 ]451 },452 {453 "cell_type": "markdown",454 "id": "b669a505-2c9d-4e14-826b-764077880a4d",455 "metadata": {},456 "source": [457 "~ 435 unique illnesses"458 ]459 },460 {461 "cell_type": "code",462 "execution_count": 13,463 "id": "a41a08b7-93e7-4fde-9105-ecd383910d30",464 "metadata": {465 "tags": []466 },467 "outputs": [],468 "source": [469 "symptomsDf = pd.DataFrame()"470 ]471 },472 {473 "cell_type": "code",474 "execution_count": 14,475 "id": "7877703d-bfe1-4c19-8acd-fad850ebf982",476 "metadata": {477 "tags": []478 },479 "outputs": [],480 "source": [481 "symptomsDf[\"ILLNESS\"] = pd.unique(illnessDf['ILLNESS'])"482 ]483 },484 {485 "cell_type": "code",486 "execution_count": 15,487 "id": "83635e1e-398e-4835-83e9-fe2de48483d6",488 "metadata": {489 "tags": []490 },491 "outputs": [],492 "source": [493 "symptomsDf[\"LABEL\"] = symptomsDf.index"494 ]495 },496 {497 "cell_type": "code",498 "execution_count": 16,499 "id": "8f86ee71-0b62-4284-abed-37b458d35e71",500 "metadata": {501 "tags": []502 },503 "outputs": [],504 "source": [505 "def enumerate_words(words):\n",506 " if len(words) == 0:\n",507 " return \"\"\n",508 " elif len(words) == 1:\n",509 " return words[0]\n",510 " else:\n",511 " # Join all words except the last one with \", \"\n",512 " joined_words = \", \".join(words[:-1])\n",513 " # Add the last word preceded by \"and \"\n",514 " result = f\"{joined_words}, and {words[-1]}\"\n",515 " return result"516 ]517 },518 {519 "cell_type": "code",520 "execution_count": 17,521 "id": "24cc6884-2ed9-4e90-b26f-7ce1362e304e",522 "metadata": {523 "tags": []524 },525 "outputs": [],526 "source": [527 "templateDf = pd.read_csv('datasets/sentence_templates.csv')"528 ]529 },530 {531 "cell_type": "code",532 "execution_count": 18,533 "id": "6352cbca-4d89-43fb-9f0d-a3abe48c1f74",534 "metadata": {535 "tags": []536 },537 "outputs": [538 {539 "data": {540 "text/html": [541 "<div>\n",542 "<style scoped>\n",543 " .dataframe tbody tr th:only-of-type {\n",544 " vertical-align: middle;\n",545 " }\n",546 "\n",547 " .dataframe tbody tr th {\n",548 " vertical-align: top;\n",549 " }\n",550 "\n",551 " .dataframe thead th {\n",552 " text-align: right;\n",553 " }\n",554 "</style>\n",555 "<table border=\"1\" class=\"dataframe\">\n",556 " <thead>\n",557 " <tr style=\"text-align: right;\">\n",558 " <th></th>\n",559 " <th>symptoms</th>\n",560 " <th>follow_up</th>\n",561 " <th>city</th>\n",562 " </tr>\n",563 " </thead>\n",564 " <tbody>\n",565 " <tr>\n",566 " <th>0</th>\n",567 " <td>\"I've been experiencing [symptom]. Could you p...</td>\n",568 " <td>\"In addition to [symptom], I've also noticed [...</td>\n",569 " <td>\"I'm looking for recommendations on healthcare...</td>\n",570 " </tr>\n",571 " <tr>\n",572 " <th>1</th>\n",573 " <td>\"I've noticed persistent [symptom], and I'm se...</td>\n",574 " <td>\"I forgot to mention earlier, but along with [...</td>\n",575 " <td>\"In [city], which areas have reputable healthc...</td>\n",576 " </tr>\n",577 " <tr>\n",578 " <th>2</th>\n",579 " <td>\"I'm dealing with symptoms like [symptom]. Do ...</td>\n",580 " <td>\"I've recently started experiencing [new sympt...</td>\n",581 " <td>\"I'm new to [city] and need to find a reliable...</td>\n",582 " </tr>\n",583 " <tr>\n",584 " <th>3</th>\n",585 " <td>\"I've recently encountered [symptom], and I'm ...</td>\n",586 " <td>\"Apart from [symptom], I've been dealing with ...</td>\n",587 " <td>\"Considering [city]'s size, can you recommend ...</td>\n",588 " </tr>\n",589 " <tr>\n",590 " <th>4</th>\n",591 " <td>\"I suspect [symptom] might be related to a hea...</td>\n",592 " <td>\"I didn't mention earlier, but I've also been ...</td>\n",593 " <td>\"I'm searching for healthcare options in [city...</td>\n",594 " </tr>\n",595 " </tbody>\n",596 "</table>\n",597 "</div>"598 ],599 "text/plain": [600 " symptoms \\\n",601 "0 \"I've been experiencing [symptom]. Could you p... \n",602 "1 \"I've noticed persistent [symptom], and I'm se... \n",603 "2 \"I'm dealing with symptoms like [symptom]. Do ... \n",604 "3 \"I've recently encountered [symptom], and I'm ... \n",605 "4 \"I suspect [symptom] might be related to a hea... \n",606 "\n",607 " follow_up \\\n",608 "0 \"In addition to [symptom], I've also noticed [... \n",609 "1 \"I forgot to mention earlier, but along with [... \n",610 "2 \"I've recently started experiencing [new sympt... \n",611 "3 \"Apart from [symptom], I've been dealing with ... \n",612 "4 \"I didn't mention earlier, but I've also been ... \n",613 "\n",614 " city \n",615 "0 \"I'm looking for recommendations on healthcare... \n",616 "1 \"In [city], which areas have reputable healthc... \n",617 "2 \"I'm new to [city] and need to find a reliable... \n",618 "3 \"Considering [city]'s size, can you recommend ... \n",619 "4 \"I'm searching for healthcare options in [city... "620 ]621 },622 "execution_count": 18,623 "metadata": {},624 "output_type": "execute_result"625 }626 ],627 "source": [628 "templateDf.head()"629 ]630 },631 {632 "cell_type": "code",633 "execution_count": 19,634 "id": "fc4d705a-00a0-49f2-a897-cc26edb3d12b",635 "metadata": {636 "tags": []637 },638 "outputs": [],639 "source": [640 "templateDf['symptoms'] = templateDf['symptoms'].str.strip('\"')\n",641 "templateDf['follow_up'] = templateDf['follow_up'].str.strip('\"')"642 ]643 },644 {645 "cell_type": "code",646 "execution_count": 20,647 "id": "5c9aff7e-4784-4093-85d3-cbe6fc53a1a2",648 "metadata": {649 "tags": []650 },651 "outputs": [652 {653 "data": {654 "text/html": [655 "<div>\n",656 "<style scoped>\n",657 " .dataframe tbody tr th:only-of-type {\n",658 " vertical-align: middle;\n",659 " }\n",660 "\n",661 " .dataframe tbody tr th {\n",662 " vertical-align: top;\n",663 " }\n",664 "\n",665 " .dataframe thead th {\n",666 " text-align: right;\n",667 " }\n",668 "</style>\n",669 "<table border=\"1\" class=\"dataframe\">\n",670 " <thead>\n",671 " <tr style=\"text-align: right;\">\n",672 " <th></th>\n",673 " <th>symptoms</th>\n",674 " <th>follow_up</th>\n",675 " <th>city</th>\n",676 " </tr>\n",677 " </thead>\n",678 " <tbody>\n",679 " <tr>\n",680 " <th>0</th>\n",681 " <td>I've been experiencing [symptom]. Could you pr...</td>\n",682 " <td>In addition to [symptom], I've also noticed [n...</td>\n",683 " <td>\"I'm looking for recommendations on healthcare...</td>\n",684 " </tr>\n",685 " <tr>\n",686 " <th>1</th>\n",687 " <td>I've noticed persistent [symptom], and I'm see...</td>\n",688 " <td>I forgot to mention earlier, but along with [s...</td>\n",689 " <td>\"In [city], which areas have reputable healthc...</td>\n",690 " </tr>\n",691 " <tr>\n",692 " <th>2</th>\n",693 " <td>I'm dealing with symptoms like [symptom]. Do y...</td>\n",694 " <td>I've recently started experiencing [new sympto...</td>\n",695 " <td>\"I'm new to [city] and need to find a reliable...</td>\n",696 " </tr>\n",697 " <tr>\n",698 " <th>3</th>\n",699 " <td>I've recently encountered [symptom], and I'm u...</td>\n",700 " <td>Apart from [symptom], I've been dealing with [...</td>\n",701 " <td>\"Considering [city]'s size, can you recommend ...</td>\n",702 " </tr>\n",703 " <tr>\n",704 " <th>4</th>\n",705 " <td>I suspect [symptom] might be related to a heal...</td>\n",706 " <td>I didn't mention earlier, but I've also been h...</td>\n",707 " <td>\"I'm searching for healthcare options in [city...</td>\n",708 " </tr>\n",709 " </tbody>\n",710 "</table>\n",711 "</div>"712 ],713 "text/plain": [714 " symptoms \\\n",715 "0 I've been experiencing [symptom]. Could you pr... \n",716 "1 I've noticed persistent [symptom], and I'm see... \n",717 "2 I'm dealing with symptoms like [symptom]. Do y... \n",718 "3 I've recently encountered [symptom], and I'm u... \n",719 "4 I suspect [symptom] might be related to a heal... \n",720 "\n",721 " follow_up \\\n",722 "0 In addition to [symptom], I've also noticed [n... \n",723 "1 I forgot to mention earlier, but along with [s... \n",724 "2 I've recently started experiencing [new sympto... \n",725 "3 Apart from [symptom], I've been dealing with [... \n",726 "4 I didn't mention earlier, but I've also been h... \n",727 "\n",728 " city \n",729 "0 \"I'm looking for recommendations on healthcare... \n",730 "1 \"In [city], which areas have reputable healthc... \n",731 "2 \"I'm new to [city] and need to find a reliable... \n",732 "3 \"Considering [city]'s size, can you recommend ... \n",733 "4 \"I'm searching for healthcare options in [city... "734 ]735 },736 "execution_count": 20,737 "metadata": {},738 "output_type": "execute_result"739 }740 ],741 "source": [742 "templateDf.head()"743 ]744 },745 {746 "cell_type": "code",747 "execution_count": 21,748 "id": "5d100315-5436-44ac-b11c-2d921a66d559",749 "metadata": {750 "tags": []751 },752 "outputs": [],753 "source": [754 "def replace_symptom(text, replacement):\n",755 " return text.replace(\"[symptom]\", replacement)\n",756 "\n",757 "def replace_new_symptom(text, replacement):\n",758 " return text.replace(\"[new symptom]\", replacement)\n",759 "\n",760 "def replace_illness(text, replacement):\n",761 " return text.replace(\"[illness]\", replacement)\n"762 ]763 },764 {765 "cell_type": "code",766 "execution_count": 22,767 "id": "e028579d-d812-48a2-9cd1-e1699af405a9",768 "metadata": {769 "tags": []770 },771 "outputs": [],772 "source": [773 "promptsDf = pd.DataFrame()"774 ]775 },776 {777 "cell_type": "code",778 "execution_count": 23,779 "id": "daacfb05-3684-47f1-8e52-56cfcbddd6eb",780 "metadata": {781 "tags": []782 },783 "outputs": [],784 "source": [785 "num_prompts_symptoms = 15\n",786 "num_prompts_symptoms_single = 3\n",787 "num_prompts_follow_up = 4"788 ]789 },790 {791 "cell_type": "code",792 "execution_count": 24,793 "id": "5a87a0c1-4a6a-4b97-969c-a18d7b660507",794 "metadata": {795 "tags": []796 },797 "outputs": [798 {799 "name": "stderr",800 "output_type": "stream",801 "text": [802 "C:\\Users\\HTMI\\AppData\\Local\\Temp\\ipykernel_15288\\3613258936.py:5: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value 'tingling, seizures, cardiac arrest, confusion, dizziness, blurred vision, muscle twitching, numbness, unconsciousness, respiratory depression, hypotension, bradycardia, and arrhythmias' has dtype incompatible with float64, please explicitly cast to a compatible dtype first.\n",803 " symptomsDf.at[symptomsDf[symptomsDf[\"ILLNESS\"]==illness].index[0], \"SYMPTOMS\"] = symptomsStr\n"804 ]805 }806 ],807 "source": [808 "for illness in symptomsDf[\"ILLNESS\"]:\n",809 " tempDf = illnessDf[illnessDf[\"ILLNESS\"] == illness]\n",810 " symptomsList = tempDf[\"SYMPTOMS\"].astype(str).tolist()\n",811 " symptomsStr = enumerate_words(symptomsList)\n",812 " symptomsDf.at[symptomsDf[symptomsDf[\"ILLNESS\"]==illness].index[0], \"SYMPTOMS\"] = symptomsStr\n",813 " \n",814 " for prompt in templateDf[\"symptoms\"].sample(n=num_prompts_symptoms, replace=False):\n",815 " random.shuffle(symptomsList)\n",816 " shuffleSymptomsStr = enumerate_words(symptomsList)\n",817 " filled_template = replace_symptom(prompt, shuffleSymptomsStr)\n",818 " filled_template = replace_illness(filled_template, illness)\n",819 " new_row = pd.DataFrame({'prompt': [filled_template], 'illness': symptomsDf.at[symptomsDf[symptomsDf[\"ILLNESS\"]==illness].index[0], \"ILLNESS\"], 'label': symptomsDf.at[symptomsDf[symptomsDf[\"ILLNESS\"]==illness].index[0], \"LABEL\"]})\n",820 " promptsDf = pd.concat([promptsDf, new_row], ignore_index=True)\n",821 " \n",822 " for symptom in symptomsList:\n",823 " for prompt in templateDf[\"symptoms\"].sample(n=num_prompts_symptoms_single, replace=False):\n",824 " filled_template = replace_symptom(prompt, symptom)\n",825 " filled_template = replace_illness(filled_template, illness)\n",826 " new_row = pd.DataFrame({'prompt': [filled_template], 'illness': symptomsDf.at[symptomsDf[symptomsDf[\"ILLNESS\"]==illness].index[0], \"ILLNESS\"], 'label': symptomsDf.at[symptomsDf[symptomsDf[\"ILLNESS\"]==illness].index[0], \"LABEL\"]})\n",827 " promptsDf = pd.concat([promptsDf, new_row], ignore_index=True)\n",828 " \n",829 " followUp = templateDf.dropna(subset=[\"follow_up\"]).sample(n=num_prompts_follow_up, replace=False)\n",830 " followUp = followUp[\"follow_up\"]\n",831 " for prompt in followUp:\n",832 " tempList = [item for item in symptomsList if item != symptom]\n",833 " if(len(tempList) == 0):\n",834 " continue\n",835 " random.shuffle(tempList)\n",836 " sublist_size = random.randint(1, min(len(tempList), 5))\n",837 " shortSymptomsStr = enumerate_words(random.sample(tempList, sublist_size))\n",838 " filled_template = replace_symptom(prompt, symptom)\n",839 " filled_template = replace_new_symptom(filled_template, shortSymptomsStr)\n",840 " filled_template = replace_illness(filled_template, illness)\n",841 " new_row = pd.DataFrame({'prompt': [filled_template], 'illness': symptomsDf.at[symptomsDf[symptomsDf[\"ILLNESS\"]==illness].index[0], \"ILLNESS\"], 'label': symptomsDf.at[symptomsDf[symptomsDf[\"ILLNESS\"]==illness].index[0], \"LABEL\"]})\n",842 " promptsDf = pd.concat([promptsDf, new_row], ignore_index=True)\n",843 " "844 ]845 },846 {847 "cell_type": "code",848 "execution_count": 25,849 "id": "efa49124-4ba6-496e-87ef-62b048783bd8",850 "metadata": {851 "tags": []852 },853 "outputs": [854 {855 "data": {856 "text/html": [857 "<div>\n",858 "<style scoped>\n",859 " .dataframe tbody tr th:only-of-type {\n",860 " vertical-align: middle;\n",861 " }\n",862 "\n",863 " .dataframe tbody tr th {\n",864 " vertical-align: top;\n",865 " }\n",866 "\n",867 " .dataframe thead th {\n",868 " text-align: right;\n",869 " }\n",870 "</style>\n",871 "<table border=\"1\" class=\"dataframe\">\n",872 " <thead>\n",873 " <tr style=\"text-align: right;\">\n",874 " <th></th>\n",875 " <th>ILLNESS</th>\n",876 " <th>LABEL</th>\n",877 " <th>SYMPTOMS</th>\n",878 " </tr>\n",879 " </thead>\n",880 " <tbody>\n",881 " <tr>\n",882 " <th>0</th>\n",883 " <td>Local anesthetic toxicity</td>\n",884 " <td>0</td>\n",885 " <td>tingling, seizures, cardiac arrest, confusion,...</td>\n",886 " </tr>\n",887 " <tr>\n",888 " <th>1</th>\n",889 " <td>Malignant hyperthermia</td>\n",890 " <td>1</td>\n",891 " <td>high fever, muscle rigidity, rapid heart rate,...</td>\n",892 " </tr>\n",893 " <tr>\n",894 " <th>2</th>\n",895 " <td>Post-dural puncture headache</td>\n",896 " <td>2</td>\n",897 " <td>neck stiffness, photophobia, nausea, vomiting,...</td>\n",898 " </tr>\n",899 " <tr>\n",900 " <th>3</th>\n",901 " <td>Anaphylaxis to anesthetic agents</td>\n",902 " <td>3</td>\n",903 " <td>hives, itching, swelling, rash, bronchospasm, ...</td>\n",904 " </tr>\n",905 " <tr>\n",906 " <th>4</th>\n",907 " <td>Awareness during anesthesia</td>\n",908 " <td>4</td>\n",909 " <td>paralysis, inability to move, pain, distress, ...</td>\n",910 " </tr>\n",911 " <tr>\n",912 " <th>...</th>\n",913 " <td>...</td>\n",914 " <td>...</td>\n",915 " <td>...</td>\n",916 " </tr>\n",917 " <tr>\n",918 " <th>430</th>\n",919 " <td>Excoriation disorder (skin-picking disorder)</td>\n",920 " <td>430</td>\n",921 " <td>Recurrent picking of one's skin, Attempts to s...</td>\n",922 " </tr>\n",923 " <tr>\n",924 " <th>431</th>\n",925 " <td>Factitious disorder</td>\n",926 " <td>431</td>\n",927 " <td>Intentionally producing symptoms of illness, D...</td>\n",928 " </tr>\n",929 " <tr>\n",930 " <th>432</th>\n",931 " <td>Impulse-control disorders</td>\n",932 " <td>432</td>\n",933 " <td>Impulsive behavior that causes harm, Failure t...</td>\n",934 " </tr>\n",935 " <tr>\n",936 " <th>433</th>\n",937 " <td>Adjustment disorders</td>\n",938 " <td>433</td>\n",939 " <td>Development of emotional or behavioral symptom...</td>\n",940 " </tr>\n",941 " <tr>\n",942 " <th>434</th>\n",943 " <td>Dissociative disorders</td>\n",944 " <td>434</td>\n",945 " <td>Alterations in consciousness or identity, Amne...</td>\n",946 " </tr>\n",947 " </tbody>\n",948 "</table>\n",949 "<p>435 rows × 3 columns</p>\n",950 "</div>"951 ],952 "text/plain": [953 " ILLNESS LABEL \\\n",954 "0 Local anesthetic toxicity 0 \n",955 "1 Malignant hyperthermia 1 \n",956 "2 Post-dural puncture headache 2 \n",957 "3 Anaphylaxis to anesthetic agents 3 \n",958 "4 Awareness during anesthesia 4 \n",959 ".. ... ... \n",960 "430 Excoriation disorder (skin-picking disorder) 430 \n",961 "431 Factitious disorder 431 \n",962 "432 Impulse-control disorders 432 \n",963 "433 Adjustment disorders 433 \n",964 "434 Dissociative disorders 434 \n",965 "\n",966 " SYMPTOMS \n",967 "0 tingling, seizures, cardiac arrest, confusion,... \n",968 "1 high fever, muscle rigidity, rapid heart rate,... \n",969 "2 neck stiffness, photophobia, nausea, vomiting,... \n",970 "3 hives, itching, swelling, rash, bronchospasm, ... \n",971 "4 paralysis, inability to move, pain, distress, ... \n",972 ".. ... \n",973 "430 Recurrent picking of one's skin, Attempts to s... \n",974 "431 Intentionally producing symptoms of illness, D... \n",975 "432 Impulsive behavior that causes harm, Failure t... \n",976 "433 Development of emotional or behavioral symptom... \n",977 "434 Alterations in consciousness or identity, Amne... \n",978 "\n",979 "[435 rows x 3 columns]"980 ]981 },982 "execution_count": 25,983 "metadata": {},984 "output_type": "execute_result"985 }986 ],987 "source": [988 "symptomsDf"989 ]990 },991 {992 "cell_type": "code",993 "execution_count": 26,994 "id": "82fe4a8f-0684-45f6-8316-a48c9d2bf593",995 "metadata": {996 "tags": []997 },998 "outputs": [999 {1000 "data": {1001 "text/plain": [1002 "(24955, 3)"1003 ]1004 },1005 "execution_count": 26,1006 "metadata": {},1007 "output_type": "execute_result"1008 }1009 ],1010 "source": [1011 "promptsDf.shape"1012 ]1013 },1014 {1015 "cell_type": "code",1016 "execution_count": 27,1017 "id": "b7a2b208-98c6-42f6-9d10-4396dff6f334",1018 "metadata": {1019 "tags": []1020 },1021 "outputs": [1022 {1023 "data": {1024 "text/html": [1025 "<div>\n",1026 "<style scoped>\n",1027 " .dataframe tbody tr th:only-of-type {\n",1028 " vertical-align: middle;\n",1029 " }\n",1030 "\n",1031 " .dataframe tbody tr th {\n",1032 " vertical-align: top;\n",1033 " }\n",1034 "\n",1035 " .dataframe thead th {\n",1036 " text-align: right;\n",1037 " }\n",1038 "</style>\n",1039 "<table border=\"1\" class=\"dataframe\">\n",1040 " <thead>\n",1041 " <tr style=\"text-align: right;\">\n",1042 " <th></th>\n",1043 " <th>prompt</th>\n",1044 " <th>illness</th>\n",1045 " <th>label</th>\n",1046 " </tr>\n",1047 " </thead>\n",1048 " <tbody>\n",1049 " <tr>\n",1050 " <th>0</th>\n",1051 " <td>I've been noticing blurred vision, confusion, ...</td>\n",1052 " <td>Local anesthetic toxicity</td>\n",1053 " <td>0</td>\n",1054 " </tr>\n",1055 " <tr>\n",1056 " <th>1</th>\n",1057 " <td>Noticed muscle twitching, unconsciousness, tin...</td>\n",1058 " <td>Local anesthetic toxicity</td>\n",1059 " <td>0</td>\n",1060 " </tr>\n",1061 " <tr>\n",1062 " <th>2</th>\n",1063 " <td>I suspect bradycardia, numbness, arrhythmias, ...</td>\n",1064 " <td>Local anesthetic toxicity</td>\n",1065 " <td>0</td>\n",1066 " </tr>\n",1067 " <tr>\n",1068 " <th>3</th>\n",1069 " <td>Noticing dizziness, blurred vision, hypotensio...</td>\n",1070 " <td>Local anesthetic toxicity</td>\n",1071 " <td>0</td>\n",1072 " </tr>\n",1073 " <tr>\n",1074 " <th>4</th>\n",1075 " <td>Experiencing blurred vision, respiratory depre...</td>\n",1076 " <td>Local anesthetic toxicity</td>\n",1077 " <td>0</td>\n",1078 " </tr>\n",1079 " <tr>\n",1080 " <th>...</th>\n",1081 " <td>...</td>\n",1082 " <td>...</td>\n",1083 " <td>...</td>\n",1084 " </tr>\n",1085 " <tr>\n",1086 " <th>24950</th>\n",1087 " <td>I've noticed Distress or impairment due to sym...</td>\n",1088 " <td>Dissociative disorders</td>\n",1089 " <td>434</td>\n",1090 " </tr>\n",1091 " <tr>\n",1092 " <th>24951</th>\n",1093 " <td>I've realized that along with Distress or impa...</td>\n",1094 " <td>Dissociative disorders</td>\n",1095 " <td>434</td>\n",1096 " </tr>\n",1097 " <tr>\n",1098 " <th>24952</th>\n",1099 " <td>I've been monitoring my symptoms, and I've ide...</td>\n",1100 " <td>Dissociative disorders</td>\n",1101 " <td>434</td>\n",1102 " </tr>\n",1103 " <tr>\n",1104 " <th>24953</th>\n",1105 " <td>Apart from Distress or impairment due to sympt...</td>\n",1106 " <td>Dissociative disorders</td>\n",1107 " <td>434</td>\n",1108 " </tr>\n",1109 " <tr>\n",1110 " <th>24954</th>\n",1111 " <td>I didn't include it earlier, but in addition t...</td>\n",1112 " <td>Dissociative disorders</td>\n",1113 " <td>434</td>\n",1114 " </tr>\n",1115 " </tbody>\n",1116 "</table>\n",1117 "<p>24955 rows × 3 columns</p>\n",1118 "</div>"1119 ],1120 "text/plain": [1121 " prompt \\\n",1122 "0 I've been noticing blurred vision, confusion, ... \n",1123 "1 Noticed muscle twitching, unconsciousness, tin... \n",1124 "2 I suspect bradycardia, numbness, arrhythmias, ... \n",1125 "3 Noticing dizziness, blurred vision, hypotensio... \n",1126 "4 Experiencing blurred vision, respiratory depre... \n",1127 "... ... \n",1128 "24950 I've noticed Distress or impairment due to sym... \n",1129 "24951 I've realized that along with Distress or impa... \n",1130 "24952 I've been monitoring my symptoms, and I've ide... \n",1131 "24953 Apart from Distress or impairment due to sympt... \n",1132 "24954 I didn't include it earlier, but in addition t... \n",1133 "\n",1134 " illness label \n",1135 "0 Local anesthetic toxicity 0 \n",1136 "1 Local anesthetic toxicity 0 \n",1137 "2 Local anesthetic toxicity 0 \n",1138 "3 Local anesthetic toxicity 0 \n",1139 "4 Local anesthetic toxicity 0 \n",1140 "... ... ... \n",1141 "24950 Dissociative disorders 434 \n",1142 "24951 Dissociative disorders 434 \n",1143 "24952 Dissociative disorders 434 \n",1144 "24953 Dissociative disorders 434 \n",1145 "24954 Dissociative disorders 434 \n",1146 "\n",1147 "[24955 rows x 3 columns]"1148 ]1149 },1150 "execution_count": 27,1151 "metadata": {},1152 "output_type": "execute_result"1153 }1154 ],1155 "source": [1156 "promptsDf"1157 ]1158 },1159 {1160 "cell_type": "code",1161 "execution_count": 28,1162 "id": "d74aaae2-97f5-46ba-b734-c8996a19854c",1163 "metadata": {1164 "tags": []1165 },1166 "outputs": [],1167 "source": [1168 "promptsDf.to_csv('datasets/prompts.csv', index=False)"1169 ]1170 },1171 {1172 "cell_type": "markdown",1173 "id": "19f7869b-3f2c-47c0-986e-3c7d8cfd2952",1174 "metadata": {},1175 "source": [1176 "### Generating dataset for user prompts on city:"1177 ]1178 },1179 {1180 "cell_type": "code",1181 "execution_count": 48,1182 "id": "3c94e938-8fe1-4c4d-bfd5-3e18b3db061e",1183 "metadata": {1184 "tags": []1185 },1186 "outputs": [],1187 "source": [1188 "cityPromptsDf = pd.read_csv('datasets/city_prompts.csv')"1189 ]1190 },1191 {1192 "cell_type": "code",1193 "execution_count": 50,1194 "id": "20522657-b7b4-4272-9ad6-7fa38ca895ed",1195 "metadata": {1196 "tags": []1197 },1198 "outputs": [],1199 "source": [1200 "cityPromptsAnnotatedDf = pd.DataFrame()"