Rafa1986/Data-Analytics-Class
0
1{2 "cells": [3 {4 "cell_type": "markdown",5 "metadata": {},6 "source": [7 "### TU257-Lab6-Demo-2-Roc\n",8 "\n",9 "In this demo notebook we will look at the ROC charts.\n",10 "\n",11 "ROC curve, which stands for “receiver operating characteristic” curve. This is a plot that displays the sensitivity and specificity.\n",12 "\n",13 "Sensitivity: The probability that the model predicts a positive outcome for an observation when indeed the outcome is positive. This is also called the “true positive rate.”\n",14 "\n",15 "Specificity: The probability that the model predicts a negative outcome for an observation when indeed the outcome is negative. This is also called the “true negative rate.”\n"16 ]17 },18 {19 "cell_type": "code",20 "execution_count": 3,21 "metadata": {},22 "outputs": [],23 "source": [24 "from matplotlib import pyplot as plt\n",25 "from sklearn import datasets\n",26 "from sklearn.tree import DecisionTreeClassifier \n",27 "from sklearn import tree\n"28 ]29 },30 {31 "cell_type": "code",32 "execution_count": 4,33 "metadata": {},34 "outputs": [35 {36 "data": {37 "text/html": [38 "<div>\n",39 "<style scoped>\n",40 " .dataframe tbody tr th:only-of-type {\n",41 " vertical-align: middle;\n",42 " }\n",43 "\n",44 " .dataframe tbody tr th {\n",45 " vertical-align: top;\n",46 " }\n",47 "\n",48 " .dataframe thead th {\n",49 " text-align: right;\n",50 " }\n",51 "</style>\n",52 "<table border=\"1\" class=\"dataframe\">\n",53 " <thead>\n",54 " <tr style=\"text-align: right;\">\n",55 " <th></th>\n",56 " <th>Age</th>\n",57 " <th>WorkClass</th>\n",58 " <th>Fnlwgt</th>\n",59 " <th>Education</th>\n",60 " <th>Edu_Num</th>\n",61 " <th>MaritalStatus</th>\n",62 " <th>Occupation</th>\n",63 " <th>Relationship</th>\n",64 " <th>Race</th>\n",65 " <th>Sex</th>\n",66 " <th>CapitalGain</th>\n",67 " <th>CapitalLoss</th>\n",68 " <th>HrPerWk</th>\n",69 " <th>Native</th>\n",70 " <th>Target</th>\n",71 " </tr>\n",72 " </thead>\n",73 " <tbody>\n",74 " <tr>\n",75 " <th>0</th>\n",76 " <td>39</td>\n",77 " <td>State-gov</td>\n",78 " <td>77516</td>\n",79 " <td>Bachelors</td>\n",80 " <td>13</td>\n",81 " <td>Never-married</td>\n",82 " <td>Adm-clerical</td>\n",83 " <td>Not-in-family</td>\n",84 " <td>White</td>\n",85 " <td>Male</td>\n",86 " <td>2174</td>\n",87 " <td>0</td>\n",88 " <td>40</td>\n",89 " <td>United-States</td>\n",90 " <td><=50K</td>\n",91 " </tr>\n",92 " <tr>\n",93 " <th>1</th>\n",94 " <td>50</td>\n",95 " <td>Self-emp-not-inc</td>\n",96 " <td>83311</td>\n",97 " <td>Bachelors</td>\n",98 " <td>13</td>\n",99 " <td>Married-civ-spouse</td>\n",100 " <td>Exec-managerial</td>\n",101 " <td>Husband</td>\n",102 " <td>White</td>\n",103 " <td>Male</td>\n",104 " <td>0</td>\n",105 " <td>0</td>\n",106 " <td>13</td>\n",107 " <td>United-States</td>\n",108 " <td><=50K</td>\n",109 " </tr>\n",110 " <tr>\n",111 " <th>2</th>\n",112 " <td>38</td>\n",113 " <td>Private</td>\n",114 " <td>215646</td>\n",115 " <td>HS-grad</td>\n",116 " <td>9</td>\n",117 " <td>Divorced</td>\n",118 " <td>Handlers-cleaners</td>\n",119 " <td>Not-in-family</td>\n",120 " <td>White</td>\n",121 " <td>Male</td>\n",122 " <td>0</td>\n",123 " <td>0</td>\n",124 " <td>40</td>\n",125 " <td>United-States</td>\n",126 " <td><=50K</td>\n",127 " </tr>\n",128 " <tr>\n",129 " <th>3</th>\n",130 " <td>53</td>\n",131 " <td>Private</td>\n",132 " <td>234721</td>\n",133 " <td>11th</td>\n",134 " <td>7</td>\n",135 " <td>Married-civ-spouse</td>\n",136 " <td>Handlers-cleaners</td>\n",137 " <td>Husband</td>\n",138 " <td>Black</td>\n",139 " <td>Male</td>\n",140 " <td>0</td>\n",141 " <td>0</td>\n",142 " <td>40</td>\n",143 " <td>United-States</td>\n",144 " <td><=50K</td>\n",145 " </tr>\n",146 " <tr>\n",147 " <th>4</th>\n",148 " <td>28</td>\n",149 " <td>Private</td>\n",150 " <td>338409</td>\n",151 " <td>Bachelors</td>\n",152 " <td>13</td>\n",153 " <td>Married-civ-spouse</td>\n",154 " <td>Prof-specialty</td>\n",155 " <td>Wife</td>\n",156 " <td>Black</td>\n",157 " <td>Female</td>\n",158 " <td>0</td>\n",159 " <td>0</td>\n",160 " <td>40</td>\n",161 " <td>Cuba</td>\n",162 " <td><=50K</td>\n",163 " </tr>\n",164 " <tr>\n",165 " <th>5</th>\n",166 " <td>37</td>\n",167 " <td>Private</td>\n",168 " <td>284582</td>\n",169 " <td>Masters</td>\n",170 " <td>14</td>\n",171 " <td>Married-civ-spouse</td>\n",172 " <td>Exec-managerial</td>\n",173 " <td>Wife</td>\n",174 " <td>White</td>\n",175 " <td>Female</td>\n",176 " <td>0</td>\n",177 " <td>0</td>\n",178 " <td>40</td>\n",179 " <td>United-States</td>\n",180 " <td><=50K</td>\n",181 " </tr>\n",182 " <tr>\n",183 " <th>6</th>\n",184 " <td>49</td>\n",185 " <td>Private</td>\n",186 " <td>160187</td>\n",187 " <td>9th</td>\n",188 " <td>5</td>\n",189 " <td>Married-spouse-absent</td>\n",190 " <td>Other-service</td>\n",191 " <td>Not-in-family</td>\n",192 " <td>Black</td>\n",193 " <td>Female</td>\n",194 " <td>0</td>\n",195 " <td>0</td>\n",196 " <td>16</td>\n",197 " <td>Jamaica</td>\n",198 " <td><=50K</td>\n",199 " </tr>\n",200 " <tr>\n",201 " <th>7</th>\n",202 " <td>52</td>\n",203 " <td>Self-emp-not-inc</td>\n",204 " <td>209642</td>\n",205 " <td>HS-grad</td>\n",206 " <td>9</td>\n",207 " <td>Married-civ-spouse</td>\n",208 " <td>Exec-managerial</td>\n",209 " <td>Husband</td>\n",210 " <td>White</td>\n",211 " <td>Male</td>\n",212 " <td>0</td>\n",213 " <td>0</td>\n",214 " <td>45</td>\n",215 " <td>United-States</td>\n",216 " <td>>50K</td>\n",217 " </tr>\n",218 " <tr>\n",219 " <th>8</th>\n",220 " <td>31</td>\n",221 " <td>Private</td>\n",222 " <td>45781</td>\n",223 " <td>Masters</td>\n",224 " <td>14</td>\n",225 " <td>Never-married</td>\n",226 " <td>Prof-specialty</td>\n",227 " <td>Not-in-family</td>\n",228 " <td>White</td>\n",229 " <td>Female</td>\n",230 " <td>14084</td>\n",231 " <td>0</td>\n",232 " <td>50</td>\n",233 " <td>United-States</td>\n",234 " <td>>50K</td>\n",235 " </tr>\n",236 " <tr>\n",237 " <th>9</th>\n",238 " <td>42</td>\n",239 " <td>Private</td>\n",240 " <td>159449</td>\n",241 " <td>Bachelors</td>\n",242 " <td>13</td>\n",243 " <td>Married-civ-spouse</td>\n",244 " <td>Exec-managerial</td>\n",245 " <td>Husband</td>\n",246 " <td>White</td>\n",247 " <td>Male</td>\n",248 " <td>5178</td>\n",249 " <td>0</td>\n",250 " <td>40</td>\n",251 " <td>United-States</td>\n",252 " <td>>50K</td>\n",253 " </tr>\n",254 " </tbody>\n",255 "</table>\n",256 "</div>"257 ],258 "text/plain": [259 " Age WorkClass Fnlwgt Education Edu_Num \\\n",260 "0 39 State-gov 77516 Bachelors 13 \n",261 "1 50 Self-emp-not-inc 83311 Bachelors 13 \n",262 "2 38 Private 215646 HS-grad 9 \n",263 "3 53 Private 234721 11th 7 \n",264 "4 28 Private 338409 Bachelors 13 \n",265 "5 37 Private 284582 Masters 14 \n",266 "6 49 Private 160187 9th 5 \n",267 "7 52 Self-emp-not-inc 209642 HS-grad 9 \n",268 "8 31 Private 45781 Masters 14 \n",269 "9 42 Private 159449 Bachelors 13 \n",270 "\n",271 " MaritalStatus Occupation Relationship Race \\\n",272 "0 Never-married Adm-clerical Not-in-family White \n",273 "1 Married-civ-spouse Exec-managerial Husband White \n",274 "2 Divorced Handlers-cleaners Not-in-family White \n",275 "3 Married-civ-spouse Handlers-cleaners Husband Black \n",276 "4 Married-civ-spouse Prof-specialty Wife Black \n",277 "5 Married-civ-spouse Exec-managerial Wife White \n",278 "6 Married-spouse-absent Other-service Not-in-family Black \n",279 "7 Married-civ-spouse Exec-managerial Husband White \n",280 "8 Never-married Prof-specialty Not-in-family White \n",281 "9 Married-civ-spouse Exec-managerial Husband White \n",282 "\n",283 " Sex CapitalGain CapitalLoss HrPerWk Native Target \n",284 "0 Male 2174 0 40 United-States <=50K \n",285 "1 Male 0 0 13 United-States <=50K \n",286 "2 Male 0 0 40 United-States <=50K \n",287 "3 Male 0 0 40 United-States <=50K \n",288 "4 Female 0 0 40 Cuba <=50K \n",289 "5 Female 0 0 40 United-States <=50K \n",290 "6 Female 0 0 16 Jamaica <=50K \n",291 "7 Male 0 0 45 United-States >50K \n",292 "8 Female 14084 0 50 United-States >50K \n",293 "9 Male 5178 0 40 United-States >50K "294 ]295 },296 "execution_count": 4,297 "metadata": {},298 "output_type": "execute_result"299 }300 ],301 "source": [302 "import pandas as pd\n",303 "\n",304 "#Load in the dataset\n",305 "colnames=['Age', 'WorkClass', 'Fnlwgt', 'Education', 'Edu_Num', 'MaritalStatus', 'Occupation', 'Relationship', 'Race', 'Sex', 'CapitalGain', 'CapitalLoss', 'HrPerWk', 'Native', 'Target'] \n",306 "df = pd.read_csv('/Users/brendan.tierney/Dropbox/4-Datasets/adult.csv', names=colnames, header=None)\n",307 "df.head(10)"308 ]309 },310 {311 "cell_type": "code",312 "execution_count": 5,313 "metadata": {},314 "outputs": [315 {316 "data": {317 "text/plain": [318 "False"319 ]320 },321 "execution_count": 5,322 "metadata": {},323 "output_type": "execute_result"324 }325 ],326 "source": [327 "df.isnull().values.any()"328 ]329 },330 {331 "cell_type": "code",332 "execution_count": 6,333 "metadata": {},334 "outputs": [335 {336 "name": "stdout",337 "output_type": "stream",338 "text": [339 "Rows : 32561\n",340 "Columns : 15\n",341 "\n",342 "Features : \n",343 " ['Age', 'WorkClass', 'Fnlwgt', 'Education', 'Edu_Num', 'MaritalStatus', 'Occupation', 'Relationship', 'Race', 'Sex', 'CapitalGain', 'CapitalLoss', 'HrPerWk', 'Native', 'Target']\n",344 "\n",345 "Missing values : 0\n",346 "\n",347 "Unique values : \n",348 " Age 73\n",349 "WorkClass 9\n",350 "Fnlwgt 21648\n",351 "Education 16\n",352 "Edu_Num 16\n",353 "MaritalStatus 7\n",354 "Occupation 15\n",355 "Relationship 6\n",356 "Race 5\n",357 "Sex 2\n",358 "CapitalGain 119\n",359 "CapitalLoss 92\n",360 "HrPerWk 94\n",361 "Native 42\n",362 "Target 2\n",363 "dtype: int64\n"364 ]365 }366 ],367 "source": [368 "print (\"Rows : \" ,df.shape[0])\n",369 "print (\"Columns : \" ,df.shape[1])\n",370 "print (\"\\nFeatures : \\n\" ,df.columns.tolist())\n",371 "print (\"\\nMissing values : \", df.isnull().sum().values.sum())\n",372 "print (\"\\nUnique values : \\n\",df.nunique())\n"373 ]374 },375 {376 "cell_type": "code",377 "execution_count": 7,378 "metadata": {},379 "outputs": [380 {381 "name": "stdout",382 "output_type": "stream",383 "text": [384 "<class 'pandas.core.frame.DataFrame'>\n",385 "RangeIndex: 32561 entries, 0 to 32560\n",386 "Data columns (total 15 columns):\n",387 " # Column Non-Null Count Dtype \n",388 "--- ------ -------------- ----- \n",389 " 0 Age 32561 non-null int64 \n",390 " 1 WorkClass 32561 non-null object\n",391 " 2 Fnlwgt 32561 non-null int64 \n",392 " 3 Education 32561 non-null object\n",393 " 4 Edu_Num 32561 non-null int64 \n",394 " 5 MaritalStatus 32561 non-null object\n",395 " 6 Occupation 32561 non-null object\n",396 " 7 Relationship 32561 non-null object\n",397 " 8 Race 32561 non-null object\n",398 " 9 Sex 32561 non-null object\n",399 " 10 CapitalGain 32561 non-null int64 \n",400 " 11 CapitalLoss 32561 non-null int64 \n",401 " 12 HrPerWk 32561 non-null int64 \n",402 " 13 Native 32561 non-null object\n",403 " 14 Target 32561 non-null object\n",404 "dtypes: int64(6), object(9)\n",405 "memory usage: 3.7+ MB\n"406 ]407 }408 ],409 "source": [410 "df.info()"411 ]412 },413 {414 "cell_type": "code",415 "execution_count": 8,416 "metadata": {},417 "outputs": [418 {419 "data": {420 "text/html": [421 "<div>\n",422 "<style scoped>\n",423 " .dataframe tbody tr th:only-of-type {\n",424 " vertical-align: middle;\n",425 " }\n",426 "\n",427 " .dataframe tbody tr th {\n",428 " vertical-align: top;\n",429 " }\n",430 "\n",431 " .dataframe thead th {\n",432 " text-align: right;\n",433 " }\n",434 "</style>\n",435 "<table border=\"1\" class=\"dataframe\">\n",436 " <thead>\n",437 " <tr style=\"text-align: right;\">\n",438 " <th></th>\n",439 " <th>Age</th>\n",440 " <th>Fnlwgt</th>\n",441 " <th>Edu_Num</th>\n",442 " <th>CapitalGain</th>\n",443 " <th>CapitalLoss</th>\n",444 " <th>HrPerWk</th>\n",445 " </tr>\n",446 " </thead>\n",447 " <tbody>\n",448 " <tr>\n",449 " <th>count</th>\n",450 " <td>32561.000000</td>\n",451 " <td>3.256100e+04</td>\n",452 " <td>32561.000000</td>\n",453 " <td>32561.000000</td>\n",454 " <td>32561.000000</td>\n",455 " <td>32561.000000</td>\n",456 " </tr>\n",457 " <tr>\n",458 " <th>mean</th>\n",459 " <td>38.581647</td>\n",460 " <td>1.897784e+05</td>\n",461 " <td>10.080679</td>\n",462 " <td>1077.648844</td>\n",463 " <td>87.303830</td>\n",464 " <td>40.437456</td>\n",465 " </tr>\n",466 " <tr>\n",467 " <th>std</th>\n",468 " <td>13.640433</td>\n",469 " <td>1.055500e+05</td>\n",470 " <td>2.572720</td>\n",471 " <td>7385.292085</td>\n",472 " <td>402.960219</td>\n",473 " <td>12.347429</td>\n",474 " </tr>\n",475 " <tr>\n",476 " <th>min</th>\n",477 " <td>17.000000</td>\n",478 " <td>1.228500e+04</td>\n",479 " <td>1.000000</td>\n",480 " <td>0.000000</td>\n",481 " <td>0.000000</td>\n",482 " <td>1.000000</td>\n",483 " </tr>\n",484 " <tr>\n",485 " <th>25%</th>\n",486 " <td>28.000000</td>\n",487 " <td>1.178270e+05</td>\n",488 " <td>9.000000</td>\n",489 " <td>0.000000</td>\n",490 " <td>0.000000</td>\n",491 " <td>40.000000</td>\n",492 " </tr>\n",493 " <tr>\n",494 " <th>50%</th>\n",495 " <td>37.000000</td>\n",496 " <td>1.783560e+05</td>\n",497 " <td>10.000000</td>\n",498 " <td>0.000000</td>\n",499 " <td>0.000000</td>\n",500 " <td>40.000000</td>\n",501 " </tr>\n",502 " <tr>\n",503 " <th>75%</th>\n",504 " <td>48.000000</td>\n",505 " <td>2.370510e+05</td>\n",506 " <td>12.000000</td>\n",507 " <td>0.000000</td>\n",508 " <td>0.000000</td>\n",509 " <td>45.000000</td>\n",510 " </tr>\n",511 " <tr>\n",512 " <th>max</th>\n",513 " <td>90.000000</td>\n",514 " <td>1.484705e+06</td>\n",515 " <td>16.000000</td>\n",516 " <td>99999.000000</td>\n",517 " <td>4356.000000</td>\n",518 " <td>99.000000</td>\n",519 " </tr>\n",520 " </tbody>\n",521 "</table>\n",522 "</div>"523 ],524 "text/plain": [525 " Age Fnlwgt Edu_Num CapitalGain CapitalLoss \\\n",526 "count 32561.000000 3.256100e+04 32561.000000 32561.000000 32561.000000 \n",527 "mean 38.581647 1.897784e+05 10.080679 1077.648844 87.303830 \n",528 "std 13.640433 1.055500e+05 2.572720 7385.292085 402.960219 \n",529 "min 17.000000 1.228500e+04 1.000000 0.000000 0.000000 \n",530 "25% 28.000000 1.178270e+05 9.000000 0.000000 0.000000 \n",531 "50% 37.000000 1.783560e+05 10.000000 0.000000 0.000000 \n",532 "75% 48.000000 2.370510e+05 12.000000 0.000000 0.000000 \n",533 "max 90.000000 1.484705e+06 16.000000 99999.000000 4356.000000 \n",534 "\n",535 " HrPerWk \n",536 "count 32561.000000 \n",537 "mean 40.437456 \n",538 "std 12.347429 \n",539 "min 1.000000 \n",540 "25% 40.000000 \n",541 "50% 40.000000 \n",542 "75% 45.000000 \n",543 "max 99.000000 "544 ]545 },546 "execution_count": 8,547 "metadata": {},548 "output_type": "execute_result"549 }550 ],551 "source": [552 "# Numerical feature of summary/description \n",553 "df.describe()"554 ]555 },556 {557 "cell_type": "code",558 "execution_count": 9,559 "metadata": {},560 "outputs": [561 {562 "data": {563 "text/plain": [564 "Age 0\n",565 "WorkClass 0\n",566 "Fnlwgt 0\n",567 "Education 0\n",568 "Edu_Num 0\n",569 "MaritalStatus 0\n",570 "Occupation 0\n",571 "Relationship 0\n",572 "Race 0\n",573 "Sex 0\n",574 "CapitalGain 0\n",575 "CapitalLoss 0\n",576 "HrPerWk 0\n",577 "Native 0\n",578 "Target 0\n",579 "dtype: int64"580 ]581 },582 "execution_count": 9,583 "metadata": {},584 "output_type": "execute_result"585 }586 ],587 "source": [588 "\n",589 "# checking \"?\" values, how many are there in the whole dataset\n",590 "df_missing = (df=='?').sum()\n",591 "df_missing"592 ]593 },594 {595 "cell_type": "code",596 "execution_count": 10,597 "metadata": {},598 "outputs": [599 {600 "data": {601 "text/html": [602 "<div>\n",603 "<style scoped>\n",604 " .dataframe tbody tr th:only-of-type {\n",605 " vertical-align: middle;\n",606 " }\n",607 "\n",608 " .dataframe tbody tr th {\n",609 " vertical-align: top;\n",610 " }\n",611 "\n",612 " .dataframe thead th {\n",613 " text-align: right;\n",614 " }\n",615 "</style>\n",616 "<table border=\"1\" class=\"dataframe\">\n",617 " <thead>\n",618 " <tr style=\"text-align: right;\">\n",619 " <th></th>\n",620 " <th>WorkClass</th>\n",621 " <th>Education</th>\n",622 " <th>MaritalStatus</th>\n",623 " <th>Occupation</th>\n",624 " <th>Relationship</th>\n",625 " <th>Race</th>\n",626 " <th>Sex</th>\n",627 " <th>Native</th>\n",628 " <th>Target</th>\n",629 " </tr>\n",630 " </thead>\n",631 " <tbody>\n",632 " <tr>\n",633 " <th>0</th>\n",634 " <td>State-gov</td>\n",635 " <td>Bachelors</td>\n",636 " <td>Never-married</td>\n",637 " <td>Adm-clerical</td>\n",638 " <td>Not-in-family</td>\n",639 " <td>White</td>\n",640 " <td>Male</td>\n",641 " <td>United-States</td>\n",642 " <td><=50K</td>\n",643 " </tr>\n",644 " <tr>\n",645 " <th>1</th>\n",646 " <td>Self-emp-not-inc</td>\n",647 " <td>Bachelors</td>\n",648 " <td>Married-civ-spouse</td>\n",649 " <td>Exec-managerial</td>\n",650 " <td>Husband</td>\n",651 " <td>White</td>\n",652 " <td>Male</td>\n",653 " <td>United-States</td>\n",654 " <td><=50K</td>\n",655 " </tr>\n",656 " <tr>\n",657 " <th>2</th>\n",658 " <td>Private</td>\n",659 " <td>HS-grad</td>\n",660 " <td>Divorced</td>\n",661 " <td>Handlers-cleaners</td>\n",662 " <td>Not-in-family</td>\n",663 " <td>White</td>\n",664 " <td>Male</td>\n",665 " <td>United-States</td>\n",666 " <td><=50K</td>\n",667 " </tr>\n",668 " <tr>\n",669 " <th>3</th>\n",670 " <td>Private</td>\n",671 " <td>11th</td>\n",672 " <td>Married-civ-spouse</td>\n",673 " <td>Handlers-cleaners</td>\n",674 " <td>Husband</td>\n",675 " <td>Black</td>\n",676 " <td>Male</td>\n",677 " <td>United-States</td>\n",678 " <td><=50K</td>\n",679 " </tr>\n",680 " <tr>\n",681 " <th>4</th>\n",682 " <td>Private</td>\n",683 " <td>Bachelors</td>\n",684 " <td>Married-civ-spouse</td>\n",685 " <td>Prof-specialty</td>\n",686 " <td>Wife</td>\n",687 " <td>Black</td>\n",688 " <td>Female</td>\n",689 " <td>Cuba</td>\n",690 " <td><=50K</td>\n",691 " </tr>\n",692 " </tbody>\n",693 "</table>\n",694 "</div>"695 ],696 "text/plain": [697 " WorkClass Education MaritalStatus Occupation \\\n",698 "0 State-gov Bachelors Never-married Adm-clerical \n",699 "1 Self-emp-not-inc Bachelors Married-civ-spouse Exec-managerial \n",700 "2 Private HS-grad Divorced Handlers-cleaners \n",701 "3 Private 11th Married-civ-spouse Handlers-cleaners \n",702 "4 Private Bachelors Married-civ-spouse Prof-specialty \n",703 "\n",704 " Relationship Race Sex Native Target \n",705 "0 Not-in-family White Male United-States <=50K \n",706 "1 Husband White Male United-States <=50K \n",707 "2 Not-in-family White Male United-States <=50K \n",708 "3 Husband Black Male United-States <=50K \n",709 "4 Wife Black Female Cuba <=50K "710 ]711 },712 "execution_count": 10,713 "metadata": {},714 "output_type": "execute_result"715 }716 ],717 "source": [718 "from sklearn import preprocessing\n",719 "\n",720 "# encode categorical variables using label Encoder\n",721 "\n",722 "# select all categorical variables\n",723 "df_categorical = df.select_dtypes(include=['object'])\n",724 "df_categorical.head()"725 ]726 },727 {728 "cell_type": "code",729 "execution_count": 11,730 "metadata": {},731 "outputs": [732 {733 "data": {734 "text/html": [735 "<div>\n",736 "<style scoped>\n",737 " .dataframe tbody tr th:only-of-type {\n",738 " vertical-align: middle;\n",739 " }\n",740 "\n",741 " .dataframe tbody tr th {\n",742 " vertical-align: top;\n",743 " }\n",744 "\n",745 " .dataframe thead th {\n",746 " text-align: right;\n",747 " }\n",748 "</style>\n",749 "<table border=\"1\" class=\"dataframe\">\n",750 " <thead>\n",751 " <tr style=\"text-align: right;\">\n",752 " <th></th>\n",753 " <th>WorkClass</th>\n",754 " <th>Education</th>\n",755 " <th>MaritalStatus</th>\n",756 " <th>Occupation</th>\n",757 " <th>Relationship</th>\n",758 " <th>Race</th>\n",759 " <th>Sex</th>\n",760 " <th>Native</th>\n",761 " <th>Target</th>\n",762 " </tr>\n",763 " </thead>\n",764 " <tbody>\n",765 " <tr>\n",766 " <th>0</th>\n",767 " <td>7</td>\n",768 " <td>9</td>\n",769 " <td>4</td>\n",770 " <td>1</td>\n",771 " <td>1</td>\n",772 " <td>4</td>\n",773 " <td>1</td>\n",774 " <td>39</td>\n",775 " <td>0</td>\n",776 " </tr>\n",777 " <tr>\n",778 " <th>1</th>\n",779 " <td>6</td>\n",780 " <td>9</td>\n",781 " <td>2</td>\n",782 " <td>4</td>\n",783 " <td>0</td>\n",784 " <td>4</td>\n",785 " <td>1</td>\n",786 " <td>39</td>\n",787 " <td>0</td>\n",788 " </tr>\n",789 " <tr>\n",790 " <th>2</th>\n",791 " <td>4</td>\n",792 " <td>11</td>\n",793 " <td>0</td>\n",794 " <td>6</td>\n",795 " <td>1</td>\n",796 " <td>4</td>\n",797 " <td>1</td>\n",798 " <td>39</td>\n",799 " <td>0</td>\n",800 " </tr>\n",801 " <tr>\n",802 " <th>3</th>\n",803 " <td>4</td>\n",804 " <td>1</td>\n",805 " <td>2</td>\n",806 " <td>6</td>\n",807 " <td>0</td>\n",808 " <td>2</td>\n",809 " <td>1</td>\n",810 " <td>39</td>\n",811 " <td>0</td>\n",812 " </tr>\n",813 " <tr>\n",814 " <th>4</th>\n",815 " <td>4</td>\n",816 " <td>9</td>\n",817 " <td>2</td>\n",818 " <td>10</td>\n",819 " <td>5</td>\n",820 " <td>2</td>\n",821 " <td>0</td>\n",822 " <td>5</td>\n",823 " <td>0</td>\n",824 " </tr>\n",825 " </tbody>\n",826 "</table>\n",827 "</div>"828 ],829 "text/plain": [830 " WorkClass Education MaritalStatus Occupation Relationship Race Sex \\\n",831 "0 7 9 4 1 1 4 1 \n",832 "1 6 9 2 4 0 4 1 \n",833 "2 4 11 0 6 1 4 1 \n",834 "3 4 1 2 6 0 2 1 \n",835 "4 4 9 2 10 5 2 0 \n",836 "\n",837 " Native Target \n",838 "0 39 0 \n",839 "1 39 0 \n",840 "2 39 0 \n",841 "3 39 0 \n",842 "4 5 0 "843 ]844 },845 "execution_count": 11,846 "metadata": {},847 "output_type": "execute_result"848 }849 ],850 "source": [851 "# apply label encoder to df_categorical\n",852 "le = preprocessing.LabelEncoder()\n",853 "df_categorical = df_categorical.apply(le.fit_transform)\n",854 "df_categorical.head()"855 ]856 },857 {858 "cell_type": "code",859 "execution_count": 12,860 "metadata": {},861 "outputs": [862 {863 "data": {864 "text/html": [865 "<div>\n",866 "<style scoped>\n",867 " .dataframe tbody tr th:only-of-type {\n",868 " vertical-align: middle;\n",869 " }\n",870 "\n",871 " .dataframe tbody tr th {\n",872 " vertical-align: top;\n",873 " }\n",874 "\n",875 " .dataframe thead th {\n",876 " text-align: right;\n",877 " }\n",878 "</style>\n",879 "<table border=\"1\" class=\"dataframe\">\n",880 " <thead>\n",881 " <tr style=\"text-align: right;\">\n",882 " <th></th>\n",883 " <th>Age</th>\n",884 " <th>Fnlwgt</th>\n",885 " <th>Edu_Num</th>\n",886 " <th>CapitalGain</th>\n",887 " <th>CapitalLoss</th>\n",888 " <th>HrPerWk</th>\n",889 " <th>WorkClass</th>\n",890 " <th>Education</th>\n",891 " <th>MaritalStatus</th>\n",892 " <th>Occupation</th>\n",893 " <th>Relationship</th>\n",894 " <th>Race</th>\n",895 " <th>Sex</th>\n",896 " <th>Native</th>\n",897 " <th>Target</th>\n",898 " </tr>\n",899 " </thead>\n",900 " <tbody>\n",901 " <tr>\n",902 " <th>0</th>\n",903 " <td>39</td>\n",904 " <td>77516</td>\n",905 " <td>13</td>\n",906 " <td>2174</td>\n",907 " <td>0</td>\n",908 " <td>40</td>\n",909 " <td>7</td>\n",910 " <td>9</td>\n",911 " <td>4</td>\n",912 " <td>1</td>\n",913 " <td>1</td>\n",914 " <td>4</td>\n",915 " <td>1</td>\n",916 " <td>39</td>\n",917 " <td>0</td>\n",918 " </tr>\n",919 " <tr>\n",920 " <th>1</th>\n",921 " <td>50</td>\n",922 " <td>83311</td>\n",923 " <td>13</td>\n",924 " <td>0</td>\n",925 " <td>0</td>\n",926 " <td>13</td>\n",927 " <td>6</td>\n",928 " <td>9</td>\n",929 " <td>2</td>\n",930 " <td>4</td>\n",931 " <td>0</td>\n",932 " <td>4</td>\n",933 " <td>1</td>\n",934 " <td>39</td>\n",935 " <td>0</td>\n",936 " </tr>\n",937 " <tr>\n",938 " <th>2</th>\n",939 " <td>38</td>\n",940 " <td>215646</td>\n",941 " <td>9</td>\n",942 " <td>0</td>\n",943 " <td>0</td>\n",944 " <td>40</td>\n",945 " <td>4</td>\n",946 " <td>11</td>\n",947 " <td>0</td>\n",948 " <td>6</td>\n",949 " <td>1</td>\n",950 " <td>4</td>\n",951 " <td>1</td>\n",952 " <td>39</td>\n",953 " <td>0</td>\n",954 " </tr>\n",955 " <tr>\n",956 " <th>3</th>\n",957 " <td>53</td>\n",958 " <td>234721</td>\n",959 " <td>7</td>\n",960 " <td>0</td>\n",961 " <td>0</td>\n",962 " <td>40</td>\n",963 " <td>4</td>\n",964 " <td>1</td>\n",965 " <td>2</td>\n",966 " <td>6</td>\n",967 " <td>0</td>\n",968 " <td>2</td>\n",969 " <td>1</td>\n",970 " <td>39</td>\n",971 " <td>0</td>\n",972 " </tr>\n",973 " <tr>\n",974 " <th>4</th>\n",975 " <td>28</td>\n",976 " <td>338409</td>\n",977 " <td>13</td>\n",978 " <td>0</td>\n",979 " <td>0</td>\n",980 " <td>40</td>\n",981 " <td>4</td>\n",982 " <td>9</td>\n",983 " <td>2</td>\n",984 " <td>10</td>\n",985 " <td>5</td>\n",986 " <td>2</td>\n",987 " <td>0</td>\n",988 " <td>5</td>\n",989 " <td>0</td>\n",990 " </tr>\n",991 " </tbody>\n",992 "</table>\n",993 "</div>"994 ],995 "text/plain": [996 " Age Fnlwgt Edu_Num CapitalGain CapitalLoss HrPerWk WorkClass \\\n",997 "0 39 77516 13 2174 0 40 7 \n",998 "1 50 83311 13 0 0 13 6 \n",999 "2 38 215646 9 0 0 40 4 \n",1000 "3 53 234721 7 0 0 40 4 \n",1001 "4 28 338409 13 0 0 40 4 \n",1002 "\n",1003 " Education MaritalStatus Occupation Relationship Race Sex Native \\\n",1004 "0 9 4 1 1 4 1 39 \n",1005 "1 9 2 4 0 4 1 39 \n",1006 "2 11 0 6 1 4 1 39 \n",1007 "3 1 2 6 0 2 1 39 \n",1008 "4 9 2 10 5 2 0 5 \n",1009 "\n",1010 " Target \n",1011 "0 0 \n",1012 "1 0 \n",1013 "2 0 \n",1014 "3 0 \n",1015 "4 0 "1016 ]1017 },1018 "execution_count": 12,1019 "metadata": {},1020 "output_type": "execute_result"1021 }1022 ],1023 "source": [1024 "# Next, Concatenate df_categorical dataframe with original df (dataframe)\n",1025 "\n",1026 "# first, Drop earlier duplicate columns which had categorical values\n",1027 "df = df.drop(df_categorical.columns,axis=1)\n",1028 "df = pd.concat([df,df_categorical],axis=1)\n",1029 "df.head()"1030 ]1031 },1032 {1033 "cell_type": "code",1034 "execution_count": 13,1035 "metadata": {},1036 "outputs": [1037 {1038 "data": {1039 "text/html": [1040 "<div>\n",1041 "<style scoped>\n",1042 " .dataframe tbody tr th:only-of-type {\n",1043 " vertical-align: middle;\n",1044 " }\n",1045 "\n",1046 " .dataframe tbody tr th {\n",1047 " vertical-align: top;\n",1048 " }\n",1049 "\n",1050 " .dataframe thead th {\n",1051 " text-align: right;\n",1052 " }\n",1053 "</style>\n",1054 "<table border=\"1\" class=\"dataframe\">\n",1055 " <thead>\n",1056 " <tr style=\"text-align: right;\">\n",1057 " <th></th>\n",1058 " <th>Age</th>\n",1059 " <th>Fnlwgt</th>\n",1060 " <th>Edu_Num</th>\n",1061 " <th>CapitalGain</th>\n",1062 " <th>CapitalLoss</th>\n",1063 " <th>HrPerWk</th>\n",1064 " <th>WorkClass</th>\n",1065 " <th>Education</th>\n",1066 " <th>MaritalStatus</th>\n",1067 " <th>Occupation</th>\n",1068 " <th>Relationship</th>\n",1069 " <th>Race</th>\n",1070 " <th>Sex</th>\n",1071 " <th>Native</th>\n",1072 " <th>Target</th>\n",1073 " </tr>\n",1074 " </thead>\n",1075 " <tbody>\n",1076 " <tr>\n",1077 " <th>Age</th>\n",1078 " <td>1.000000</td>\n",1079 " <td>-0.076646</td>\n",1080 " <td>0.036527</td>\n",1081 " <td>0.077674</td>\n",1082 " <td>0.057775</td>\n",1083 " <td>0.068756</td>\n",1084 " <td>0.003787</td>\n",1085 " <td>-0.010508</td>\n",1086 " <td>-0.266288</td>\n",1087 " <td>-0.020947</td>\n",1088 " <td>-0.263698</td>\n",1089 " <td>0.028718</td>\n",1090 " <td>0.088832</td>\n",1091 " <td>-0.001151</td>\n",1092 " <td>0.234037</td>\n",1093 " </tr>\n",1094 " <tr>\n",1095 " <th>Fnlwgt</th>\n",1096 " <td>-0.076646</td>\n",1097 " <td>1.000000</td>\n",1098 " <td>-0.043195</td>\n",1099 " <td>0.000432</td>\n",1100 " <td>-0.010252</td>\n",1101 " <td>-0.018768</td>\n",1102 " <td>-0.016656</td>\n",1103 " <td>-0.028145</td>\n",1104 " <td>0.028153</td>\n",1105 " <td>0.001597</td>\n",1106 " <td>0.008931</td>\n",1107 " <td>-0.021291</td>\n",1108 " <td>0.026858</td>\n",1109 " <td>-0.051966</td>\n",1110 " <td>-0.009463</td>\n",1111 " </tr>\n",1112 " <tr>\n",1113 " <th>Edu_Num</th>\n",1114 " <td>0.036527</td>\n",1115 " <td>-0.043195</td>\n",1116 " <td>1.000000</td>\n",1117 " <td>0.122630</td>\n",1118 " <td>0.079923</td>\n",1119 " <td>0.148123</td>\n",1120 " <td>0.052085</td>\n",1121 " <td>0.359153</td>\n",1122 " <td>-0.069304</td>\n",1123 " <td>0.109697</td>\n",1124 " <td>-0.094153</td>\n",1125 " <td>0.031838</td>\n",1126 " <td>0.012280</td>\n",1127 " <td>0.050840</td>\n",1128 " <td>0.335154</td>\n",1129 " </tr>\n",1130 " <tr>\n",1131 " <th>CapitalGain</th>\n",1132 " <td>0.077674</td>\n",1133 " <td>0.000432</td>\n",1134 " <td>0.122630</td>\n",1135 " <td>1.000000</td>\n",1136 " <td>-0.031615</td>\n",1137 " <td>0.078409</td>\n",1138 " <td>0.033835</td>\n",1139 " <td>0.030046</td>\n",1140 " <td>-0.043393</td>\n",1141 " <td>0.025505</td>\n",1142 " <td>-0.057919</td>\n",1143 " <td>0.011145</td>\n",1144 " <td>0.048480</td>\n",1145 " <td>-0.001982</td>\n",1146 " <td>0.223329</td>\n",1147 " </tr>\n",1148 " <tr>\n",1149 " <th>CapitalLoss</th>\n",1150 " <td>0.057775</td>\n",1151 " <td>-0.010252</td>\n",1152 " <td>0.079923</td>\n",1153 " <td>-0.031615</td>\n",1154 " <td>1.000000</td>\n",1155 " <td>0.054256</td>\n",1156 " <td>0.012216</td>\n",1157 " <td>0.016746</td>\n",1158 " <td>-0.034187</td>\n",1159 " <td>0.017987</td>\n",1160 " <td>-0.061062</td>\n",1161 " <td>0.018899</td>\n",1162 " <td>0.045567</td>\n",1163 " <td>0.000419</td>\n",1164 " <td>0.150526</td>\n",1165 " </tr>\n",1166 " <tr>\n",1167 " <th>HrPerWk</th>\n",1168 " <td>0.068756</td>\n",1169 " <td>-0.018768</td>\n",1170 " <td>0.148123</td>\n",1171 " <td>0.078409</td>\n",1172 " <td>0.054256</td>\n",1173 " <td>1.000000</td>\n",1174 " <td>0.138962</td>\n",1175 " <td>0.055510</td>\n",1176 " <td>-0.190519</td>\n",1177 " <td>0.080383</td>\n",1178 " <td>-0.248974</td>\n",1179 " <td>0.041910</td>\n",1180 " <td>0.229309</td>\n",1181 " <td>-0.002671</td>\n",1182 " <td>0.229689</td>\n",1183 " </tr>\n",1184 " <tr>\n",1185 " <th>WorkClass</th>\n",1186 " <td>0.003787</td>\n",1187 " <td>-0.016656</td>\n",1188 " <td>0.052085</td>\n",1189 " <td>0.033835</td>\n",1190 " <td>0.012216</td>\n",1191 " <td>0.138962</td>\n",1192 " <td>1.000000</td>\n",1193 " <td>0.023513</td>\n",1194 " <td>-0.064731</td>\n",1195 " <td>0.254892</td>\n",1196 " <td>-0.090461</td>\n",1197 " <td>0.049742</td>\n",1198 " <td>0.095981</td>\n",1199 " <td>-0.007690</td>\n",1200 " <td>0.051604</td>\n",