Rafa1986/Data-Analytics-Class
0
1{2 "cells": [3 {4 "cell_type": "markdown",5 "metadata": {},6 "source": [7 "### TU257 - AutoML Demo"8 ]9 },10 {11 "cell_type": "markdown",12 "metadata": {},13 "source": [14 "#### Run this notebook and examine what is produced for each cell.\n",15 "\n",16 "#### Add your own comments to enrich the information given to make it more meaningful to you.\n",17 "\n",18 "#### In this notebook we will have a look at two different AutoML libraries. See the notes/website for links to posts illustrating other AutoML libraries"19 ]20 },21 {22 "cell_type": "code",23 "execution_count": 1,24 "metadata": {},25 "outputs": [26 {27 "data": {28 "text/html": [29 "<div>\n",30 "<style scoped>\n",31 " .dataframe tbody tr th:only-of-type {\n",32 " vertical-align: middle;\n",33 " }\n",34 "\n",35 " .dataframe tbody tr th {\n",36 " vertical-align: top;\n",37 " }\n",38 "\n",39 " .dataframe thead th {\n",40 " text-align: right;\n",41 " }\n",42 "</style>\n",43 "<table border=\"1\" class=\"dataframe\">\n",44 " <thead>\n",45 " <tr style=\"text-align: right;\">\n",46 " <th></th>\n",47 " <th>Age</th>\n",48 " <th>WorkClass</th>\n",49 " <th>Fnlwgt</th>\n",50 " <th>Education</th>\n",51 " <th>Edu_Num</th>\n",52 " <th>MaritalStatus</th>\n",53 " <th>Occupation</th>\n",54 " <th>Relationship</th>\n",55 " <th>Race</th>\n",56 " <th>Sex</th>\n",57 " <th>CapitalGain</th>\n",58 " <th>CapitalLoss</th>\n",59 " <th>HrPerWk</th>\n",60 " <th>Native</th>\n",61 " <th>Target</th>\n",62 " </tr>\n",63 " </thead>\n",64 " <tbody>\n",65 " <tr>\n",66 " <th>0</th>\n",67 " <td>39</td>\n",68 " <td>State-gov</td>\n",69 " <td>77516</td>\n",70 " <td>Bachelors</td>\n",71 " <td>13</td>\n",72 " <td>Never-married</td>\n",73 " <td>Adm-clerical</td>\n",74 " <td>Not-in-family</td>\n",75 " <td>White</td>\n",76 " <td>Male</td>\n",77 " <td>2174</td>\n",78 " <td>0</td>\n",79 " <td>40</td>\n",80 " <td>United-States</td>\n",81 " <td><=50K</td>\n",82 " </tr>\n",83 " <tr>\n",84 " <th>1</th>\n",85 " <td>50</td>\n",86 " <td>Self-emp-not-inc</td>\n",87 " <td>83311</td>\n",88 " <td>Bachelors</td>\n",89 " <td>13</td>\n",90 " <td>Married-civ-spouse</td>\n",91 " <td>Exec-managerial</td>\n",92 " <td>Husband</td>\n",93 " <td>White</td>\n",94 " <td>Male</td>\n",95 " <td>0</td>\n",96 " <td>0</td>\n",97 " <td>13</td>\n",98 " <td>United-States</td>\n",99 " <td><=50K</td>\n",100 " </tr>\n",101 " <tr>\n",102 " <th>2</th>\n",103 " <td>38</td>\n",104 " <td>Private</td>\n",105 " <td>215646</td>\n",106 " <td>HS-grad</td>\n",107 " <td>9</td>\n",108 " <td>Divorced</td>\n",109 " <td>Handlers-cleaners</td>\n",110 " <td>Not-in-family</td>\n",111 " <td>White</td>\n",112 " <td>Male</td>\n",113 " <td>0</td>\n",114 " <td>0</td>\n",115 " <td>40</td>\n",116 " <td>United-States</td>\n",117 " <td><=50K</td>\n",118 " </tr>\n",119 " <tr>\n",120 " <th>3</th>\n",121 " <td>53</td>\n",122 " <td>Private</td>\n",123 " <td>234721</td>\n",124 " <td>11th</td>\n",125 " <td>7</td>\n",126 " <td>Married-civ-spouse</td>\n",127 " <td>Handlers-cleaners</td>\n",128 " <td>Husband</td>\n",129 " <td>Black</td>\n",130 " <td>Male</td>\n",131 " <td>0</td>\n",132 " <td>0</td>\n",133 " <td>40</td>\n",134 " <td>United-States</td>\n",135 " <td><=50K</td>\n",136 " </tr>\n",137 " <tr>\n",138 " <th>4</th>\n",139 " <td>28</td>\n",140 " <td>Private</td>\n",141 " <td>338409</td>\n",142 " <td>Bachelors</td>\n",143 " <td>13</td>\n",144 " <td>Married-civ-spouse</td>\n",145 " <td>Prof-specialty</td>\n",146 " <td>Wife</td>\n",147 " <td>Black</td>\n",148 " <td>Female</td>\n",149 " <td>0</td>\n",150 " <td>0</td>\n",151 " <td>40</td>\n",152 " <td>Cuba</td>\n",153 " <td><=50K</td>\n",154 " </tr>\n",155 " <tr>\n",156 " <th>5</th>\n",157 " <td>37</td>\n",158 " <td>Private</td>\n",159 " <td>284582</td>\n",160 " <td>Masters</td>\n",161 " <td>14</td>\n",162 " <td>Married-civ-spouse</td>\n",163 " <td>Exec-managerial</td>\n",164 " <td>Wife</td>\n",165 " <td>White</td>\n",166 " <td>Female</td>\n",167 " <td>0</td>\n",168 " <td>0</td>\n",169 " <td>40</td>\n",170 " <td>United-States</td>\n",171 " <td><=50K</td>\n",172 " </tr>\n",173 " <tr>\n",174 " <th>6</th>\n",175 " <td>49</td>\n",176 " <td>Private</td>\n",177 " <td>160187</td>\n",178 " <td>9th</td>\n",179 " <td>5</td>\n",180 " <td>Married-spouse-absent</td>\n",181 " <td>Other-service</td>\n",182 " <td>Not-in-family</td>\n",183 " <td>Black</td>\n",184 " <td>Female</td>\n",185 " <td>0</td>\n",186 " <td>0</td>\n",187 " <td>16</td>\n",188 " <td>Jamaica</td>\n",189 " <td><=50K</td>\n",190 " </tr>\n",191 " <tr>\n",192 " <th>7</th>\n",193 " <td>52</td>\n",194 " <td>Self-emp-not-inc</td>\n",195 " <td>209642</td>\n",196 " <td>HS-grad</td>\n",197 " <td>9</td>\n",198 " <td>Married-civ-spouse</td>\n",199 " <td>Exec-managerial</td>\n",200 " <td>Husband</td>\n",201 " <td>White</td>\n",202 " <td>Male</td>\n",203 " <td>0</td>\n",204 " <td>0</td>\n",205 " <td>45</td>\n",206 " <td>United-States</td>\n",207 " <td>>50K</td>\n",208 " </tr>\n",209 " <tr>\n",210 " <th>8</th>\n",211 " <td>31</td>\n",212 " <td>Private</td>\n",213 " <td>45781</td>\n",214 " <td>Masters</td>\n",215 " <td>14</td>\n",216 " <td>Never-married</td>\n",217 " <td>Prof-specialty</td>\n",218 " <td>Not-in-family</td>\n",219 " <td>White</td>\n",220 " <td>Female</td>\n",221 " <td>14084</td>\n",222 " <td>0</td>\n",223 " <td>50</td>\n",224 " <td>United-States</td>\n",225 " <td>>50K</td>\n",226 " </tr>\n",227 " <tr>\n",228 " <th>9</th>\n",229 " <td>42</td>\n",230 " <td>Private</td>\n",231 " <td>159449</td>\n",232 " <td>Bachelors</td>\n",233 " <td>13</td>\n",234 " <td>Married-civ-spouse</td>\n",235 " <td>Exec-managerial</td>\n",236 " <td>Husband</td>\n",237 " <td>White</td>\n",238 " <td>Male</td>\n",239 " <td>5178</td>\n",240 " <td>0</td>\n",241 " <td>40</td>\n",242 " <td>United-States</td>\n",243 " <td>>50K</td>\n",244 " </tr>\n",245 " </tbody>\n",246 "</table>\n",247 "</div>"248 ],249 "text/plain": [250 " Age WorkClass Fnlwgt Education Edu_Num \\\n",251 "0 39 State-gov 77516 Bachelors 13 \n",252 "1 50 Self-emp-not-inc 83311 Bachelors 13 \n",253 "2 38 Private 215646 HS-grad 9 \n",254 "3 53 Private 234721 11th 7 \n",255 "4 28 Private 338409 Bachelors 13 \n",256 "5 37 Private 284582 Masters 14 \n",257 "6 49 Private 160187 9th 5 \n",258 "7 52 Self-emp-not-inc 209642 HS-grad 9 \n",259 "8 31 Private 45781 Masters 14 \n",260 "9 42 Private 159449 Bachelors 13 \n",261 "\n",262 " MaritalStatus Occupation Relationship Race \\\n",263 "0 Never-married Adm-clerical Not-in-family White \n",264 "1 Married-civ-spouse Exec-managerial Husband White \n",265 "2 Divorced Handlers-cleaners Not-in-family White \n",266 "3 Married-civ-spouse Handlers-cleaners Husband Black \n",267 "4 Married-civ-spouse Prof-specialty Wife Black \n",268 "5 Married-civ-spouse Exec-managerial Wife White \n",269 "6 Married-spouse-absent Other-service Not-in-family Black \n",270 "7 Married-civ-spouse Exec-managerial Husband White \n",271 "8 Never-married Prof-specialty Not-in-family White \n",272 "9 Married-civ-spouse Exec-managerial Husband White \n",273 "\n",274 " Sex CapitalGain CapitalLoss HrPerWk Native Target \n",275 "0 Male 2174 0 40 United-States <=50K \n",276 "1 Male 0 0 13 United-States <=50K \n",277 "2 Male 0 0 40 United-States <=50K \n",278 "3 Male 0 0 40 United-States <=50K \n",279 "4 Female 0 0 40 Cuba <=50K \n",280 "5 Female 0 0 40 United-States <=50K \n",281 "6 Female 0 0 16 Jamaica <=50K \n",282 "7 Male 0 0 45 United-States >50K \n",283 "8 Female 14084 0 50 United-States >50K \n",284 "9 Male 5178 0 40 United-States >50K "285 ]286 },287 "execution_count": 1,288 "metadata": {},289 "output_type": "execute_result"290 }291 ],292 "source": [293 "import pandas as pd\n",294 "\n",295 "#Load in the dataset\n",296 "colnames=['Age', 'WorkClass', 'Fnlwgt', 'Education', 'Edu_Num', 'MaritalStatus', 'Occupation', 'Relationship', 'Race', 'Sex', 'CapitalGain', 'CapitalLoss', 'HrPerWk', 'Native', 'Target'] \n",297 "df = pd.read_csv('/Users/brendan.tierney/Dropbox/4-Datasets/adult.csv', names=colnames, header=None)\n",298 "df.head(10)"299 ]300 },301 {302 "cell_type": "code",303 "execution_count": 2,304 "metadata": {},305 "outputs": [306 {307 "data": {308 "text/plain": [309 "False"310 ]311 },312 "execution_count": 2,313 "metadata": {},314 "output_type": "execute_result"315 }316 ],317 "source": [318 "df.isnull().values.any()"319 ]320 },321 {322 "cell_type": "code",323 "execution_count": 3,324 "metadata": {},325 "outputs": [326 {327 "name": "stdout",328 "output_type": "stream",329 "text": [330 "Rows : 32561\n",331 "Columns : 15\n",332 "\n",333 "Features : \n",334 " ['Age', 'WorkClass', 'Fnlwgt', 'Education', 'Edu_Num', 'MaritalStatus', 'Occupation', 'Relationship', 'Race', 'Sex', 'CapitalGain', 'CapitalLoss', 'HrPerWk', 'Native', 'Target']\n",335 "\n",336 "Missing values : 0\n",337 "\n",338 "Unique values : \n",339 " Age 73\n",340 "WorkClass 9\n",341 "Fnlwgt 21648\n",342 "Education 16\n",343 "Edu_Num 16\n",344 "MaritalStatus 7\n",345 "Occupation 15\n",346 "Relationship 6\n",347 "Race 5\n",348 "Sex 2\n",349 "CapitalGain 119\n",350 "CapitalLoss 92\n",351 "HrPerWk 94\n",352 "Native 42\n",353 "Target 2\n",354 "dtype: int64\n"355 ]356 }357 ],358 "source": [359 "print (\"Rows : \" ,df.shape[0])\n",360 "print (\"Columns : \" ,df.shape[1])\n",361 "print (\"\\nFeatures : \\n\" ,df.columns.tolist())\n",362 "print (\"\\nMissing values : \", df.isnull().sum().values.sum())\n",363 "print (\"\\nUnique values : \\n\",df.nunique())"364 ]365 },366 {367 "cell_type": "code",368 "execution_count": 4,369 "metadata": {},370 "outputs": [371 {372 "name": "stdout",373 "output_type": "stream",374 "text": [375 "<class 'pandas.core.frame.DataFrame'>\n",376 "RangeIndex: 32561 entries, 0 to 32560\n",377 "Data columns (total 15 columns):\n",378 " # Column Non-Null Count Dtype \n",379 "--- ------ -------------- ----- \n",380 " 0 Age 32561 non-null int64 \n",381 " 1 WorkClass 32561 non-null object\n",382 " 2 Fnlwgt 32561 non-null int64 \n",383 " 3 Education 32561 non-null object\n",384 " 4 Edu_Num 32561 non-null int64 \n",385 " 5 MaritalStatus 32561 non-null object\n",386 " 6 Occupation 32561 non-null object\n",387 " 7 Relationship 32561 non-null object\n",388 " 8 Race 32561 non-null object\n",389 " 9 Sex 32561 non-null object\n",390 " 10 CapitalGain 32561 non-null int64 \n",391 " 11 CapitalLoss 32561 non-null int64 \n",392 " 12 HrPerWk 32561 non-null int64 \n",393 " 13 Native 32561 non-null object\n",394 " 14 Target 32561 non-null object\n",395 "dtypes: int64(6), object(9)\n",396 "memory usage: 3.7+ MB\n"397 ]398 }399 ],400 "source": [401 "df.info()"402 ]403 },404 {405 "cell_type": "code",406 "execution_count": 5,407 "metadata": {},408 "outputs": [409 {410 "data": {411 "text/html": [412 "<div>\n",413 "<style scoped>\n",414 " .dataframe tbody tr th:only-of-type {\n",415 " vertical-align: middle;\n",416 " }\n",417 "\n",418 " .dataframe tbody tr th {\n",419 " vertical-align: top;\n",420 " }\n",421 "\n",422 " .dataframe thead th {\n",423 " text-align: right;\n",424 " }\n",425 "</style>\n",426 "<table border=\"1\" class=\"dataframe\">\n",427 " <thead>\n",428 " <tr style=\"text-align: right;\">\n",429 " <th></th>\n",430 " <th>Age</th>\n",431 " <th>Fnlwgt</th>\n",432 " <th>Edu_Num</th>\n",433 " <th>CapitalGain</th>\n",434 " <th>CapitalLoss</th>\n",435 " <th>HrPerWk</th>\n",436 " </tr>\n",437 " </thead>\n",438 " <tbody>\n",439 " <tr>\n",440 " <th>count</th>\n",441 " <td>32561.000000</td>\n",442 " <td>3.256100e+04</td>\n",443 " <td>32561.000000</td>\n",444 " <td>32561.000000</td>\n",445 " <td>32561.000000</td>\n",446 " <td>32561.000000</td>\n",447 " </tr>\n",448 " <tr>\n",449 " <th>mean</th>\n",450 " <td>38.581647</td>\n",451 " <td>1.897784e+05</td>\n",452 " <td>10.080679</td>\n",453 " <td>1077.648844</td>\n",454 " <td>87.303830</td>\n",455 " <td>40.437456</td>\n",456 " </tr>\n",457 " <tr>\n",458 " <th>std</th>\n",459 " <td>13.640433</td>\n",460 " <td>1.055500e+05</td>\n",461 " <td>2.572720</td>\n",462 " <td>7385.292085</td>\n",463 " <td>402.960219</td>\n",464 " <td>12.347429</td>\n",465 " </tr>\n",466 " <tr>\n",467 " <th>min</th>\n",468 " <td>17.000000</td>\n",469 " <td>1.228500e+04</td>\n",470 " <td>1.000000</td>\n",471 " <td>0.000000</td>\n",472 " <td>0.000000</td>\n",473 " <td>1.000000</td>\n",474 " </tr>\n",475 " <tr>\n",476 " <th>25%</th>\n",477 " <td>28.000000</td>\n",478 " <td>1.178270e+05</td>\n",479 " <td>9.000000</td>\n",480 " <td>0.000000</td>\n",481 " <td>0.000000</td>\n",482 " <td>40.000000</td>\n",483 " </tr>\n",484 " <tr>\n",485 " <th>50%</th>\n",486 " <td>37.000000</td>\n",487 " <td>1.783560e+05</td>\n",488 " <td>10.000000</td>\n",489 " <td>0.000000</td>\n",490 " <td>0.000000</td>\n",491 " <td>40.000000</td>\n",492 " </tr>\n",493 " <tr>\n",494 " <th>75%</th>\n",495 " <td>48.000000</td>\n",496 " <td>2.370510e+05</td>\n",497 " <td>12.000000</td>\n",498 " <td>0.000000</td>\n",499 " <td>0.000000</td>\n",500 " <td>45.000000</td>\n",501 " </tr>\n",502 " <tr>\n",503 " <th>max</th>\n",504 " <td>90.000000</td>\n",505 " <td>1.484705e+06</td>\n",506 " <td>16.000000</td>\n",507 " <td>99999.000000</td>\n",508 " <td>4356.000000</td>\n",509 " <td>99.000000</td>\n",510 " </tr>\n",511 " </tbody>\n",512 "</table>\n",513 "</div>"514 ],515 "text/plain": [516 " Age Fnlwgt Edu_Num CapitalGain CapitalLoss \\\n",517 "count 32561.000000 3.256100e+04 32561.000000 32561.000000 32561.000000 \n",518 "mean 38.581647 1.897784e+05 10.080679 1077.648844 87.303830 \n",519 "std 13.640433 1.055500e+05 2.572720 7385.292085 402.960219 \n",520 "min 17.000000 1.228500e+04 1.000000 0.000000 0.000000 \n",521 "25% 28.000000 1.178270e+05 9.000000 0.000000 0.000000 \n",522 "50% 37.000000 1.783560e+05 10.000000 0.000000 0.000000 \n",523 "75% 48.000000 2.370510e+05 12.000000 0.000000 0.000000 \n",524 "max 90.000000 1.484705e+06 16.000000 99999.000000 4356.000000 \n",525 "\n",526 " HrPerWk \n",527 "count 32561.000000 \n",528 "mean 40.437456 \n",529 "std 12.347429 \n",530 "min 1.000000 \n",531 "25% 40.000000 \n",532 "50% 40.000000 \n",533 "75% 45.000000 \n",534 "max 99.000000 "535 ]536 },537 "execution_count": 5,538 "metadata": {},539 "output_type": "execute_result"540 }541 ],542 "source": [543 "# Numerical feature of summary/description \n",544 "df.describe()"545 ]546 },547 {548 "cell_type": "code",549 "execution_count": 6,550 "metadata": {},551 "outputs": [552 {553 "data": {554 "text/plain": [555 "Age 0\n",556 "WorkClass 0\n",557 "Fnlwgt 0\n",558 "Education 0\n",559 "Edu_Num 0\n",560 "MaritalStatus 0\n",561 "Occupation 0\n",562 "Relationship 0\n",563 "Race 0\n",564 "Sex 0\n",565 "CapitalGain 0\n",566 "CapitalLoss 0\n",567 "HrPerWk 0\n",568 "Native 0\n",569 "Target 0\n",570 "dtype: int64"571 ]572 },573 "execution_count": 6,574 "metadata": {},575 "output_type": "execute_result"576 }577 ],578 "source": [579 "# checking \"?\" values, how many are there in the whole dataset\n",580 "df_missing = (df=='?').sum()\n",581 "df_missing"582 ]583 },584 {585 "cell_type": "code",586 "execution_count": 7,587 "metadata": {},588 "outputs": [589 {590 "data": {591 "text/plain": [592 " <=50K 0.75919\n",593 " >50K 0.24081\n",594 "Name: Target, dtype: float64"595 ]596 },597 "execution_count": 7,598 "metadata": {},599 "output_type": "execute_result"600 }601 ],602 "source": [603 "df['Target'].value_counts()/len(df) #calculate percentages"604 ]605 },606 {607 "cell_type": "code",608 "execution_count": 8,609 "metadata": {},610 "outputs": [611 {612 "data": {613 "text/html": [614 "<div>\n",615 "<style scoped>\n",616 " .dataframe tbody tr th:only-of-type {\n",617 " vertical-align: middle;\n",618 " }\n",619 "\n",620 " .dataframe tbody tr th {\n",621 " vertical-align: top;\n",622 " }\n",623 "\n",624 " .dataframe thead th {\n",625 " text-align: right;\n",626 " }\n",627 "</style>\n",628 "<table border=\"1\" class=\"dataframe\">\n",629 " <thead>\n",630 " <tr style=\"text-align: right;\">\n",631 " <th></th>\n",632 " <th>WorkClass</th>\n",633 " <th>Education</th>\n",634 " <th>MaritalStatus</th>\n",635 " <th>Occupation</th>\n",636 " <th>Relationship</th>\n",637 " <th>Race</th>\n",638 " <th>Sex</th>\n",639 " <th>Native</th>\n",640 " <th>Target</th>\n",641 " </tr>\n",642 " </thead>\n",643 " <tbody>\n",644 " <tr>\n",645 " <th>0</th>\n",646 " <td>State-gov</td>\n",647 " <td>Bachelors</td>\n",648 " <td>Never-married</td>\n",649 " <td>Adm-clerical</td>\n",650 " <td>Not-in-family</td>\n",651 " <td>White</td>\n",652 " <td>Male</td>\n",653 " <td>United-States</td>\n",654 " <td><=50K</td>\n",655 " </tr>\n",656 " <tr>\n",657 " <th>1</th>\n",658 " <td>Self-emp-not-inc</td>\n",659 " <td>Bachelors</td>\n",660 " <td>Married-civ-spouse</td>\n",661 " <td>Exec-managerial</td>\n",662 " <td>Husband</td>\n",663 " <td>White</td>\n",664 " <td>Male</td>\n",665 " <td>United-States</td>\n",666 " <td><=50K</td>\n",667 " </tr>\n",668 " <tr>\n",669 " <th>2</th>\n",670 " <td>Private</td>\n",671 " <td>HS-grad</td>\n",672 " <td>Divorced</td>\n",673 " <td>Handlers-cleaners</td>\n",674 " <td>Not-in-family</td>\n",675 " <td>White</td>\n",676 " <td>Male</td>\n",677 " <td>United-States</td>\n",678 " <td><=50K</td>\n",679 " </tr>\n",680 " <tr>\n",681 " <th>3</th>\n",682 " <td>Private</td>\n",683 " <td>11th</td>\n",684 " <td>Married-civ-spouse</td>\n",685 " <td>Handlers-cleaners</td>\n",686 " <td>Husband</td>\n",687 " <td>Black</td>\n",688 " <td>Male</td>\n",689 " <td>United-States</td>\n",690 " <td><=50K</td>\n",691 " </tr>\n",692 " <tr>\n",693 " <th>4</th>\n",694 " <td>Private</td>\n",695 " <td>Bachelors</td>\n",696 " <td>Married-civ-spouse</td>\n",697 " <td>Prof-specialty</td>\n",698 " <td>Wife</td>\n",699 " <td>Black</td>\n",700 " <td>Female</td>\n",701 " <td>Cuba</td>\n",702 " <td><=50K</td>\n",703 " </tr>\n",704 " </tbody>\n",705 "</table>\n",706 "</div>"707 ],708 "text/plain": [709 " WorkClass Education MaritalStatus Occupation \\\n",710 "0 State-gov Bachelors Never-married Adm-clerical \n",711 "1 Self-emp-not-inc Bachelors Married-civ-spouse Exec-managerial \n",712 "2 Private HS-grad Divorced Handlers-cleaners \n",713 "3 Private 11th Married-civ-spouse Handlers-cleaners \n",714 "4 Private Bachelors Married-civ-spouse Prof-specialty \n",715 "\n",716 " Relationship Race Sex Native Target \n",717 "0 Not-in-family White Male United-States <=50K \n",718 "1 Husband White Male United-States <=50K \n",719 "2 Not-in-family White Male United-States <=50K \n",720 "3 Husband Black Male United-States <=50K \n",721 "4 Wife Black Female Cuba <=50K "722 ]723 },724 "execution_count": 8,725 "metadata": {},726 "output_type": "execute_result"727 }728 ],729 "source": [730 "from sklearn import preprocessing\n",731 "\n",732 "# encode categorical variables using label Encoder\n",733 "\n",734 "# select all categorical variables\n",735 "df_categorical = df.select_dtypes(include=['object'])\n",736 "df_categorical.head()"737 ]738 },739 {740 "cell_type": "code",741 "execution_count": 9,742 "metadata": {},743 "outputs": [744 {745 "data": {746 "text/html": [747 "<div>\n",748 "<style scoped>\n",749 " .dataframe tbody tr th:only-of-type {\n",750 " vertical-align: middle;\n",751 " }\n",752 "\n",753 " .dataframe tbody tr th {\n",754 " vertical-align: top;\n",755 " }\n",756 "\n",757 " .dataframe thead th {\n",758 " text-align: right;\n",759 " }\n",760 "</style>\n",761 "<table border=\"1\" class=\"dataframe\">\n",762 " <thead>\n",763 " <tr style=\"text-align: right;\">\n",764 " <th></th>\n",765 " <th>WorkClass</th>\n",766 " <th>Education</th>\n",767 " <th>MaritalStatus</th>\n",768 " <th>Occupation</th>\n",769 " <th>Relationship</th>\n",770 " <th>Race</th>\n",771 " <th>Sex</th>\n",772 " <th>Native</th>\n",773 " <th>Target</th>\n",774 " </tr>\n",775 " </thead>\n",776 " <tbody>\n",777 " <tr>\n",778 " <th>0</th>\n",779 " <td>7</td>\n",780 " <td>9</td>\n",781 " <td>4</td>\n",782 " <td>1</td>\n",783 " <td>1</td>\n",784 " <td>4</td>\n",785 " <td>1</td>\n",786 " <td>39</td>\n",787 " <td>0</td>\n",788 " </tr>\n",789 " <tr>\n",790 " <th>1</th>\n",791 " <td>6</td>\n",792 " <td>9</td>\n",793 " <td>2</td>\n",794 " <td>4</td>\n",795 " <td>0</td>\n",796 " <td>4</td>\n",797 " <td>1</td>\n",798 " <td>39</td>\n",799 " <td>0</td>\n",800 " </tr>\n",801 " <tr>\n",802 " <th>2</th>\n",803 " <td>4</td>\n",804 " <td>11</td>\n",805 " <td>0</td>\n",806 " <td>6</td>\n",807 " <td>1</td>\n",808 " <td>4</td>\n",809 " <td>1</td>\n",810 " <td>39</td>\n",811 " <td>0</td>\n",812 " </tr>\n",813 " <tr>\n",814 " <th>3</th>\n",815 " <td>4</td>\n",816 " <td>1</td>\n",817 " <td>2</td>\n",818 " <td>6</td>\n",819 " <td>0</td>\n",820 " <td>2</td>\n",821 " <td>1</td>\n",822 " <td>39</td>\n",823 " <td>0</td>\n",824 " </tr>\n",825 " <tr>\n",826 " <th>4</th>\n",827 " <td>4</td>\n",828 " <td>9</td>\n",829 " <td>2</td>\n",830 " <td>10</td>\n",831 " <td>5</td>\n",832 " <td>2</td>\n",833 " <td>0</td>\n",834 " <td>5</td>\n",835 " <td>0</td>\n",836 " </tr>\n",837 " </tbody>\n",838 "</table>\n",839 "</div>"840 ],841 "text/plain": [842 " WorkClass Education MaritalStatus Occupation Relationship Race Sex \\\n",843 "0 7 9 4 1 1 4 1 \n",844 "1 6 9 2 4 0 4 1 \n",845 "2 4 11 0 6 1 4 1 \n",846 "3 4 1 2 6 0 2 1 \n",847 "4 4 9 2 10 5 2 0 \n",848 "\n",849 " Native Target \n",850 "0 39 0 \n",851 "1 39 0 \n",852 "2 39 0 \n",853 "3 39 0 \n",854 "4 5 0 "855 ]856 },857 "execution_count": 9,858 "metadata": {},859 "output_type": "execute_result"860 }861 ],862 "source": [863 "# apply label encoder to df_categorical\n",864 "le = preprocessing.LabelEncoder()\n",865 "df_categorical = df_categorical.apply(le.fit_transform)\n",866 "df_categorical.head()"867 ]868 },869 {870 "cell_type": "code",871 "execution_count": 10,872 "metadata": {},873 "outputs": [874 {875 "data": {876 "text/html": [877 "<div>\n",878 "<style scoped>\n",879 " .dataframe tbody tr th:only-of-type {\n",880 " vertical-align: middle;\n",881 " }\n",882 "\n",883 " .dataframe tbody tr th {\n",884 " vertical-align: top;\n",885 " }\n",886 "\n",887 " .dataframe thead th {\n",888 " text-align: right;\n",889 " }\n",890 "</style>\n",891 "<table border=\"1\" class=\"dataframe\">\n",892 " <thead>\n",893 " <tr style=\"text-align: right;\">\n",894 " <th></th>\n",895 " <th>Age</th>\n",896 " <th>Fnlwgt</th>\n",897 " <th>Edu_Num</th>\n",898 " <th>CapitalGain</th>\n",899 " <th>CapitalLoss</th>\n",900 " <th>HrPerWk</th>\n",901 " <th>WorkClass</th>\n",902 " <th>Education</th>\n",903 " <th>MaritalStatus</th>\n",904 " <th>Occupation</th>\n",905 " <th>Relationship</th>\n",906 " <th>Race</th>\n",907 " <th>Sex</th>\n",908 " <th>Native</th>\n",909 " <th>Target</th>\n",910 " </tr>\n",911 " </thead>\n",912 " <tbody>\n",913 " <tr>\n",914 " <th>0</th>\n",915 " <td>39</td>\n",916 " <td>77516</td>\n",917 " <td>13</td>\n",918 " <td>2174</td>\n",919 " <td>0</td>\n",920 " <td>40</td>\n",921 " <td>7</td>\n",922 " <td>9</td>\n",923 " <td>4</td>\n",924 " <td>1</td>\n",925 " <td>1</td>\n",926 " <td>4</td>\n",927 " <td>1</td>\n",928 " <td>39</td>\n",929 " <td>0</td>\n",930 " </tr>\n",931 " <tr>\n",932 " <th>1</th>\n",933 " <td>50</td>\n",934 " <td>83311</td>\n",935 " <td>13</td>\n",936 " <td>0</td>\n",937 " <td>0</td>\n",938 " <td>13</td>\n",939 " <td>6</td>\n",940 " <td>9</td>\n",941 " <td>2</td>\n",942 " <td>4</td>\n",943 " <td>0</td>\n",944 " <td>4</td>\n",945 " <td>1</td>\n",946 " <td>39</td>\n",947 " <td>0</td>\n",948 " </tr>\n",949 " <tr>\n",950 " <th>2</th>\n",951 " <td>38</td>\n",952 " <td>215646</td>\n",953 " <td>9</td>\n",954 " <td>0</td>\n",955 " <td>0</td>\n",956 " <td>40</td>\n",957 " <td>4</td>\n",958 " <td>11</td>\n",959 " <td>0</td>\n",960 " <td>6</td>\n",961 " <td>1</td>\n",962 " <td>4</td>\n",963 " <td>1</td>\n",964 " <td>39</td>\n",965 " <td>0</td>\n",966 " </tr>\n",967 " <tr>\n",968 " <th>3</th>\n",969 " <td>53</td>\n",970 " <td>234721</td>\n",971 " <td>7</td>\n",972 " <td>0</td>\n",973 " <td>0</td>\n",974 " <td>40</td>\n",975 " <td>4</td>\n",976 " <td>1</td>\n",977 " <td>2</td>\n",978 " <td>6</td>\n",979 " <td>0</td>\n",980 " <td>2</td>\n",981 " <td>1</td>\n",982 " <td>39</td>\n",983 " <td>0</td>\n",984 " </tr>\n",985 " <tr>\n",986 " <th>4</th>\n",987 " <td>28</td>\n",988 " <td>338409</td>\n",989 " <td>13</td>\n",990 " <td>0</td>\n",991 " <td>0</td>\n",992 " <td>40</td>\n",993 " <td>4</td>\n",994 " <td>9</td>\n",995 " <td>2</td>\n",996 " <td>10</td>\n",997 " <td>5</td>\n",998 " <td>2</td>\n",999 " <td>0</td>\n",1000 " <td>5</td>\n",1001 " <td>0</td>\n",1002 " </tr>\n",1003 " </tbody>\n",1004 "</table>\n",1005 "</div>"1006 ],1007 "text/plain": [1008 " Age Fnlwgt Edu_Num CapitalGain CapitalLoss HrPerWk WorkClass \\\n",1009 "0 39 77516 13 2174 0 40 7 \n",1010 "1 50 83311 13 0 0 13 6 \n",1011 "2 38 215646 9 0 0 40 4 \n",1012 "3 53 234721 7 0 0 40 4 \n",1013 "4 28 338409 13 0 0 40 4 \n",1014 "\n",1015 " Education MaritalStatus Occupation Relationship Race Sex Native \\\n",1016 "0 9 4 1 1 4 1 39 \n",1017 "1 9 2 4 0 4 1 39 \n",1018 "2 11 0 6 1 4 1 39 \n",1019 "3 1 2 6 0 2 1 39 \n",1020 "4 9 2 10 5 2 0 5 \n",1021 "\n",1022 " Target \n",1023 "0 0 \n",1024 "1 0 \n",1025 "2 0 \n",1026 "3 0 \n",1027 "4 0 "1028 ]1029 },1030 "execution_count": 10,1031 "metadata": {},1032 "output_type": "execute_result"1033 }1034 ],1035 "source": [1036 "# Next, Concatenate df_categorical dataframe with original df (dataframe)\n",1037 "\n",1038 "# first, Drop earlier duplicate columns which had categorical values\n",1039 "df = df.drop(df_categorical.columns,axis=1)\n",1040 "df = pd.concat([df,df_categorical],axis=1)\n",1041 "df.head()"1042 ]1043 },1044 {1045 "cell_type": "code",1046 "execution_count": 11,1047 "metadata": {},1048 "outputs": [1049 {1050 "data": {1051 "text/html": [1052 "<div>\n",1053 "<style scoped>\n",1054 " .dataframe tbody tr th:only-of-type {\n",1055 " vertical-align: middle;\n",1056 " }\n",1057 "\n",1058 " .dataframe tbody tr th {\n",1059 " vertical-align: top;\n",1060 " }\n",1061 "\n",1062 " .dataframe thead th {\n",1063 " text-align: right;\n",1064 " }\n",1065 "</style>\n",1066 "<table border=\"1\" class=\"dataframe\">\n",1067 " <thead>\n",1068 " <tr style=\"text-align: right;\">\n",1069 " <th></th>\n",1070 " <th>Age</th>\n",1071 " <th>Fnlwgt</th>\n",1072 " <th>Edu_Num</th>\n",1073 " <th>CapitalGain</th>\n",1074 " <th>CapitalLoss</th>\n",1075 " <th>HrPerWk</th>\n",1076 " <th>WorkClass</th>\n",1077 " <th>Education</th>\n",1078 " <th>MaritalStatus</th>\n",1079 " <th>Occupation</th>\n",1080 " <th>Relationship</th>\n",1081 " <th>Race</th>\n",1082 " <th>Sex</th>\n",1083 " <th>Native</th>\n",1084 " <th>Target</th>\n",1085 " </tr>\n",1086 " </thead>\n",1087 " <tbody>\n",1088 " <tr>\n",1089 " <th>Age</th>\n",1090 " <td>1.000000</td>\n",1091 " <td>-0.076646</td>\n",1092 " <td>0.036527</td>\n",1093 " <td>0.077674</td>\n",1094 " <td>0.057775</td>\n",1095 " <td>0.068756</td>\n",1096 " <td>0.003787</td>\n",1097 " <td>-0.010508</td>\n",1098 " <td>-0.266288</td>\n",1099 " <td>-0.020947</td>\n",1100 " <td>-0.263698</td>\n",1101 " <td>0.028718</td>\n",1102 " <td>0.088832</td>\n",1103 " <td>-0.001151</td>\n",1104 " <td>0.234037</td>\n",1105 " </tr>\n",1106 " <tr>\n",1107 " <th>Fnlwgt</th>\n",1108 " <td>-0.076646</td>\n",1109 " <td>1.000000</td>\n",1110 " <td>-0.043195</td>\n",1111 " <td>0.000432</td>\n",1112 " <td>-0.010252</td>\n",1113 " <td>-0.018768</td>\n",1114 " <td>-0.016656</td>\n",1115 " <td>-0.028145</td>\n",1116 " <td>0.028153</td>\n",1117 " <td>0.001597</td>\n",1118 " <td>0.008931</td>\n",1119 " <td>-0.021291</td>\n",1120 " <td>0.026858</td>\n",1121 " <td>-0.051966</td>\n",1122 " <td>-0.009463</td>\n",1123 " </tr>\n",1124 " <tr>\n",1125 " <th>Edu_Num</th>\n",1126 " <td>0.036527</td>\n",1127 " <td>-0.043195</td>\n",1128 " <td>1.000000</td>\n",1129 " <td>0.122630</td>\n",1130 " <td>0.079923</td>\n",1131 " <td>0.148123</td>\n",1132 " <td>0.052085</td>\n",1133 " <td>0.359153</td>\n",1134 " <td>-0.069304</td>\n",1135 " <td>0.109697</td>\n",1136 " <td>-0.094153</td>\n",1137 " <td>0.031838</td>\n",1138 " <td>0.012280</td>\n",1139 " <td>0.050840</td>\n",1140 " <td>0.335154</td>\n",1141 " </tr>\n",1142 " <tr>\n",1143 " <th>CapitalGain</th>\n",1144 " <td>0.077674</td>\n",1145 " <td>0.000432</td>\n",1146 " <td>0.122630</td>\n",1147 " <td>1.000000</td>\n",1148 " <td>-0.031615</td>\n",1149 " <td>0.078409</td>\n",1150 " <td>0.033835</td>\n",1151 " <td>0.030046</td>\n",1152 " <td>-0.043393</td>\n",1153 " <td>0.025505</td>\n",1154 " <td>-0.057919</td>\n",1155 " <td>0.011145</td>\n",1156 " <td>0.048480</td>\n",1157 " <td>-0.001982</td>\n",1158 " <td>0.223329</td>\n",1159 " </tr>\n",1160 " <tr>\n",1161 " <th>CapitalLoss</th>\n",1162 " <td>0.057775</td>\n",1163 " <td>-0.010252</td>\n",1164 " <td>0.079923</td>\n",1165 " <td>-0.031615</td>\n",1166 " <td>1.000000</td>\n",1167 " <td>0.054256</td>\n",1168 " <td>0.012216</td>\n",1169 " <td>0.016746</td>\n",1170 " <td>-0.034187</td>\n",1171 " <td>0.017987</td>\n",1172 " <td>-0.061062</td>\n",1173 " <td>0.018899</td>\n",1174 " <td>0.045567</td>\n",1175 " <td>0.000419</td>\n",1176 " <td>0.150526</td>\n",1177 " </tr>\n",1178 " <tr>\n",1179 " <th>HrPerWk</th>\n",1180 " <td>0.068756</td>\n",1181 " <td>-0.018768</td>\n",1182 " <td>0.148123</td>\n",1183 " <td>0.078409</td>\n",1184 " <td>0.054256</td>\n",1185 " <td>1.000000</td>\n",1186 " <td>0.138962</td>\n",1187 " <td>0.055510</td>\n",1188 " <td>-0.190519</td>\n",1189 " <td>0.080383</td>\n",1190 " <td>-0.248974</td>\n",1191 " <td>0.041910</td>\n",1192 " <td>0.229309</td>\n",1193 " <td>-0.002671</td>\n",1194 " <td>0.229689</td>\n",1195 " </tr>\n",1196 " <tr>\n",1197 " <th>WorkClass</th>\n",1198 " <td>0.003787</td>\n",1199 " <td>-0.016656</td>\n",1200 " <td>0.052085</td>\n",