mdreyer5/Week1-Lab

1# -*- coding: utf-8 -*-2"""Lab2222.ipynb3 4Automatically generated by Colaboratory.5 6Original file is located at7    https://colab.research.google.com/drive/1OUGOeTdmMbccW_st3Ao8nHDR5wm_VUNg8"""9 10from google.colab import drive11 12drive.mount("/content/ML_Course")13 14cd /content/ML_Course/MyDrive/ML_Course15 16import pandas as pd17housing = pd.read_csv("housing.csv")18housing.head(n = 5)19 20housing.columns21 22housing.describe()23 24housing.info()25 26# Commented out IPython magic to ensure Python compatibility.27# %matplotlib inline28import matplotlib.pyplot as plt29housing.hist(bins=50, figsize=(20,15))30plt.show()31 32# to make this notebook's output identical at every run33import numpy as np34np.random.seed(10)35 36# For illustration only. Sklearn has train_test_split()37def split_train_test(data, test_ratio):38    shuffled_indices = np.random.permutation(len(data))39    test_set_size = int(len(data) * test_ratio)40    test_indices = shuffled_indices[:test_set_size]41    train_indices = shuffled_indices[test_set_size:]42    return data.iloc[train_indices], data.iloc[test_indices]43 44# run the function to get the train & test set45train_set, test_set = split_train_test(housing, 0.2)46 47train_set.info()48 49test_set.info()50 51from sklearn.model_selection import train_test_split52train_set, test_set = train_test_split(housing, test_size=0.2, random_state=10)53 54train_set.info()55 56test_set.info()57 58test_set.to_csv('blind_test.csv', index = False)59 60train_set.plot(kind="scatter", x="longitude", y="latitude", alpha=0.4, 61    s=train_set["population"]/100, label="population", figsize=(10,7),62    c="median_house_value", cmap=plt.get_cmap("jet"), colorbar=True,63    sharex=False)64plt.legend()65plt.show()66 67train_set.info()68 69train_set[train_set.isna().any(axis=1)]70 71train_set_clean = train_set.dropna(subset=["total_bedrooms"])72train_set_clean73 74train_set_clean.info()75 76train_labels = train_set_clean["median_house_value"].copy() # get labels for output label Y77train_features = train_set_clean.drop("median_house_value", axis=1) # drop labels to get features X for training set78train_features.info()79 80train_features.head()81 82train_features.columns83 84train_features.info()85 86train_features.describe()87 88train_labels89 90train_features.hist(bins=50, figsize=(12,9))91 92train_features.describe()93 94from sklearn.preprocessing import MinMaxScaler95scaler = MinMaxScaler() ## define the transformer96scaler.fit(train_features) ## call .fit() method to calculate the min and max value for each column in dataset97 98print("Min of each column: ",scaler.data_min_)99print("Max of each column: ",scaler.data_max_)100 101train_features.describe()102 103train_features_normalized = scaler.transform(train_features)104train_features_normalized105 106pd.DataFrame(train_features_normalized).hist(bins=50, figsize=(12,9))107plt.show()108 109## 1. split data to get train and test set110from sklearn.model_selection import train_test_split111train_set, test_set = train_test_split(housing, test_size=0.2, random_state=10)112 113## 2. clean the missing values114train_set_clean = train_set.dropna(subset=["total_bedrooms"])115train_set_clean116 117## 2. derive training features and training labels 118train_labels = train_set_clean["median_house_value"].copy() # get labels for output label Y119train_features = train_set_clean.drop("median_house_value", axis=1) # drop labels to get features X for training set120 121 122## 4. scale the numeric features in training set123from sklearn.preprocessing import MinMaxScaler124scaler = MinMaxScaler() ## define the transformer125scaler.fit(train_features) ## call .fit() method to calculate the min and max value for each column in dataset126 127train_features_normalized = scaler.transform(train_features)128train_features_normalized129 130from sklearn.linear_model import LinearRegression ## import the LinearRegression Function131lin_reg = LinearRegression() ## Initialize the class132lin_reg.fit(train_features_normalized, train_labels) # feed the training data X, and label Y for supervised learning133# feed the training data X, and label Y for supervised learning134 135training_predictions = lin_reg.predict(train_features_normalized)136training_predictions.shape137 138train_labels139 140## plot scatter plot 141import matplotlib.pyplot as plt142plt.scatter(training_predictions, train_labels )143plt.xlabel('training_predictions', fontsize=15,color="red")144plt.ylabel('train_label', fontsize=15,color="green")145plt.title('Scatter plot for training_predictions and train_label', fontsize=15)146plt.xlim(0,np.max(training_predictions)) # remove the predictions that have negative prices147plt.show()148 149import numpy as np150np.corrcoef(training_predictions, train_labels)151 152import pandas as pd 153prediction_summary = pd.DataFrame({'predicted_label':training_predictions, 'actual_label':train_labels})154prediction_summary155 156prediction_summary['error'] = prediction_summary['actual_label'] - prediction_summary['predicted_label']157prediction_summary158 159from sklearn.metrics import mean_squared_error160lin_mse = mean_squared_error(train_labels, training_predictions)161lin_rmse = np.sqrt(lin_mse)162lin_rmse163 164## Step 1: training the data using decision tree algorithm165from sklearn.tree import DecisionTreeRegressor ## import the DecisionTree Function166tree_reg = DecisionTreeRegressor(random_state=10) ## Initialize the class167tree_reg.fit(train_features_normalized, train_labels) # feed the training data X, and label Y for supervised learning168 169### Step 2: make a prediction using tree model170training_predictions_trees = tree_reg.predict(train_features_normalized)171training_predictions_trees172 173## Step 3: visualize the scatter plot between predictions and actual labels174import matplotlib.pyplot as plt175plt.scatter(training_predictions_trees, train_labels )176plt.xlabel('training_predictions_trees', fontsize=15,color="red")177plt.ylabel('train_label', fontsize=15,color="green")178plt.title('Scatter plot for training_predictions_trees and train_label', fontsize=15)179plt.xlim(0,np.max(training_predictions_trees)) # remove the predictions that have negative prices180plt.show()181 182from sklearn.metrics import mean_squared_error183tree_mse = mean_squared_error(train_labels, training_predictions_trees)184tree_rmse = np.sqrt(tree_mse)185tree_rmse186 187## 1. clean the missing values in test set188test_set_clean = test_set.dropna(subset=["total_bedrooms"])189test_set_clean190 191## 2. derive test features and test labels. In this case, test labels are only used for evaluation192test_labels = test_set_clean["median_house_value"].copy() # get labels for output label Y193test_features = test_set_clean.drop("median_house_value", axis=1) # drop labels to get features X for training set194 195 196## 4. scale the numeric features in test set. 197## important note: do not apply fit function on the test set, using same scalar from training set198test_features_normalized = scaler.transform(test_features)199test_features_normalized200 201### Step 5: make a prediction using tree model202test_predictions_trees = tree_reg.predict(test_features_normalized)203test_predictions_trees204 205from sklearn.metrics import mean_squared_error206test_tree_mse = mean_squared_error(test_labels, test_predictions_trees)207test_tree_rmse = np.sqrt(test_tree_mse)208test_tree_rmse209 210# Step 1: install Gradio211!pip install --quiet gradio212 213# Step 2: import library214import gradio as gr215print(gr.__version__)216 217# Step 3.1: Define a simple "Hello World" function218# requirement: input is text, output is text219def greet(name):220      return "Hello " + name + "!!"221 222# Step 3.2: Define the input component (text style) and output component (text style) to create a simple GUI223import gradio as gr224input_module = gr.inputs.Textbox(label = "Input Text")225output_module = gr.outputs.Textbox(label = "Output Text")226 227# Step 3.3: Put all three component together into the gradio's interface function 228gr.Interface(fn=greet, inputs=input_module, outputs=output_module).launch()229 230# Step 5.1: Define a simple "image-to-text" function231# requirement: input is text, output is text232 233def caption(image):234    return "Image is processed!!"235 236# Step 5.2: Define the input component (image style) and output component (text style) to create a simple GUI237import gradio as gr238input_module = gr.inputs.Image(label = "Input Image")239 240output_module = gr.outputs.Textbox(label = "Output Text")241 242# Step 5.3: Put all three component together into the gradio's interface function 243gr.Interface(fn=caption, inputs=input_module, outputs=output_module).launch()244 245# Step 6.1: Define different input components246import gradio as gr247 248# a. define text data type249input_module1 = gr.inputs.Textbox(label = "Input Text")250 251# b. define image data type252input_module2 = gr.inputs.Image(label = "Input Image")253 254# c. define Number data type255input_module3 = gr.inputs.Number(label = "Input Number")256 257# d. define Slider data type258input_module4 = gr.inputs.Slider(1, 100, step=5, label = "Input Slider")259 260# e. define Checkbox data type261input_module5 = gr.inputs.Checkbox(label = "Does it work?")262 263# f. define Radio data type264input_module6 = gr.inputs.Radio(choices=["park", "zoo", "road"], label = "Input Radio")265 266# g. define Dropdown data type267input_module7 = gr.inputs.Dropdown(choices=["park", "zoo", "road"], label = "Input Dropdown")268 269# Step 6.2: Define different output components270# a. define text data type271output_module1 = gr.outputs.Textbox(label = "Output Text")272 273# b. define image data type274output_module2 = gr.outputs.Image(label = "Output Image")275 276# you can define more output components277 278# Step 6.3: Define a new function that accommodates the input modules.279def multi_inputs(input1, input2, input3, input4, input5, input6, input7 ):280    import numpy as np281    ## processing inputs282 283    ## return outputs284    output1 = "Processing inputs and return outputs" # text output example285    output2 = np.random.rand(6,6) # image-like array output example286    return output1,output2287 288# Step 6.4: Put all three component together into the gradio's interface function 289gr.Interface(fn=multi_inputs, 290             inputs=[input_module1, input_module2, input_module3,291                     input_module4, input_module5, input_module6,292                     input_module7], 293             outputs=[output_module1, output_module2]294            ).launch()295 296# Step 6.1: Define different input components297import gradio as gr298 299# a. define text data type300input_module1 = gr.inputs.Slider(-124.35,-114.35, step =0.5,label = "Longitude")301 302# b. define image data type303input_module2 = gr.inputs.Slider(32,41, step =0.5,label = "Latitude")304 305# c. define Number data type306input_module3 = gr.inputs.Slider(1,52, step = 1,label = "Housing_median_age(Year)")307 308# d. define Slider data type309input_module4 = gr.inputs.Slider(1, 40000, step=1, label = "Total_rooms")310 311# e. define Checkbox data type312input_module5 = gr.inputs.Slider(1, 6441,label = "Total_bedrooms")313 314# f. define Radio data type315input_module6 = gr.inputs.Slider(1,6441,step = 1,label = "Population")316 317# g. define Dropdown data type318input_module7 = gr.inputs.Slider(1,6081,step = 1,label = "Households")319 320input_module8 = gr.inputs.Slider(0,15,step = 1,label = "Median_income")321 322# Step 6.2: Define different output components323# a. define text data type324output_module1 = gr.outputs.Textbox(label = "Predicted Housing Prices")325 326# b. define image data type327output_module2 = gr.outputs.Image(label = "Output Image")328 329# you can define more output components330 331train_set.columns332 333#save machinel earning model to local drive334import pickle335#save 336with open('tree_reg.pkl','wb') as f:337  pickle.dump(tree_reg,f)338 339ls340 341# Step 6.3: Define a new function that accommodates the input modules.342def machine_learning_model(input1, input2, input3, input4, input5, input6, input7, input8):343    print('Start ML process')344    import numpy as np345    import pandas as pd346    print(input1, input2, input3, input4, input5, input6, input7, input8)347    #1. process the user submission348    new_feature = np.array([[input1, input2, input3, input4, input5, input6, input7, input8]])349    print(new_feature)350    351    test_set = pd.DataFrame(new_feature, columns = ['longitude', 'latitude', 'housing_median_age', 'total_rooms',352       'total_bedrooms', 'population', 'households', 'median_income'])353    354    ## 1. clean the missing values in test set355    test_set_clean = test_set.dropna(subset=["total_bedrooms"])356    test_set_clean357 358    ## 2. derive test features and test labels. In this case, test labels are only used for evaluation359    #test_labels = test_set_clean["median_house_value"].copy() # get labels for output label Y360    #test_features = test_set_clean.drop("median_house_value", axis=1) # drop labels to get features X for training set361 362    test_features_normalized = scaler.transform(test_set_clean)363    print("test_features_normalized: ", test_features_normalized)364    365    with open('tree_reg.pkl','rb') as f:366      tree_reg = pickle.load(f)367    print("Start processing")368 369    output1 = 'This is the output'370    output2 = np.random.rand(28,28)371 372    #2. follow the data preprocessing steps as we have done in the test data373    #2.2 Check missing values in total_bedrroms374    # 2.2 feature normalization375 376    #3. load pre trained machine learning377 378    379    #4 apply loaded modeld380    test_predictions_trees = tree_reg.predict(test_features_normalized)381    print("Predicition is :",test_predictions_trees)382    383    import matplotlib.pyplot as plt384 385    train_set.plot(kind="scatter", x="longitude", y="latitude", alpha=0.4, 386    s=train_set["population"]/100, label="population", figsize=(10,7),387    c="median_house_value", cmap=plt.get_cmap("jet"), colorbar=True,388    sharex=False)389    plt.legend()390    391    #plt.show()392    plt.xlim(-124.35,-114.35)393    plt.ylim(32,41)394    plt.plot([input1],[input2],marker = "X",markersize = 20, markeredgecolor="yellow", markerfacecolor="black")395    plt.savefig('test.png')396    #5 send back the prediciton397    return test_predictions_trees,'test.png'398 399gr.Interface(fn=machine_learning_model, 400             inputs=[input_module1, input_module2, input_module3,401                     input_module4, input_module5, input_module6,402                     input_module7, input_module8], 403             outputs=[output_module1, output_module2]404            ).launch(debug = True)