HouBioLab/Demo7_RegressionGradientDescent

1### ML Demo: regression models and gradient descent2 3import gradio as gr4import matplotlib5import matplotlib.pyplot as plt6import numpy as np7from sklearn.linear_model import LinearRegression8 9def cal_mse(X,y,b,w):10  thetas = np.array([[b], [w]])11  X_b = np.c_[np.ones((len(X), 1)), X]  # add x0 = 1 to each instance12  y_predict = X_b.dot(thetas)13  mse = np.mean((y_predict-y)**2)14  return mse15 16def gradient_descent(intercept=4, slope=3, intercept_random=4, slope_random=3, gradient_descent=False, learning_rate= 0.01, iteration=100):17  ### (1) generate simulated data points18  X = 2 * np.random.rand(100, 1)19  y = intercept + slope * X + np.random.randn(100, 1)20 21  ### (2) fit regression model22  lin_reg = LinearRegression()23  lin_reg.fit(X, y)24 25  ### (3) make a prediction on training data26  y_predict = lin_reg.predict(X)27  y_predict28 29  ### (4) Draw baseline linear Line30  fig = plt.figure(figsize=(12,20))31 32  plt.subplot(3,1,1)33  plt.plot(X, y_predict, "r-", linewidth=2, label = "Line of best fit")34  plt.plot(X, y, "b.")35 36 37  ### (4.2) Draw random line38  if intercept_random != intercept or slope_random != slope: #avoid overlap39    X_new = np.array([[0], [2]])40    X_new_b = np.c_[np.ones((2, 1)), X_new]  # add x0 = 1 to each instance41    y_predict = X_new_b.dot(np.array([intercept_random, slope_random]))42    plt.plot(X_new, y_predict, "g-", linewidth=2, label = "Random line")43    44    45    ### (4.3) Apply gradient desc46    if gradient_descent:47      b = intercept_random48      w = slope_random49 50      lr = learning_rate # learning rate51      iteration = iteration52 53      # Store initial values for plotting.54      b_history = [b]55      w_history = [w]56 57      train_mse = []58      # Iterations59      for i in range(iteration):60          b_grad = 0.061          w_grad = 0.062          for n in range(len(X)):        63              b_grad = b_grad  - (y[n,0] - b - w*X[n,0])*1.064              w_grad = w_grad  - (y[n,0] - b - w*X[n,0])*X[n,0]65          b_grad /= len(X)66          w_grad /= len(X)67          68          # Update parameters.69          b = b - lr * b_grad 70          w = w - lr * w_grad71          72          # Store parameters for plotting73          b_history.append(b)74          w_history.append(w)75          76          train_mse.append(cal_mse(X,y,b,w))77 78          if i == int(iteration/4):79            X_tmp = np.array([[0], [2]])80            X_tmp_b = np.c_[np.ones((2, 1)), X_tmp]  # add x0 = 1 to each instance81            y_predict_tmp = X_tmp_b.dot(np.array([b, w]))82            plt.plot(X_tmp, y_predict_tmp, "brown", linewidth=2, label = "Fitted line in iteration "+str(i))83              84          if i == int(iteration/3):85            X_tmp = np.array([[0], [2]])86            X_tmp_b = np.c_[np.ones((2, 1)), X_tmp]  # add x0 = 1 to each instance87            y_predict_tmp = X_tmp_b.dot(np.array([b, w]))88            plt.plot(X_tmp, y_predict_tmp, "blue", linewidth=2, label = "Fitted line in iteration "+str(i))89              90          if i == int(iteration/2):91            X_tmp = np.array([[0], [2]])92            X_tmp_b = np.c_[np.ones((2, 1)), X_tmp]  # add x0 = 1 to each instance93            y_predict_tmp = X_tmp_b.dot(np.array([b, w]))94            plt.plot(X_tmp, y_predict_tmp, "gray", linewidth=2, label = "Fitted line in iteration "+str(i))95              96          if i == int(iteration-1):97            X_tmp = np.array([[0], [2]])98            X_tmp_b = np.c_[np.ones((2, 1)), X_tmp]  # add x0 = 1 to each instance99            y_predict_tmp = X_tmp_b.dot(np.array([b, w]))100            plt.plot(X_tmp, y_predict_tmp, "black", linewidth=2, label = "Fitted line in iteration "+str(i))101              102  plt.xlabel("$x_1$", fontsize=22)103  plt.ylabel("$y$", rotation=0, fontsize=22)104  plt.xticks(fontsize=18)105  plt.yticks(fontsize=18)106  plt.axis([np.min(X)*0.1, np.max(X)*1.1, np.min(y)*0.1, np.max(y)*1.1])107  plt.title("Linear Regression model predictions", fontsize=22)108  plt.legend(fontsize=18)109 110 111 112 113 114  ### (5) Visualize loss function115  plt.subplot(3,1,2)116 117  ### (5.1) generate grid of parameters118  b = np.arange(-10,10,0.1) #bias119  w = np.arange(-10,10,0.1) #weight120 121  ### (5.2) Calculate MSE over parameters122  Z =  np.zeros((len(w), len(b)))123 124  for i in range(len(w)):125      for j in range(len(b)):126        w0 = w[i]127        b0 = b[j]128        Z[i][j] = cal_mse(X, y, b0, w0)129 130 131  ### (5.3) Get optimal parameters132  theta0_best = lin_reg.intercept_[0]133  theta1_best = lin_reg.coef_[0][0]134 135 136  ### (5.4) Draw the contour graph 137  plt.contourf(b,w,Z, 50, alpha=0.5, cmap=plt.get_cmap('jet'))138 139  ### (5.5) Add optimal loss140  plt.plot(theta0_best, theta1_best, 'x', ms=12, markeredgewidth=3, color='orange')141  plt.text(theta0_best, theta1_best,'MSE:'+str(np.round(cal_mse(X,y,theta0_best, theta1_best),2)), color='red', fontsize=22)142  143  ### (5.6) Add loss of random lines144  if intercept_random != intercept or slope_random != slope: #avoid overlap145    plt.plot(intercept_random, slope_random, 'o', ms=5, markeredgewidth=3, color='orange')146    plt.text(intercept_random, slope_random,'MSE:'+str(np.round(cal_mse(X,y,intercept_random, slope_random),2)), fontsize=22)147    148    ### (5.7) draw gradient updates149    if gradient_descent:150      plt.plot(b_history, w_history, 'o-', ms=3, lw=1.5, color='black')151 152 153 154  plt.title("Visualization of Gradient Descent Process", fontsize=22)155  plt.xlabel("$Intercept$", fontsize=22)156  plt.ylabel("$Slope$", rotation=0, fontsize=22)157  plt.xticks(fontsize=18)158  plt.yticks(fontsize=18)159  plt.xlim(-10,10)160  plt.ylim(-10,10)161 162 163  ### 6. Visualize the learning curves 164  if gradient_descent:165      plt.subplot(3,1,3)166      plt.plot(train_mse,label="train_loss (lr="+str(learning_rate)+")")167      plt.xlabel('Iteration',fontweight="bold",fontsize = 22)168      plt.ylabel('Loss',fontweight="bold",fontsize = 22)169      plt.title("Learning curve: Loss VS Epochs",fontweight="bold",fontsize = 22)170      plt.legend(fontsize=18)171      plt.xticks(fontsize=18)172      plt.yticks(fontsize=18)173 174  #plt.show()175  fig.tight_layout()176  plt.savefig('plot_line.png', dpi=300)177  return 'plot_line.png'178 179 180 181 182 183 184#### Define input component185input_intercept = gr.Slider(1, 8, step=0.5, value=1, label='(Baseline) Intercept')186input_slope = gr.Slider(-8, 8, step=0.5, value=2, label='(Baseline) Slope')187 188input_intercept_random = gr.Slider(-8, 8, step=0.5, value=3, label='(Random) Intercept')189input_slope_random = gr.Slider(-8, 8, step=0.5, value=-1, label='(Random) Slope')190 191input_gradients = gr.Checkbox(label="Apply Gradient Descent")192 193input_learningrate = gr.Slider(0,1, step=0.001, value=0.01, label='Learning Rate')194input_iteration = gr.Slider(1, 1000, step=5, value=800, label='Iteration')195 196 197#### Define output component198output_plot1 = gr.Image(label="Regression plot")199 200 201### configure gradio, detailed can be found at https://www.gradio.app/docs/#i_slider202interface = gr.Interface(fn=gradient_descent, 203                         inputs=[input_intercept, input_slope, input_intercept_random, input_slope_random, input_gradients, input_learningrate, input_iteration], 204                         outputs=[output_plot1],205                         examples_per_page = 2,206                         examples = [[4, 3, -7, -5, True, 0.0001, 100], [1, 2, -7, -8, False, 0.0001, 100]], 207                         title="ML Demo: Regression models \n (Function approximation by Gradient Descent)", 208                         description= "Click examples to generate random dataset and select gradient descent parameters",209                         theme = 'huggingface',210                         #layout = 'vertical'211                         )212                         213interface.launch(debug=True) 214