HouBioLab/Demo8_RegressionGradientDecentCompare

1### CSCI 4750/5750: regression models2### SLU-CS: Jie Hou3 4import gradio as gr5import matplotlib6import matplotlib.pyplot as plt7import numpy as np8from sklearn.linear_model import LinearRegression9 10def cal_mse(X,y,b,w):11  thetas = np.array([[b], [w]])12  X_b = np.c_[np.ones((len(X), 1)), X]  # add x0 = 1 to each instance13  y_predict = X_b.dot(thetas)14  mse = np.mean((y_predict-y)**2)15  return mse16 17def gradient_descent(n_samples=100, intercept=4, slope=3, intercept_random=4, slope_random=3, gradient_descent='False', gradient_descent_type = 'Batch GradientDescent' , learning_rate= 0.01, iteration=100, mini_batchsize = 32):18  if n_samples < mini_batchsize:19      mini_batchsize = n_samples20  ### (1) generate simulated data points21  X = 2 * np.random.rand(n_samples, 1)22  y = intercept + slope * X + np.random.randn(n_samples, 1)23 24  ### (2) fit regression model25  lin_reg = LinearRegression()26  lin_reg.fit(X, y)27 28  ### (3) make a prediction on training data29  y_predict = lin_reg.predict(X)30  y_predict31 32  ### (4) Draw baseline linear Line33  fig = plt.figure(figsize=(12,18))34 35  plt.subplot(3,1,1)36  plt.plot(X, y_predict, "r-", linewidth=2, label = "Line of best fit")37  plt.plot(X, y, "b.")38 39 40  ### (4.2) Draw random line41  if intercept_random != intercept or slope_random != slope: #avoid overlap42    X_new = np.array([[0], [2]])43    X_new_b = np.c_[np.ones((2, 1)), X_new]  # add x0 = 1 to each instance44    y_predict = X_new_b.dot(np.array([intercept_random, slope_random]))45    plt.plot(X_new, y_predict, "g-", linewidth=2, label = "Random line")46    47    48    ### (4.3) Apply gradient desc49    if gradient_descent:50      b = intercept_random51      w = slope_random52 53      lr = learning_rate # learning rate54      iteration = iteration55 56      if gradient_descent_type == 'Batch GradientDescent':57          # Store initial values for plotting.58          b_history = [b]59          w_history = [w]60 61          train_mse = []62          # Iterations63          for i in range(iteration):64              b_grad = 0.065              w_grad = 0.066              for n in range(len(X)):        67                  b_grad = b_grad  - 2*(y[n,0] - b - w*X[n,0])*1.068                  w_grad = w_grad  - 2*(y[n,0] - b - w*X[n,0])*X[n,0]69              b_grad /= len(X)70              w_grad /= len(X)71              72              # Update parameters.73              b = b - lr * b_grad 74              w = w - lr * w_grad75              76              # Store parameters for plotting77              b_history.append(b)78              w_history.append(w)79              80              train_mse.append(cal_mse(X,y,b,w))81      elif gradient_descent_type == 'Stochastic GradientDescent':82          # Store initial values for plotting.83          b_history = [b]84          w_history = [w]85 86          train_mse = []87          # Iterations88          for i in range(iteration):89              for n in range(len(X)):90                  random_index = np.random.randint(len(X))        91                  b_grad = -2*(y[random_index,0] - b - w*X[random_index,0])*1.092                  w_grad = -2*(y[random_index,0] - b - w*X[random_index,0])*X[random_index,0]93                  94                  # Update parameters.95                  b = b - lr * b_grad 96                  w = w - lr * w_grad97              98                  # Store parameters for plotting99                  b_history.append(b)100                  w_history.append(w)101              102                  train_mse.append(cal_mse(X,y,b,w))103      if gradient_descent_type == 'Mini-Batch GradientDescent':104          # Store initial values for plotting.105          b_history = [b]106          w_history = [w]107 108          train_mse = []109          # Iterations110          minibatch_size = mini_batchsize111          for i in range(iteration):112              # shuffle dataset113              shuffled_indices = np.random.permutation(len(X))114              X_b_shuffled = X[shuffled_indices]115              y_shuffled = y[shuffled_indices]116              for k in range(0, len(X), minibatch_size):117                  X_mini = X_b_shuffled[k:k+minibatch_size]118                  y_mini = y_shuffled[k:k+minibatch_size]119                         120                  b_grad = 0.0121                  w_grad = 0.0122                  for n in range(len(X_mini)):        123                      b_grad = b_grad  - 2*(y_mini[n,0] - b - w*X_mini[n,0])*1.0124                      w_grad = w_grad  - 2*(y_mini[n,0] - b - w*X_mini[n,0])*X_mini[n,0]125                  b_grad /= len(X_mini)126                  w_grad /= len(X_mini)127                  128                  # Update parameters.129                  b = b - lr * b_grad 130                  w = w - lr * w_grad131                  132                  # Store parameters for plotting133                  b_history.append(b)134                  w_history.append(w)135                  136                  train_mse.append(cal_mse(X,y,b,w))137  138  plt.xlabel("$x_1$", fontsize=22)139  plt.ylabel("$y$", rotation=0, fontsize=22)140  plt.xticks(fontsize=18)141  plt.yticks(fontsize=18)142  plt.axis([np.min(X)*0.1, np.max(X)*1.1, np.min(y)*0.1, np.max(y)*1.1])143  plt.title("Linear Regression model predictions", fontsize=22)144  plt.legend(fontsize=18)145  plt.xlim(0,2)146  plt.ylim(-10,10)147 148 149 150 151 152  ### (5) Visualize loss function153  plt.subplot(3,1,2)154 155  ### (5.1) generate grid of parameters156  b = np.arange(-10,10,0.1) #bias157  w = np.arange(-10,10,0.1) #weight158 159  ### (5.2) Calculate MSE over parameters160  Z =  np.zeros((len(w), len(b)))161 162  for i in range(len(w)):163      for j in range(len(b)):164        w0 = w[i]165        b0 = b[j]166        Z[i][j] = cal_mse(X, y, b0, w0)167 168 169  ### (5.3) Get optimal parameters170  theta0_best = lin_reg.intercept_[0]171  theta1_best = lin_reg.coef_[0][0]172 173 174  ### (5.4) Draw the contour graph 175  plt.contourf(b,w,Z, 50, alpha=0.5, cmap=plt.get_cmap('jet'))176 177  ### (5.5) Add optimal loss178  plt.plot(theta0_best, theta1_best, 'x', ms=12, markeredgewidth=3, color='orange')179  plt.text(theta0_best, theta1_best,'MSE:'+str(np.round(cal_mse(X,y,theta0_best, theta1_best),2)), color='red', fontsize=22)180  181 182  ### (5.6) Add loss of random lines183  if intercept_random != intercept or slope_random != slope: #avoid overlap184    plt.plot(intercept_random, slope_random, 'o', ms=5, markeredgewidth=3, color='orange')185    plt.text(intercept_random, slope_random,'MSE:'+str(np.round(cal_mse(X,y,intercept_random, slope_random),2)), fontsize=22)186    187    ### (5.7) draw gradient updates188    if gradient_descent:189      plt.plot(b_history, w_history, 'o-', ms=3, lw=1.5, color='black')190      plt.title("Visualization of Gradient Descent Process ("+gradient_descent_type+")", fontsize=22)191    else:192      plt.title("Visualization of Loss Function Map", fontsize=22)193  else:194    plt.title("Visualization of Loss Function Map", fontsize=22)195  plt.xlabel("$Intercept$", fontsize=22)196  plt.ylabel("$Slope$", rotation=0, fontsize=22)197  plt.xticks(fontsize=18)198  plt.yticks(fontsize=18)199  plt.xlim(-10,10)200  plt.ylim(-10,10)201 202 203  ### 6. Visualize the learning curves 204  if gradient_descent:205      plt.subplot(3,1,3)206      plt.plot(train_mse,label="train_loss (lr="+str(learning_rate)+")")207      plt.xlabel('Iteration',fontweight="bold",fontsize = 22)208      plt.ylabel('Loss',fontweight="bold",fontsize = 22)209      plt.title("Learning curve: Loss VS Epochs",fontweight="bold",fontsize = 22)210      plt.legend(fontsize=18)211      plt.xticks(fontsize=18)212      plt.yticks(fontsize=18)213 214  #plt.show()215  fig.tight_layout()216  plt.savefig('plot_line.png', dpi=300)217  return 'plot_line.png'218 219 220#### Define input component221input_sample = gr.Slider(1, 5000, step=50, value=100, label='N samples')222input_intercept = gr.Slider(1, 8, step=0.5, value=4, label='(Baseline) Intercept')223input_slope = gr.Slider(-8, 8, step=0.5, value=2.8, label='(Baseline) Slope')224 225input_intercept_random = gr.Slider(-8, 8, step=0.5, value=-7.5, label='(Random) Intercept')226input_slope_random = gr.Slider(-8, 8, step=0.5, value=7.5, label='(Random) Slope')227 228input_gradients = gr.Checkbox(label="Apply Gradient Descent")229#input_gradients_type = gr.inputs.CheckboxGroup(['Batch GradientDescient', 'Stochastic GradientDescent', 'Mini-Batch GradientDescent'],label="Type of Gradient Descent")230input_gradients_type = gr.Dropdown(['Batch GradientDescent', 'Stochastic GradientDescent', 'Mini-Batch GradientDescent'],label="Type of Gradient Descent")231 232 233input_batchsize = gr.Slider(1, 64, step=1, value=32, label='Batch size for Mini-BatchGD')234 235input_learningrate = gr.Slider(0,2, step=0.001, value=0.001, label='Learning Rate')236input_iteration = gr.Slider(1, 1000, step=2, value=100, label='Iteration')237 238 239#### Define output component240output_plot1 = gr.Image(label="Regression plot")241 242 243### configure gradio, detailed can be found at https://www.gradio.app/docs/#i_slider244interface = gr.Interface(fn=gradient_descent, 245                         inputs=[input_sample, input_intercept, input_slope, input_intercept_random, input_slope_random, input_gradients, input_gradients_type, input_learningrate, input_iteration, input_batchsize], 246                         outputs=[output_plot1],247                         examples_per_page = 2,248                         #examples = [[4, 3, -7, -5, True, 0.0001, 100], [1, 2, -7, -8, False, 0.0001, 100]], 249                         title="ML Demo: Regression models (Batch/Mini-Batch/Stochastic Gradient Descent)", 250                         description= "Click examples to generate random dataset and select gradient descent parameters",251                         theme = 'huggingface',252                         #layout = 'vertical'253                         )254                         255interface.launch(debug=True)