IoannisTr/Tech_Stocks_Trading_Assistant
31
1from asyncio.constants import LOG_THRESHOLD_FOR_CONNLOST_WRITES2import yfinance as yf3import pandas as pd4import numpy as np5import plotly.graph_objs as go6from stocks import *7from transformers import AutoModelForSequenceClassification, pipeline, AutoTokenizer8import os9from random import random10os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'11import tensorflow as tf12import math13import datetime14import random 15import time16#import kaleido 17from sklearn.preprocessing import MinMaxScaler 18import matplotlib.pyplot as plt19#import warnings20import tensorflow as tf21from tensorflow import keras22from keras.layers import Dropout, Activation 23from keras import layers24from keras.callbacks import EarlyStopping25from sklearn.metrics import r2_score26import plotly.graph_objs as go 27import plotly.io as pio28pio.templates29 30model = AutoModelForSequenceClassification.from_pretrained("fine_tuned_FinBERT", from_tf=False, config="config.json") 31tokenizer = AutoTokenizer.from_pretrained("fine_tuned_FinBERT/tokenizer/")32 33class Models(object):34 def __init__(self):35 self.stock_data = Stock_Data()36 37 def bollinger_bands_20d_2std(self, ticker):38 '''39 This method calculates the Bollinger Bands with a Rolling average of the last 20 days and 2 standard deviations. In a plot, 40 this would be represented as 3 lines: a rolling average, an upper bound (rolling average + 2 standard deviations) and a lower41 bound (rolling average - 2 standard deviations). When the price of a stock is between the rolling average and lower bound, it is42 considered as oversold, so it makes sense to buy, if it is between the roll. avg. and the upper bound, it is considered as 43 overbought, so it makes sense to sell, if it is equal to the roll.avg. it is neutral and if it is outside the bounds, it is 44 considered an Unusual Event. The function returns the outlook of the stock (either "Buy", or "Sell" or "Hold" or "Unusual Event")45 '''46 if self.stock_data.status_getter(ticker) != "Open":47 return "Market Closed"48 else:49 data = self.stock_data.stock_data_getter(ticker)50 low_high_closing_df = pd.DataFrame(data)51 low_high_closing_df = data.iloc[:, 4:5] # Getting only the "Adj Close" column52 low_high_closing_df = low_high_closing_df.tail(40) # Getting the last 40 days53 54 low_high_closing_df["rolling_avg_20d"] = low_high_closing_df['Adj Close'].rolling(20, min_periods = 20).mean()55 low_high_closing_df["sd"] = low_high_closing_df["Adj Close"].rolling(20, min_periods = 20).std()56 low_high_closing_df = low_high_closing_df.tail(20) # Keeping the last 20 days only57 58 recent_data = low_high_closing_df.iloc[-1, :].to_list() # Creating a Series object with the most recent data (last row only)59 60 upper_bound = recent_data[1] + 2*recent_data[2] # Upper Bound61 lower_bound = recent_data[1] - 2*recent_data[2] # Lower Bound62 mean_20d = recent_data[1] # Rolling average of last 20 days63 64 if self.stock_data.current_price_getter(ticker) is None:65 return "Market Closed"66 else:67 message = ""68 69 if self.stock_data.current_price_getter(ticker) < mean_20d and self.stock_data.current_price_getter(ticker) >= lower_bound:70 message = "Buy"71 elif self.stock_data.current_price_getter(ticker) > mean_20d and self.stock_data.current_price_getter(ticker) <= upper_bound:72 message = "Sell"73 elif self.stock_data.current_price_getter(ticker) == mean_20d:74 message = "Hold"75 elif self.stock_data.current_price_getter(ticker) <= lower_bound or self.stock_data.current_price_getter(ticker) >= upper_bound:76 message = "Unusual Event"77 return message78 79 def bollinger_bands_10d_1point5std(self, ticker):80 '''81 This method calculates the Bollinger Bands with a Rolling average of the last 10 days and 1.5 standard deviations. In a plot, 82 this would be represented as 3 lines: a rolling average, an upper bound (rolling average + 1.5 standard deviations) and a lower83 bound (rolling average - 1.5 standard deviations). When the price of a stock is between the rolling average and lower bound, it is84 considered as oversold, so it makes sense to buy, if it is between the roll. avg. and the upper bound, it is considered as 85 overbought, so it makes sense to sell, if it is equal to the roll.avg. it is neutral and if it is outside the bounds, it is 86 considered an Unusual Event. The function returns the outlook of the stock (either "Buy", or "Sell" or "Hold" or "Unusual Event")87 '''88 if self.stock_data.status_getter(ticker) != "Open":89 return "Market Closed"90 else:91 data = self.stock_data.stock_data_getter(ticker)92 93 low_high_closing_df = pd.DataFrame(data)94 low_high_closing_df = data.iloc[:, 4:5] # Getting only the "Adj Close" column95 low_high_closing_df = low_high_closing_df.tail(20) # Getting the last 20 days96 97 low_high_closing_df["rolling_avg_10d"] = low_high_closing_df['Adj Close'].rolling(10, min_periods = 10).mean()98 low_high_closing_df["sd"] = low_high_closing_df["Adj Close"].rolling(10, min_periods = 10).std()99 low_high_closing_df = low_high_closing_df.tail(10) # Keeping the last 10 days only100 101 recent_data = low_high_closing_df.iloc[-1, :].to_list() # Creating a Series object with the most recent data (last row only)102 103 upper_bound = recent_data[1] + 1.5*recent_data[2] # Upper Bound104 lower_bound = recent_data[1] - 1.5*recent_data[2] # Lower Bound105 mean_10d = recent_data[1] # Rolling average of last 10 days106 107 if self.stock_data.current_price_getter(ticker) is None:108 return "Market Closed"109 else:110 message = ""111 112 if self.stock_data.current_price_getter(ticker) < mean_10d and self.stock_data.current_price_getter(ticker) >= lower_bound:113 message = "Buy"114 elif self.stock_data.current_price_getter(ticker) > mean_10d and self.stock_data.current_price_getter(ticker) <= upper_bound:115 message = "Sell"116 elif self.stock_data.current_price_getter(ticker) == mean_10d:117 message = "Hold"118 elif self.stock_data.current_price_getter(ticker) <= lower_bound or self.stock_data.current_price_getter(ticker) >= upper_bound:119 message = "Unusual Event"120 return message121 122 def bollinger_bands_50d_3std(self, ticker):123 '''124 This method calculates the Bollinger Bands with a Rolling average of the last 50 days and 3 standard deviations. In a plot, 125 this would be represented as 3 lines: a rolling average, an upper bound (rolling average + 3 standard deviations) and a lower126 bound (rolling average - 3 standard deviations). When the price of a stock is between the rolling average and lower bound, it is127 considered as oversold, so it makes sense to buy, if it is between the roll. avg. and the upper bound, it is considered as 128 overbought, so it makes sense to sell, if it is equal to the roll.avg. it is neutral and if it is outside the bounds, it is 129 considered an Unusual Event. The function returns the outlook of the stock (either "Buy", or "Sell" or "Hold" or "Unusual Event")130 '''131 if self.stock_data.status_getter(ticker) != "Open":132 return "Market Closed"133 else:134 data = self.stock_data.stock_data_getter(ticker)135 136 low_high_closing_df = pd.DataFrame(data)137 low_high_closing_df = data.iloc[:, 4:5] # Getting only the "Adj Close" column138 low_high_closing_df = low_high_closing_df.tail(100) # Getting the last 100 days139 140 low_high_closing_df["rolling_avg_50d"] = low_high_closing_df['Adj Close'].rolling(50, min_periods = 50).mean()141 low_high_closing_df["sd"] = low_high_closing_df["Adj Close"].rolling(50, min_periods = 50).std()142 low_high_closing_df = low_high_closing_df.tail(50) # Keeping the last 50 days only143 144 recent_data = low_high_closing_df.iloc[-1, :].to_list() # Creating a Series object with the most recent data (last row only)145 146 upper_bound = recent_data[1] + 3*recent_data[2] # Upper Bound147 lower_bound = recent_data[1] - 3*recent_data[2] # Lower Bound148 mean_50d = recent_data[1] # Rolling average of last 50 days149 150 # Finding the outlook dependent on the current price151 if self.stock_data.current_price_getter(ticker) is None:152 return "Market Closed"153 else:154 message = ""155 if self.stock_data.current_price_getter(ticker) < mean_50d and self.stock_data.current_price_getter(ticker) >= lower_bound:156 message = "Buy"157 elif self.stock_data.current_price_getter(ticker) > mean_50d and self.stock_data.current_price_getter(ticker) <= upper_bound:158 message = "Sell"159 elif self.stock_data.current_price_getter(ticker) == mean_50d:160 message = "Hold"161 elif self.stock_data.current_price_getter(ticker) <= lower_bound or self.stock_data.current_price_getter(ticker) >= upper_bound:162 message = "Unusual Event"163 return message164 165 def MACD(self, ticker):166 '''167 This method calculates the MACD (Mean Average Convergence Divergence) for a stock. The decision of whether to buy or sell168 a stock when using this method, depends on the difference of two "lines". The 1st one is called "MACD" and is equal to the 169 difference between the Exponential Moving Average of the adjusted closing price of the last 12 days, and the Moving Average170 of the adjusted closing price of the last 26 days. The 2nd line is the 9 day moving average of the adj. closing price. 171 When MACD > 9 day M.A. it is considered that there is an uptrend, else, an downtrend. 172 At last, when MACD line crosses the 9 day M.A. from "above", a "Sell" signal is given, 173 while it crosses it from below, a "Buy" signal is given.174 '''175 if self.stock_data.status_getter(ticker) != "Open":176 return "Market Closed"177 else:178 data = self.stock_data.stock_data_getter(ticker)179 180 low_high_closing_df = pd.DataFrame(data)181 low_high_closing_df = data.iloc[:, 4:5] # Getting only the "Adj Close" column182 low_high_closing_df = low_high_closing_df.tail(52) # Getting the last 52 days183 184 185 # Get the 12-day EMA of the closing price186 low_high_closing_df['EMA_12d'] = low_high_closing_df['Adj Close'].ewm(span=12, adjust=False, min_periods=12).mean()187 # Get the 26-day MA of the closing price188 low_high_closing_df['MA_26d'] = low_high_closing_df['Adj Close'].ewm(span=26, adjust=False, min_periods=26).mean()189 # Subtract the 26-day EMA from the 12-Day EMA to get the MACD190 low_high_closing_df['MACD'] = low_high_closing_df['EMA_12d'] - low_high_closing_df['MA_26d']191 # Making the signal line192 low_high_closing_df['MA_9d'] = low_high_closing_df['MACD'].ewm(span=9, adjust=False, min_periods=9).mean()193 194 low_high_closing_df['Diff'] = low_high_closing_df['MACD'] - low_high_closing_df['MA_9d'] 195 196 Diff = low_high_closing_df['Diff'].astype(float)197 198 if self.stock_data.current_price_getter(ticker) is None:199 return "Market Closed"200 else:201 message = ""202 203 if Diff.iloc[-1] < 0:204 if Diff.iloc[-2] >= 0:205 message = "Downtrend and sell signal"206 else:207 message = "Downtrend and no signal"208 else:209 if Diff.iloc[-2] <= 0:210 message = "Uptrend and buy signal"211 else:212 message = "Uptrend and no signal"213 return message214 215 def finbert_headlines_sentiment(self, ticker):216 '''217 This method uses a the "weights" and the "tokenizer" of a fine-tuned Fin-BERT model, which is a BERT model that 218 was furtherly trained on financial data. The "article_parser()" method scraps www.marketwatch.com and returns the219 last 17 headers of the chosen stock's articles. The, the FinBERT model classifies each one of them as either "Positive"220 or "Negative" or "Neutral", and a score is assigned to each header (+100, -100, and 0) correspondingly. At last, a221 rolling average of window size = 5 is used to "smooth" the sentiment line of the "plotly" plot that is returned.222 '''223 224 articles_df = self.stock_data.article_parser(ticker)225 articles_list = articles_df["headline"].tolist()226 227 clf = pipeline("text-classification", model=model, tokenizer=tokenizer) 228 outputs_list = clf(articles_list)229 230 sentiments = []231 232 for item in outputs_list:233 sentiments.append(item["label"])234 235 sentiments_df = pd.DataFrame(sentiments)236 sentiments_df.rename(columns = {0:'sentiment'}, inplace = True)237 238 sentiments_df["sentiment"] = sentiments_df["sentiment"].apply(lambda x: 100 if x == "positive" else -100 if x=="negative" else 0) 239 sentiments_df["roll_avg"] = round(sentiments_df["sentiment"].rolling(5, min_periods = 1).mean(), 2)240 sentiments_df = sentiments_df.tail(12).reset_index()241 242 pd.options.plotting.backend = "plotly"243 244 fig = sentiments_df["roll_avg"].plot(title="Sentiment Analysis of the last 12 www.marketwatch.com articles about " + ticker, 245 246 template="plotly_dark",247 labels=dict(index="12 most recent article headlines", value="sentiment score (rolling avg. of window size 5)"))248 fig.update_traces(line=dict(color="#3D9140", width=3))249 fig.update_layout(yaxis_range=[-100,100])250 fig.update_layout(xaxis_range=[0,12])251 fig.update_layout(showlegend=False)252 fig.add_hline(y=0, line_width=1.5, line_color="black")253 254 current_sentiment = sentiments_df["roll_avg"].tail(1).values[0]255 256 return {'fig': fig, 'current_sentiment': current_sentiment}257 258 def LSTM_7_days_price_predictor(self, ticker):259 '''260 This method predicts the price of a chosen stock for the next 7 days as of today, by using the daily adjusted closing 261 prices for the last 2 years. At first, a 60-day window of historical prices (i-60) is created as our feature data (x_train)262 and the following 60-days window as label data (y_train). For every stock available, we have manually defined different 263 parameters so that they fit as good as it gets to the model. Finally we combute the R2 metric and make the predictions. At 264 last, we proceed with the predictions. The model looks back in our data (60 days back) and predicta for the following 7 days.265 '''266 267 stock_data = self.stock_data.LSTM_stock_data_getter(ticker)268 stock_data=pd.DataFrame(data=stock_data).drop(['Open','High','Low','Close', 'Volume'],axis=1).reset_index()269 stock_data['Date'] = pd.to_datetime(stock_data['Date'])270 stock_data=stock_data.dropna()271 272 # Data Preprocessing273 random.seed(1997)274 close_prices = stock_data['Adj Close']275 values = close_prices.values276 training_data_len = math.ceil(len(values)* 0.8)277 278 scaler = MinMaxScaler(feature_range=(0,1))279 scaled_data = scaler.fit_transform(values.reshape(-1,1))280 train_data = scaled_data[0: training_data_len, :]281 282 x_train = []283 y_train = []284 285 for i in range(60, len(train_data)):286 x_train.append(train_data[i-60:i, 0])287 y_train.append(train_data[i, 0])288 289 x_train, y_train = np.array(x_train), np.array(y_train)290 x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))291 292 # Preparation of test set293 test_data = scaled_data[training_data_len-60: , : ]294 x_test = []295 y_test = values[training_data_len:]296 297 for i in range(60, len(test_data)):298 x_test.append(test_data[i-60:i, 0])299 300 x_test = np.array(x_test)301 x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))302 303 ##### Setting Up LSTM Network Architecture and the Training of the LSTM Model304 def LSTM_trainer(seed, DROPOUT, LSTM_units,patience,batch_size, epochs):305 306 tf.random.set_seed(seed)307 DROPOUT = DROPOUT308 global model_lstm309 model_lstm = keras.Sequential()310 model_lstm.add(layers.LSTM(LSTM_units, return_sequences=True, input_shape=(x_train.shape[1], 1)))311 model_lstm.add(Dropout(rate=DROPOUT))312 model_lstm.add(layers.LSTM(LSTM_units, return_sequences=False))313 model_lstm.add(Dropout(rate=DROPOUT))314 model_lstm.add(layers.Dense(25))315 model_lstm.add(Dropout(rate=DROPOUT))316 model_lstm.add(layers.Dense(1))317 model_lstm.add(Activation('linear'))318 319 print('\n')320 print("Compiling the LSTM Model for the " + str(ticker) + " stock....\n")321 t0 = time.time()322 model_lstm.compile(optimizer='adam', loss='mean_squared_error',metrics=['mae'])323 callback=EarlyStopping(monitor='val_loss',324 min_delta=0,325 patience=patience,326 verbose=1, mode='auto')327 model_lstm.fit(x_train, 328 y_train,329 batch_size= batch_size, 330 epochs=epochs,331 validation_split=0.1,# ...holding out 10% of the data for validation 332 shuffle=True,verbose=0,callbacks=[callback])333 t1 = time.time()334 global ex_time335 ex_time = round(t1-t0, 2)336 print("Compiling took :",ex_time,"seconds")337 338 predictions = model_lstm.predict(x_test)339 predictions = scaler.inverse_transform(predictions)340 #rmse = np.sqrt(np.mean(((predictions - y_test) ** 2)))341 global r_squared_score 342 global rmse343 r_squared_score = round(r2_score(y_test, predictions),2)344 rmse = np.sqrt(np.mean(((predictions - y_test) ** 2)))345 #print('Rmse Score: ', round(rmse),2)346 print('R2 Score: ', r_squared_score)347 348 if ticker == 'AAPL':349 LSTM_trainer(1, 0.2, 100,2, 20, 30)350 elif ticker == 'NVDA':351 LSTM_trainer(2, 0.2, 100,2, 30, 50) 352 elif ticker == 'PYPL':353 LSTM_trainer(6, 0.2, 100,10,25, 30)354 elif ticker == 'MSFT':355 LSTM_trainer(4, 0.1, 80, 2,20, 40)356 elif ticker == 'TSLA':357 LSTM_trainer(5, 0.1, 120, 4,20, 25)358 elif ticker == 'AMZN':359 LSTM_trainer(6, 0.1, 120,2, 20, 25) 360 elif ticker == 'SPOT':361 LSTM_trainer(9, 0.2, 200,5, 20, 40)362 #elif ticker == 'TWTR' :363 # LSTM_trainer(15, 0.2, 100,4,20, 40)364 elif ticker == 'UBER':365 LSTM_trainer(15, 0.2, 100,7,20, 40)366 elif ticker == 'adanipower.ns':367 LSTM_trainer(15, 0.2, 120,8,20, 40)368 elif ticker == 'GOOG':369 LSTM_trainer(15, 0.2, 100,3,20, 25)370 371 # Unseen Predictions for the next 7 days 372 close_data = scaled_data373 look_back = 60374 375 def predict(num_prediction, model):376 prediction_list = close_data[-look_back:]377 378 for _ in range(num_prediction):379 x = prediction_list[-look_back:]380 x = x.reshape((1, look_back, 1))381 382 out = model.predict(x)[0][0]383 prediction_list = np.append(prediction_list, out)384 prediction_list = prediction_list[look_back-1:]385 386 return prediction_list387 388 def predict_dates(num_prediction):389 last_date = stock_data['Date'].values[-1]390 prediction_dates = pd.date_range(last_date, periods=num_prediction+1).tolist()391 return prediction_dates392 393 num_prediction = 7394 395 forecast = predict(num_prediction, model_lstm)396 forecast_dates = predict_dates(num_prediction)397 398 plt.figure(figsize=(25,10))399 forecast = forecast.reshape(-1, 1)400 forecast_inverse = scaler.inverse_transform(forecast)401 402 # Ploting the Actual Prices and the Predictions of them for the next 7 days403 base = stock_data['Date'].iloc[[-1]] # Here we create our base date (the last existing date with actual prices)404 testdata = pd.DataFrame(forecast_inverse)# Here we create a data frame that contains the prediction prices and an empty column for their dates405 testdata['Date'] = ""406 testdata.columns = ["Adj Close","Date"]407 testdata = testdata.iloc[1:,:]408 testdata["Label"] = "" # Let's add a column "Label" that would show if the respective price is a prediction or not409 testdata["Label"] = "Prediction"410 testdata = testdata[["Date", "Adj Close", "Label"]]411 412 date_list = [base + datetime.timedelta(days=x+1) for x in range(testdata.shape[0]+1)] 413 date_list = pd.DataFrame(date_list)414 date_list.columns = ["Date"]415 date_list.reset_index(inplace = True)416 date_list.drop(["index"], axis = 1, inplace = True)417 date_list.index = date_list.index + 1418 testdata.Date = date_list419 420 stock_data["Label"] = ""421 stock_data["Label"] = "Actual price"422 finaldf = pd.concat([stock_data,testdata], axis=0) # Here we concatenate the "testdata" and the original data frame "df" into a final one423 finaldf.reset_index(inplace = True)424 finaldf.drop(["index"], axis = 1, inplace = True)425 finaldf['Date'] = pd.to_datetime(finaldf['Date'])426 427 plt.rcParams["figure.figsize"] = (25,10)428 #We create two different data frames, one that contains the actual prices and one that has only the predictions429 finaldfPredictions = finaldf.iloc[-8:] 430 finaldfActuals = finaldf.iloc[:-7]431 432 plot_1 = go.Scatter(433 x = finaldfActuals['Date'],434 y = finaldfActuals['Adj Close'],435 mode = 'lines',436 name = 'Historical Data (2 years)',437 line=dict(width=1,color='#3D9140'))438 plot_2 = go.Scatter(439 x = finaldfPredictions['Date'],440 y = finaldfPredictions['Adj Close'],441 mode = 'lines',442 name = '7-day Prediction',443 line=dict(width=1,color="#EE3B3B"))444 plot_3 = go.Scatter(445 x = finaldfPredictions['Date'][:1],446 y = finaldfPredictions['Adj Close'][:1],447 mode = 'markers',448 name = 'Latest Actual Closing Price',449 line=dict(width=1))450 451 layout = go.Layout(452 title = 'Next 7 days stock price prediction of ' + str(ticker),453 xaxis = {'title' : "Date"},454 yaxis = {'title' : "Price ($)"}455 )456 fig = go.Figure(data=[plot_1, plot_2,plot_3], layout=layout)457 fig.update_layout(template='plotly_dark',autosize=True)458 fig.update_layout(legend=dict(459 orientation="h",460 yanchor="bottom",461 y=1.02,462 xanchor="right",463 x=1),464 annotations = [dict(x=0.5,465 y=0, 466 xref='paper',467 yref='paper',468 text="Current In Sample R- Squared : " + str(r_squared_score*100) + " % \n",469 showarrow = False)],470 xaxis=dict(showgrid=False),471 yaxis=dict(showgrid=False)472 473 474 )475 fig.add_annotation(x=0.5, 476 y=0.05,477 xref='paper',478 yref='paper',479 text="Current In Sample Root Mean Square Error : " + str(round(rmse,2)) + " % ",480 showarrow=False)481 482 return fig483