aipoc/ICE_AIStockFinaceTools
0
1# os.system("pip install langchain-openai")2from langchain_openai import AzureChatOpenAI3import os4import pdfplumber5from langchain.chains.mapreduce import MapReduceChain6from langchain.text_splitter import CharacterTextSplitter7from langchain.chains.summarize import load_summarize_chain8 9from langchain_community.document_loaders import UnstructuredFileLoader10from langchain.prompts import PromptTemplate11import logging12import json13from typing import List14import mimetypes15import validators16import requests17import tempfile18from langchain.chains import create_extraction_chain19from GoogleNews import GoogleNews20import pandas as pd21import requests22import gradio as gr23import re24from langchain_community.document_loaders import WebBaseLoader25from langchain.chains.combine_documents.stuff import StuffDocumentsChain26from transformers import pipeline27import plotly.express as px28from langchain_community.document_loaders import CSVLoader29from langchain_community.chat_models import ChatOpenAI30from langchain.chains.llm import LLMChain31import yfinance as yf32import pandas as pd33import nltk34from nltk.tokenize import sent_tokenize35from openai import AzureOpenAI36from langchain.prompts import PromptTemplate37from langchain.chains import load_summarize_chain38from langchain.chat_models import AzureChatOpenAI39 40 41 42 43class KeyValueExtractor:44 45 def __init__(self):46 47 """48 Initialize the ContractSummarizer object.49 50 Parameters:51 pdf_file_path (str): The path to the input PDF file.52 """53 self.model = "facebook/bart-large-mnli"54 self.client = AzureOpenAI(api_key=os.getenv("AZURE_OPENAI_KEY"), 55 api_version="2024-02-01",56 azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")57 )58 59 60 61 def get_url(self,keyword):62 return f"https://finance.yahoo.com/quote/{keyword}?p={keyword}"63 64 def get_each_link_summary(self,url):65 66 loader = WebBaseLoader(url)67 docs = loader.load()68 text_splitter = CharacterTextSplitter.from_tiktoken_encoder(69 chunk_size=3000, chunk_overlap=20070 )71 72 # Split the documents into chunks73 split_docs = text_splitter.split_documents(docs)74 75 # Prepare the prompt template for summarization76 prompt_template = """The give text is Finance Stock Details for one company i want to get values for77 Previous Close : [value]78 Open : [value]79 Bid : [value]80 Ask : [value]81 Day's Range : [value]82 52 Week Range : [value]83 Volume : [value]84 Avg. Volume : [value]85 Market Cap : [value]86 Beta (5Y Monthly) : [value]87 PE Ratio (TTM) : [value]88 EPS (TTM) : [value]89 Earnings Date : [value]90 Forward Dividend & Yield : [value]91 Ex-Dividend Date : [value]92 1y Target Est : [value]93 these details form that and Write a abractive summary about those details:94 Given Text: {text}95 CONCISE SUMMARY:"""96 prompt = PromptTemplate.from_template(prompt_template)97 98 # Prepare the template for refining the summary with additional context99 refine_template = (100 "Your job is to produce a final summary\n"101 "We have provided an existing summary up to a certain point: {existing_answer}\n"102 "We have the opportunity to refine the existing summary"103 "(only if needed) with some more context below.\n"104 "------------\n"105 "{text}\n"106 "------------\n"107 "Given the new context, refine the original summary"108 "If the context isn't useful, return the original summary."109 )110 refine_prompt = PromptTemplate.from_template(refine_template)111 112 # Load the summarization chain using the ChatOpenAI language model113 chain = load_summarize_chain(114 llm = AzureChatOpenAI(azure_deployment = "GPT-4o"),115 chain_type="refine",116 question_prompt=prompt,117 refine_prompt=refine_prompt,118 return_intermediate_steps=True,119 input_key="input_documents",120 output_key="output_text",121 )122 123 # Generate the refined summary using the loaded summarization chain124 result = chain({"input_documents": split_docs}, return_only_outputs=True)125 print(result["output_text"])126 127 return result["output_text"]128 129 def one_day_summary(self,content) -> None:130 131 conversation = [132 {"role": "system", "content": "You are a helpful assistant."},133 {"role": "user", "content": f"i want detailed Summary from given finance details. i want information like what happen today comparing last day good or bad Bullish or Bearish like these details i want summary. content in backticks.```{content}```."}134 ]135 136 # Call OpenAI GPT-3.5-turbo137 chat_completion = self.client.chat.completions.create(138 model = "GPT-4o",139 messages = conversation,140 max_tokens=1000,141 temperature=0142 )143 144 response = chat_completion.choices[0].message.content145 return response 146 147 # # Use OpenAI's Completion API to analyze the text and extract key-value pairs148 # response = openai.Completion.create(149 # engine="text-davinci-003", # You can choose a different engine as well150 # temperature = 0,151 # prompt=f"i want detailed Summary from given finance details. i want information like what happen today comparing last day good or bad Bullish or Bearish like these details i want summary. content in backticks.```{content}```.",152 # max_tokens=1000 # You can adjust the length of the response153 # )154 155 # # Extract and return the chatbot's reply156 # result = response['choices'][0]['text'].strip()157 # print(result)158 # return result159 160 def extract_key_value_pair(self,content) -> None:161 162 """163 Extract key-value pairs from the refined summary.164 165 Prints the extracted key-value pairs.166 """167 168 try:169 conversation = [170 {"role": "system", "content": "You are a helpful assistant."},171 {"role": "user", "content": f"Get maximum count meaningfull key value pairs. content in backticks.```{content}```."}172 ]173 174 # Call OpenAI GPT-3.5-turbo175 chat_completion = self.client.chat.completions.create(176 model = "GPT-4o",177 messages = conversation,178 max_tokens=1000,179 temperature=0180 )181 response = chat_completion.choices[0].message.content182 return response 183 184 except Exception as e:185 # If an error occurs during the key-value extraction process, log the error186 logging.error(f"Error while extracting key-value pairs: {e}")187 print("Error:", e)188 189 def analyze_sentiment_for_graph(self, text):190 191 pipe = pipeline("zero-shot-classification", model=self.model)192 labels=["Positive", "Negative", "Neutral"]193 result = pipe(text, labels)194 sentiment_scores = {195 result['labels'][0]: result['scores'][0],196 result['labels'][1]: result['scores'][1],197 result['labels'][2]: result['scores'][2]198 }199 return sentiment_scores200 201 def display_graph(self,text):202 203 sentiment_scores = self.analyze_sentiment_for_graph(text)204 labels = sentiment_scores.keys()205 scores = sentiment_scores.values()206 fig = px.bar(x=scores, y=labels, orientation='h', color=labels, color_discrete_map={"Negative": "red", "Positive": "green", "Neutral": "gray"})207 fig.update_traces(texttemplate='%{x:.2f}%', textposition='outside')208 fig.update_layout(title="Sentiment Analysis",width=800)209 210 formatted_pairs = []211 for key, value in sentiment_scores.items():212 formatted_value = round(value, 2) # Round the value to two decimal places213 formatted_pairs.append(f"{key} : {formatted_value}")214 215 result_string = '\t'.join(formatted_pairs)216 217 return fig218 219 def get_finance_data(self,symbol):220 221 # Define the stock symbol and date range222 start_date = '2022-08-19'223 end_date = '2023-08-19'224 225 # Fetch historical OHLC data using yfinance226 data = yf.download(symbol, start=start_date, end=end_date)227 228 # Select only the OHLC columns229 ohlc_data = data[['Open', 'High', 'Low', 'Close']]230 231 csv_path = "ohlc_data.csv"232 # Save the OHLC data to a CSV file233 ohlc_data.to_csv(csv_path)234 return csv_path235 236 def csv_to_dataframe(self,csv_path):237 238 # Replace 'your_file.csv' with the actual path to your CSV file239 csv_file_path = csv_path240 # Read the CSV file into a DataFrame241 df = pd.read_csv(csv_file_path)242 # Now you can work with the 'df' DataFrame243 return df # Display the first few rows of the DataFrame244 245 def save_dataframe_in_text_file(self,df):246 247 output_file_path = 'output.txt'248 249 # Convert the DataFrame to a text file250 df.to_csv(output_file_path, sep='\t', index=False)251 252 return output_file_path253 254 def csv_loader(self,output_file_path):255 256 loader = UnstructuredFileLoader(output_file_path, strategy="fast")257 docs = loader.load()258 259 return docs260 261 def document_text_spilliter(self,docs):262 263 """264 Split documents into chunks for efficient processing.265 266 Returns:267 List[str]: List of split document chunks.268 """269 270 # Initialize the text splitter with specified chunk size and overlap271 text_splitter = CharacterTextSplitter.from_tiktoken_encoder(272 chunk_size=1000, chunk_overlap=200273 )274 275 # Split the documents into chunks276 split_docs = text_splitter.split_documents(docs)277 278 # Return the list of split document chunks279 return split_docs280 281 def change_bullet_points(self,text):282 283 nltk.download('punkt') # Download the sentence tokenizer data (only need to run this once)284 285 # Example passage286 passage = text287 288 # Tokenize the passage into sentences289 sentences = sent_tokenize(passage)290 bullet_string = ""291 # Print the extracted sentences292 for sentence in sentences:293 bullet_string+="* "+sentence+"\n"294 295 return bullet_string296 297 def one_year_summary(self, keyword):298 try:299 # Step 1: Get the finance data and convert to DataFrame300 csv_path = self.get_finance_data(keyword)301 print(f"CSV path: {csv_path}") # For debugging, ensure it's correct.302 df = self.csv_to_dataframe(csv_path)303 if df is None or df.empty:304 raise ValueError("The DataFrame is empty. Please check the CSV content.")305 306 # Step 2: Save the DataFrame to a text file307 output_file_path = self.save_dataframe_in_text_file(df)308 print(f"Output file saved at: {output_file_path}")309 310 # Step 3: Load and split the document data311 docs = self.csv_loader(output_file_path)312 if not docs:313 raise ValueError("No content was loaded from the CSV file.")314 315 split_docs = self.document_text_spilliter(docs)316 if not split_docs:317 raise ValueError("Document splitting failed. No valid chunks were created.")318 319 # Step 4: Prepare the summarization prompt320 prompt_template = """Analyze the Financial Details and Write a brief and concise summary of how the company performed:321 {text}322 CONCISE SUMMARY:"""323 prompt = PromptTemplate.from_template(prompt_template)324 325 # Step 5: Prepare the refine prompt for summarization chain326 refine_template = (327 "Your job is to produce a final summary\n"328 "We have provided an existing summary up to a certain point: {existing_answer}\n"329 "We have the opportunity to refine the existing summary "330 "(only if needed) with some more context below.\n"331 "------------\n"332 "{text}\n"333 "------------\n"334 "Given the new context, refine the original summary. "335 "If the context isn't useful, return the original summary."336 "10 lines of summary are enough."337 )338 refine_prompt = PromptTemplate.from_template(refine_template)339 340 # Step 6: Load the summarization chain with Azure ChatGPT341 chain = load_summarize_chain(342 llm=AzureChatOpenAI(azure_deployment="GPT-4o"),343 chain_type="refine",344 question_prompt=prompt,345 refine_prompt=refine_prompt,346 return_intermediate_steps=True,347 input_key="input_documents",348 output_key="output_text",349 )350 351 # Step 7: Generate the summary352 result = chain({"input_documents": split_docs}, return_only_outputs=True)353 354 # Step 8: Process and return the summary355 one_year_perfomance_summary = self.change_bullet_points(result["output_text"])356 357 # Log final summary358 print(f"Generated Summary: {one_year_perfomance_summary}")359 360 return one_year_perfomance_summary361 except Exception as e:362 print(f"Error during one_year_summary processing: {str(e)}")363 return None364 365 def main(self,keyword):366 367 368 clean_url = self.get_url(keyword)369 link_summary = self.get_each_link_summary(clean_url)370 clean_summary = self.one_day_summary(link_summary)371 key_value = self.extract_key_value_pair(clean_summary)372 373 return clean_summary, key_value374 375 def company_names(self,input_text):376 words = input_text.split("-")377 return words[1]378 379 def gradio_interface(self):380 381 with gr.Blocks(css="style.css",theme='SherlockRamos/Feliz') as app:382 gr.HTML("""383 <style>384 .footer {385 display: none !important;386 }387 footer {388 display: none !important;389 }390 #foot {391 display: none !important;392 }393 .svelte-1fzp3xt {394 display: none !important;395 }396 #root > div > div > div {397 padding-bottom: 0 !important;398 }399 .custom-footer {400 text-align: center;401 padding: 10px;402 font-size: 14px;403 color: #333;404 }405 </style>406 """)407 gr.HTML("""<div><center><img src="https://seeklogo.com/images/I/intercontinental-exchange-logo-5117BA0846-seeklogo.com.png" alt="Broadridge" style="width:100px;height:100px;"></center></div>""") 408 409 gr.HTML("""<center class="darkblue" text-align:center;padding:30px;'><center>410 <center><h1 class ="center" style="color:#fff"></h1></center>411 <br><center><h1 style="color:#000">Finance Tool for Investors</h1></center>""") 412 # gr.HTML("""<center class="darkblue" style='background-color:rgb(0,1,36); text-align:center;padding:25px;'><center><h1 class ="center">413 # <img src="file=logo.png" height="110px" width="280px"></h1></center>414 # <br><h1 style="color:#fff"> </h1></center>""")415 with gr.Row(elem_id="col-container"):416 with gr.Column(scale=1.0, min_width=150, ):417 input_news = gr.Textbox(label="Company Name")418 with gr.Accordion("List_of_Companies", open = False):419 with gr.Row(elem_id="col-container"):420 with gr.Column(scale=1.0, min_width=150 ):421 gr.Examples(422 [["Apple Inc. - AAPL"], ["Microsoft Corporation - MSFT"],["Amazon.com Inc. - AMZN"],["Facebook Inc. - FB"],["Tesla Inc. - TSLA"]],423 [input_news],424 input_news,425 fn=self.company_names,426 cache_examples=True,427 )428 429 with gr.Row(elem_id="col-container"):430 with gr.Column(scale=1.0, min_width=150):431 analyse = gr.Button("Analyse")432 with gr.Row(elem_id="col-container"):433 with gr.Column(scale=0.50, min_width=150):434 result_summary = gr.Textbox(label="Summary For Last Day Perfomance", lines = 12)435 with gr.Column(scale=0.50, min_width=150):436 key_value_pair_result = gr.Textbox(label="Discussed Topics", lines = 12)437 with gr.Row(elem_id="col-container"):438 with gr.Column(scale=1.0, min_width=0):439 plot_for_day =gr.Plot(label="Sentiment for Last Day")440 441 plot_for_day.width = 500442 plot_for_day.height = 600443 with gr.Row(elem_id="col-container"):444 with gr.Column(scale=1.0, min_width=150):445 analyse_sentiment = gr.Button("Analyse Sentiment For Last Day")446 with gr.Row(elem_id="col-container"):447 with gr.Column(scale=1.0, min_width=150, ):448 one_year_summary = gr.Textbox(label="Summary For One Year Performance",lines = 12)449 with gr.Row(elem_id="col-container"):450 with gr.Column(scale=1.0, min_width=150):451 one_year = gr.Button("Analyse One Year Summary")452 with gr.Row(elem_id="col-container"):453 with gr.Column(scale=1.0, min_width=0):454 plot_for_year =gr.Plot(label="Sentiment for One Year")455 plot_for_day.width = 500456 plot_for_day.height = 600457 with gr.Row(elem_id="col-container"):458 with gr.Column(scale=1.0, min_width=150):459 analyse_sentiment_for_year = gr.Button("Analyse Sentiment For One Year")460 461 analyse.click(self.main, input_news, [result_summary,key_value_pair_result])462 analyse_sentiment.click(self.display_graph,result_summary,[plot_for_day])463 one_year.click(self.one_year_summary,input_news,one_year_summary)464 analyse_sentiment_for_year.click(self.display_graph,one_year_summary,[plot_for_year])465 466 app.launch(debug=True)467 468if __name__ == "__main__":469 470 text_process = KeyValueExtractor()471 text_process.gradio_interface()