hra/ChatGPT-Tech-Radar

1from gpt_index import GPTListIndex, SimpleWebPageReader, BeautifulSoupWebReader, GPTSimpleVectorIndex,LLMPredictor2from IPython.display import Markdown, display3from langchain.agents import load_tools, Tool, initialize_agent4from langchain.llms import OpenAI5from langchain.agents import ZeroShotAgent, Tool, AgentExecutor6from langchain.agents import initialize_agent, Tool7from langchain import LLMChain8from langchain import PromptTemplate9import gradio as gr10import pandas as pd11import openai12from sklearn.manifold import TSNE13from sklearn.cluster import KMeans14from openai.embeddings_utils import get_embedding15 16import numpy as np17import matplotlib.pyplot as plt18import matplotlib19import datetime20from datetime import datetime, date, time, timedelta21import os22from PIL import Image23from PIL import ImageOps24from PIL import Image, ImageDraw, ImageFont25from io import BytesIO26import requests27 28import gcsfs29fs = gcsfs.GCSFileSystem(project='createinsightsproject',token='anon')30fs.ls('trends_chrome_extension_bucket')31print('Started')32###download both text and image from cloud to display33with fs.open('trends_chrome_extension_bucket/lastradartext.txt', 'rb') as file:34    data_old = file.read()35print(data_old)36value1,value2,value3,value4,value5,value6=str(data_old.decode()).split('SEPERATOR')37 38img_data = requests.get('https://storage.googleapis.com/trends_chrome_extension_bucket/lasttechradar.png').content39with open('lasttechradar.png', 'wb') as handler:40    handler.write(img_data)41 42def getlastimage():43    #print('Came into getlastimage')44    img_data = requests.get('https://storage.googleapis.com/trends_chrome_extension_bucket/lasttechradar.png').content45    with open('lasttechradar1.png', 'wb') as handler:46        handler.write(img_data)47 48    with fs.open('trends_chrome_extension_bucket/lastradartext.txt', 'rb') as file:49        data_old = file.read()50    #print(data_old)51    value1,value2,value3,value4,value5,value6=str(data_old.decode()).split('SEPERATOR')52    return 'lasttechradar1.png',value1.strip(),value2.strip(),value3.strip(),value4.strip(),value5.strip(),value6.strip()53 54 55def getstuff(openapikey):56  dateforfilesave=datetime.today().strftime("%d-%m-%Y %I:%M%p")57  print(dateforfilesave)58  os.environ['OPENAI_API_KEY'] = str(openapikey)59 60  mainlistofanswers=[]61  for each in ['www.mckinsey.com','www.bcg.com','www.bain.com','www.accenture.com']:62    print(each)63    Input_URL = "https://"+each 64    documents = SimpleWebPageReader(html_to_text=True).load_data([Input_URL])65    index = GPTSimpleVectorIndex(documents)66    print('Came here 0')67    #@title # Creating your Langchain Agent68    def querying_db(query: str):69      response = index.query(query)70      return response71 72    tools = [73        Tool(74            name = "QueryingDB",75            func=querying_db,76            description="This function takes a query string as input and returns the most relevant answer from the documentation as output"77        )]78    llm = OpenAI(temperature=0,openai_api_key=openapikey)79    print('Came here 1')80    query_string = "what are the top technologies mentioned?" 81 82    agent = initialize_agent(tools, llm, agent="zero-shot-react-description", verbose=True)83    result = agent.run(query_string)84    mainlistofanswers.append(result)85  86  print('Came here 2')87  print(mainlistofanswers)88  newlistoftech=[]89  newlistofcompanies=[]90  for i in range(len(mainlistofanswers)):91    each=mainlistofanswers[i]92    each=each.replace("The top technologies mentioned are ","").replace("The technologies mentioned are ","")93    each=each.replace(":","").replace(" and ",",").replace("and ",",").replace(" and",",").replace(" the "," ").replace("the "," ").replace(" the"," ").strip()94    for item in each.split(","):95        if item!='':96          newlistoftech.append(item.strip())97          newlistofcompanies.append(i)98  tech_df=pd.DataFrame()99  tech_df['tech']=newlistoftech100  tech_df['company']=newlistofcompanies101  print(newlistoftech)102  print('Came here 3')103  embedding_model = "text-embedding-ada-002"104  embedding_encoding = "cl100k_base"  # this the encoding for text-embedding-ada-002105  max_tokens = 8000  # the maximum for text-embedding-ada-002 is 8191106 107  tech_df["embedding"] = tech_df['tech'].apply(lambda x: get_embedding(x, engine=embedding_model))108 109  print('Came here 4')110 111  # Load the embeddings112 113  # Convert to a list of lists of floats114  matrix = np.array(tech_df['embedding'].to_list())115  perplexityvalue=max(int(len(tech_df['embedding'].to_list()))/2,5) ###original value was a constant of 15116    117  # Create a t-SNE model and transform the data118  tsne = TSNE(n_components=2, perplexity=perplexityvalue, random_state=42, init='random', learning_rate=200)119  vis_dims = tsne.fit_transform(matrix)120 121  n_clusters = 5122 123  kmeans = KMeans(n_clusters=n_clusters, init="k-means++", random_state=42)124  kmeans.fit(matrix)125  labels = kmeans.labels_126  tech_df["Cluster"] = labels127  print('Came here 5')128  colors = ["red", "darkorange", "darkgrey", "blue", "darkgreen"]129  x = [x for x,y in vis_dims]130  y = [y for x,y in vis_dims]131  color_indices = tech_df['Cluster'].values 132 133  colormap = matplotlib.colors.ListedColormap(colors)134  #plt.scatter(x, y, c=color_indices, cmap=colormap, alpha=0.3,)135  fig, ax = plt.subplots(figsize=(12,8))136  ax.scatter(x, y, c=color_indices, cmap=colormap, alpha=1, s=100)137 138  for i, txt in enumerate(tech_df['tech'].tolist()):139      ax.annotate(txt, (x[i], y[i]),fontsize=14)140 141  plt.title("Top Technologies as of "+dateforfilesave,fontsize=20)142  plt.axis('off')143  plt.savefig('lasttechradar.png', bbox_inches='tight')144  print('Came here 6')145  response = openai.Completion.create(146        engine="text-davinci-003",147        prompt=f'I will give you top technologies list. Write a paragraph on it.\n\nTechnologies:'+",".join(tech_df['tech'].tolist()),148        temperature=0,149        max_tokens=1024,150        top_p=1,151        frequency_penalty=0,152        presence_penalty=0,153    )154  print(response["choices"][0]["text"].replace("\n", ""))155  desc_tmp=response["choices"][0]["text"].replace("\n", "")156  print('Came here 7')157  # Reading a review which belong to each group.158  rev_per_cluster = 5159 160  clusterstextlist=[]161  for i in range(n_clusters):162      print(f"Cluster {i} Theme:", end=" ")163 164      reviews = "\n".join(tech_df[tech_df['Cluster'] == i]['tech'].tolist())165      response = openai.Completion.create(166          engine="text-davinci-003",167          prompt=f'What do the following technologies have in common?\n\nCustomer reviews:\n"""\n{reviews}\n"""\n\nTheme:',168          temperature=0,169          max_tokens=64,170          top_p=1,171          frequency_penalty=0,172          presence_penalty=0,173      )174      print(response["choices"][0]["text"].replace("\n", ""))175 176      print(reviews)177      clusterstextlist.append("Cluster "+str(i)+"\nTheme:"+response["choices"][0]["text"].replace("\n", "")+'\n'+reviews+'\n'+"-" * 10+'\n\n')178  179  textlist=[mainlistofanswers[0],"SEPERATOR",mainlistofanswers[1],"SEPERATOR",mainlistofanswers[2],"SEPERATOR",mainlistofanswers[3],"SEPERATOR",desc_tmp,"SEPERATOR","".join(clusterstextlist)]180  ###create file with new info locally & upload to bucket181  with open('lastradartext.txt', 'w') as f:182    for line in textlist:183        f.write(f"{line}\n")184        185  with fs.open('trends_chrome_extension_bucket/lastradartext.txt', 'wb') as file:186      for line in textlist:187        file.write(f"{line}\n".encode())188          189  print('Came here 8')   190 191  ###read it and put in output192 193  with open('lastradartext.txt', 'r') as file:194    data_old = file.read()195  value1,value2,value3,value4,value5,value6=str(data_old).split('SEPERATOR')196 197  ###upload image to cloud for next run display198  with open('lasttechradar.png','rb') as image_file:199    image_string = image_file.read()200    with fs.open('trends_chrome_extension_bucket/lasttechradar.png', 'wb') as file:201        file.write(image_string)202    203  return 'lasttechradar.png',mainlistofanswers[0],mainlistofanswers[1],mainlistofanswers[2],mainlistofanswers[3],desc_tmp,"".join(clusterstextlist)204 205with gr.Blocks() as demo:206    gr.Markdown("<h1><center>ChatGPT Technology Radar</center></h1>")207    gr.Markdown(208        """What are the top technologies as of now? Let us query top consulting company websites & use ChatGPT to understand. \n\nShowcases ChatGPT integrated with real data. It shows how to get real-time data and marry it with ChatGPT capabilities. This demonstrates 'Chain of Thought' thinking using ChatGPT.\nLangChain & GPT-Index are both used.\n ![visitors](https://visitor-badge.glitch.me/badge?page_id=hra.ChatGPT-Tech-Radar)"""209        )210    211    with gr.Row() as row:212      textboxopenapi = gr.Textbox(placeholder="Enter OpenAPI Key...", lines=1,label='OpenAPI Key')213      btn = gr.Button("Refresh")214    with gr.Row() as row:215      with gr.Column():216        output_image = gr.components.Image(label="Tech Radar",value='lasttechradar.png')    217      with gr.Column():218        outputMck = gr.Textbox(placeholder=value1, lines=1,label='McKinsey View')219        outputBcg = gr.Textbox(placeholder=value2, lines=1,label='BCG View')220        outputBain = gr.Textbox(placeholder=value3, lines=1,label='Bain View')221        outputAcc = gr.Textbox(placeholder=value4, lines=1,label='Accenture View')222    with gr.Row() as row:223      with gr.Column():224        outputdesc = gr.Textbox(placeholder=value5, lines=1,label='Description')225      with gr.Column():226        outputclusters = gr.Textbox(placeholder=value6, lines=1,label='Clusters')227 228 229    btn.click(getstuff, inputs=[textboxopenapi],outputs=[output_image,outputMck,outputBcg,outputBain,outputAcc,outputdesc,outputclusters])230    231    demo.load(getlastimage,[],[output_image,outputMck,outputBcg,outputBain,outputAcc,outputdesc,outputclusters])232    233demo.launch(debug=True)