awacke1/PythonAIPairProgrammer
2
1import streamlit as st2import streamlit.components.v1 as components3import os4import base645import glob6import io7import json8import mistune9import pytz10import math11import requests12import sys13import time14import re15import textract16import zipfile 17import random18import httpx # add 11/13/2319import asyncio20from openai import OpenAI21#from openai import AsyncOpenAI22from datetime import datetime23from xml.etree import ElementTree as ET24from bs4 import BeautifulSoup25from collections import deque26from audio_recorder_streamlit import audio_recorder27from dotenv import load_dotenv28from PyPDF2 import PdfReader29from langchain.text_splitter import CharacterTextSplitter30from langchain.embeddings import OpenAIEmbeddings31from langchain.vectorstores import FAISS32from langchain.chat_models import ChatOpenAI33from langchain.memory import ConversationBufferMemory34from langchain.chains import ConversationalRetrievalChain35from templates import css, bot_template, user_template36from io import BytesIO37from contextlib import redirect_stdout38# code import tests39import seaborn40import plotly41import vega_datasets42import bokeh43import holoviews44import plotnine45import graphviz46import tensorflow47import torch48 49# set page config once50st.set_page_config(page_title="Python AI Pair Programmer", layout="wide")51 52# UI for sidebar controls53should_save = st.sidebar.checkbox("💾 Save", value=True)54col1, col2, col3, col4 = st.columns(4)55with col1:56 with st.expander("Settings 🧠💾", expanded=True):57 # File type for output, model choice58 menu = ["txt", "htm", "xlsx", "csv", "md", "py"]59 choice = st.sidebar.selectbox("Output File Type:", menu)60 model_choice = st.sidebar.radio("Select Model:", ('gpt-3.5-turbo', 'gpt-3.5-turbo-0301'))61 62# Define a context dictionary to maintain the state between exec calls63context = {}64 65def create_file(filename, prompt, response, should_save=True):66 if not should_save:67 return68 69 # Extract base filename without extension70 base_filename, ext = os.path.splitext(filename)71 72 # Initialize the combined content73 combined_content = ""74 75 # Add Prompt with markdown title and emoji76 combined_content += "# Prompt 📝\n" + prompt + "\n\n"77 78 # Add Response with markdown title and emoji79 combined_content += "# Response 💬\n" + response + "\n\n"80 81 # Check for code blocks in the response82 resources = re.findall(r"```([\s\S]*?)```", response)83 for resource in resources:84 # Check if the resource contains Python code85 if "python" in resource.lower():86 # Remove the 'python' keyword from the code block87 cleaned_code = re.sub(r'^\s*python', '', resource, flags=re.IGNORECASE | re.MULTILINE)88 89 # Add Code Results title with markdown and emoji90 combined_content += "# Code Results 🚀\n"91 92 # Redirect standard output to capture it93 original_stdout = sys.stdout94 sys.stdout = io.StringIO()95 96 # Execute the cleaned Python code within the context97 try:98 exec(cleaned_code, context)99 code_output = sys.stdout.getvalue()100 combined_content += f"```\n{code_output}\n```\n\n"101 realtimeEvalResponse = "# Code Results 🚀\n" + "```" + code_output + "```\n\n"102 st.code(realtimeEvalResponse)103 104 except Exception as e:105 combined_content += f"```python\nError executing Python code: {e}\n```\n\n"106 107 # Restore the original standard output108 sys.stdout = original_stdout109 else:110 # Add non-Python resources with markdown and emoji111 combined_content += "# Resource 🛠️\n" + "```" + resource + "```\n\n"112 113 # Save the combined content to a Markdown file114 if should_save:115 with open(f"{base_filename}.md", 'w') as file:116 file.write(combined_content)117 st.code(combined_content)118 119 # Create a Base64 encoded link for the file120 with open(f"{base_filename}.md", 'rb') as file:121 encoded_file = base64.b64encode(file.read()).decode()122 href = f'<a href="data:file/markdown;base64,{encoded_file}" download="{filename}">Download File 📄</a>'123 st.markdown(href, unsafe_allow_html=True)124 125 126# Read it aloud 127def readitaloud(result):128 documentHTML5='''129 <!DOCTYPE html>130 <html>131 <head>132 <title>Read It Aloud</title>133 <script type="text/javascript">134 function readAloud() {135 const text = document.getElementById("textArea").value;136 const speech = new SpeechSynthesisUtterance(text);137 window.speechSynthesis.speak(speech);138 }139 </script>140 </head>141 <body>142 <h1>🔊 Read It Aloud</h1>143 <textarea id="textArea" rows="10" cols="80">144 '''145 documentHTML5 = documentHTML5 + result146 documentHTML5 = documentHTML5 + '''147 </textarea>148 <br>149 <button onclick="readAloud()">🔊 Read Aloud</button>150 </body>151 </html>152 '''153 154 components.html(documentHTML5, width=800, height=300)155 #return result156 157def generate_filename(prompt, file_type):158 central = pytz.timezone('US/Central')159 safe_date_time = datetime.now(central).strftime("%m%d_%H%M")160 replaced_prompt = prompt.replace(" ", "_").replace("\n", "_")161 safe_prompt = "".join(x for x in replaced_prompt if x.isalnum() or x == "_")[:90]162 return f"{safe_date_time}_{safe_prompt}.{file_type}"163 164# Chat and Chat with files165def chat_with_model(prompt, document_section, model_choice='gpt-3.5-turbo'):166 model = model_choice167 conversation = [{'role': 'system', 'content': 'You are a python script writer.'}]168 conversation.append({'role': 'user', 'content': prompt})169 if len(document_section)>0:170 conversation.append({'role': 'assistant', 'content': document_section})171 start_time = time.time()172 report = []173 res_box = st.empty()174 collected_chunks = []175 collected_messages = []176 key = os.getenv('OPENAI_API_KEY')177 178 client = OpenAI(179 api_key= os.getenv('OPENAI_API_KEY')180 )181 stream = client.chat.completions.create(182 model='gpt-3.5-turbo',183 messages=conversation,184 stream=True,185 )186 all_content = "" # Initialize an empty string to hold all content187 for part in stream:188 chunk_message = (part.choices[0].delta.content or "")189 collected_messages.append(chunk_message) # save the message190 content=part.choices[0].delta.content191 try:192 if len(content) > 0:193 report.append(content)194 all_content += content 195 result = "".join(report).strip()196 res_box.markdown(f'*{result}*') 197 except:198 st.write(' ')199 full_reply_content = all_content200 st.write("Elapsed time:")201 st.write(time.time() - start_time)202 filename = generate_filename(full_reply_content, choice)203 create_file(filename, prompt, full_reply_content, should_save)204 readitaloud(full_reply_content)205 return full_reply_content206 207def chat_with_file_contents(prompt, file_content, model_choice='gpt-3.5-turbo'):208 conversation = [{'role': 'system', 'content': 'You are a helpful assistant.'}]209 conversation.append({'role': 'user', 'content': prompt})210 if len(file_content)>0:211 conversation.append({'role': 'assistant', 'content': file_content})212 client = OpenAI(213 api_key= os.getenv('OPENAI_API_KEY')214 )215 response = client.chat.completions.create(model=model_choice, messages=conversation)216 return response['choices'][0]['message']['content']217 218def link_button_with_emoji(url, title, emoji_summary):219 emojis = ["💉", "🏥", "🌡️", "🩺", "🔬", "💊", "🧪", "👨⚕️", "👩⚕️"]220 random_emoji = random.choice(emojis)221 st.markdown(f"[{random_emoji} {emoji_summary} - {title}]({url})")222 223python_parts = {224 "Syntax": {"emoji": "✏️", "details": "Variables, Comments, Printing"},225 "Data Types": {"emoji": "📊", "details": "Numbers, Strings, Lists, Tuples, Sets, Dictionaries"},226 "Control Structures": {"emoji": "🔁", "details": "If, Elif, Else, Loops, Break, Continue"},227 "Functions": {"emoji": "🔧", "details": "Defining, Calling, Parameters, Return Values"},228 "Classes": {"emoji": "🏗️", "details": "Creating, Inheritance, Methods, Properties"},229 "API Interaction": {"emoji": "🌐", "details": "Requests, JSON Parsing, HTTP Methods"},230 "Data Visualization Libraries1": {"emoji": "📈", "details": "matplotlib"},231 "Data Visualization Libraries2": {"emoji": "📈", "details": "seaborn"},232 "Data Visualization Libraries3": {"emoji": "📈", "details": "plotly"},233 "Data Visualization Libraries4": {"emoji": "📈", "details": "altair"},234 "Data Visualization Libraries5": {"emoji": "📈", "details": "bokeh"},235 "Data Visualization Libraries6": {"emoji": "📈", "details": "pydeck"},236 "Data Visualization Libraries7": {"emoji": "📈", "details": "holoviews"},237 "Data Visualization Libraries8": {"emoji": "📈", "details": "plotnine"},238 "Data Visualization Libraries9": {"emoji": "📈", "details": "graphviz"},239 "Error Handling": {"emoji": "⚠️", "details": "Try, Except, Finally, Raising"},240 "Scientific & Data Analysis Libraries": {"emoji": "🧪", "details": "Numpy, Pandas, Scikit-Learn, TensorFlow, SciPy, Pillow"},241 "Advanced Concepts": {"emoji": "🧠", "details": "Decorators, Generators, Context Managers, Metaclasses, Asynchronous Programming"},242 "Web & Network Libraries": {"emoji": "🕸️", "details": "Flask, Django, Requests, BeautifulSoup, HTTPX, Asyncio"},243 "Streamlit & Extensions1": {"emoji": "💡", "details": "Streamlit"},244 "Streamlit & Extensions2": {"emoji": "💡", "details": "Streamlit-AgGrid"},245 "Streamlit & Extensions3": {"emoji": "💡", "details": "Streamlit-Folium"},246 "Streamlit & Extensions4": {"emoji": "💡", "details": "Streamlit-Pandas-Profiling"},247 "Streamlit & Extensions5": {"emoji": "💡", "details": "Streamlit-Vega-Lite, Gradio"},248 "Gradio": {"emoji": "💡", "details": "gradio"},249 "File Handling & Serialization": {"emoji": "📁", "details": "PyPDF2, Pytz, Json, Base64, Zipfile, Random, Glob, IO"},250 "Machine Learning & AI": {"emoji": "🤖", "details": "OpenAI, LangChain, HuggingFace"},251 "Text & Data Extraction": {"emoji": "🔍", "details": "TikToken, Textract, SQLAlchemy, Pillow"},252 "XML & Collections Libraries": {"emoji": "📚", "details": "XML, Collections"},253 "Top PyPI Libraries1": {"emoji": "🏆", "details": "Requests, Pillow, SQLAlchemy, Flask, Django, SciPy, Beautiful Soup, PyTest, PyGame, Twisted"},254 "Top PyPI Libraries2": {"emoji": "🏆", "details": "numpy, pandas, matplotlib, requests, beautifulsoup4"},255 "Top PyPI Libraries3": {"emoji": "🏆", "details": "langchain, openai, PyPDF2, pytz"},256 "Top PyPI Libraries4": {"emoji": "🏆", "details": "streamlit, audio_recorder_streamlit, gradio"},257 "Top PyPI Libraries5": {"emoji": "🏆", "details": "tiktoken, textract, glob, io"},258 "Top PyPI Libraries6": {"emoji": "🏆", "details": "matplotlib, seaborn, plotly, altair, bokeh, pydeck"},259 "Top PyPI Libraries7": {"emoji": "🏆", "details": "streamlit, streamlit-aggrid, streamlit-folium, streamlit-pandas-profiling, streamlit-vega-lite"},260 "Top PyPI Libraries8": {"emoji": "🏆", "details": "holoviews, plotnine, graphviz"},261 "Top PyPI Libraries9": {"emoji": "🏆", "details": "json, base64, zipfile, random"},262 "Top PyPI Libraries10": {"emoji": "🏆", "details": "httpx, asyncio, xml, collections, huggingface "}263}264 265 266response_placeholders = {}267example_placeholders = {}268 269def display_python_parts_old2():270 st.title("Python Interactive Learning Platform")271 272 for part, content in python_parts.items():273 with st.expander(f"{content['emoji']} {part} - {content['details']}", expanded=False):274 if st.button(f"Show Example for {part}", key=f"example_{part}"):275 example = "Write short python script examples with mock data in python list dictionary for inputs for " + part276 example_placeholders[part] = example277 st.code(example_placeholders[part], language="python")278 response = chat_with_model(f'Write python script with short code examples for: {content["details"]}', part)279 response_placeholders[part] = response280 st.write(f"#### {content['emoji']} {part} Example")281 st.code(response_placeholders[part], language="python")282 283 if st.button(f"Take Quiz on {part}", key=f"quiz_{part}"):284 quiz = "Write Python script quiz examples with mock static data inputs for " + part285 response = chat_with_model(f'Write python code blocks for quiz program: {quiz}', part)286 response_placeholders[part] = response287 st.write(f"#### {content['emoji']} {part} Quiz")288 st.code(response_placeholders[part], language="python")289 290 prompt = f"Write python script with a few advanced coding examples using mock data input for {content['details']}"291 if st.button(f"Explore {part}", key=part):292 response = chat_with_model(prompt, part)293 response_placeholders[part] = response294 st.write(f"#### {content['emoji']} {part} Details")295 st.code(response_placeholders[part], language="python")296 297 298def display_python_parts():299 st.title("Python Interactive Learning Platform")300 for part, content in python_parts.items():301 with st.expander(f"{content['emoji']} {part} - {content['details']}", expanded=False):302 if st.button(f"Show Example for {part}", key=f"example_{part}"):303 example = "Python script example with mock example inputs for " + part304 example_placeholders[part] = example305 st.code(example_placeholders[part], language="python")306 response = chat_with_model('Create detailed advanced python script code examples for:' + example_placeholders[part], part)307 if st.button(f"Take Quiz on {part}", key=f"quiz_{part}"):308 quiz = "Python script quiz example with mock example inputs for " + part309 response = chat_with_model(quiz, part)310 prompt = f"Learn about advanced coding examples using mock example inputs for {content['details']}"311 if st.button(f"Explore {part}", key=part):312 response = chat_with_model(prompt, part)313 response_placeholders[part] = response314 if part in response_placeholders:315 st.markdown(f"**Response:** {response_placeholders[part]}")316 317def add_paper_buttons_and_links():318 page = st.sidebar.radio("Choose a page:", ["Python Pair Programmer"])319 if page == "Python Pair Programmer":320 display_python_parts()321 322 col1, col2, col3, col4 = st.columns(4)323 324 with col1:325 with st.expander("MemGPT 🧠💾", expanded=False):326 link_button_with_emoji("https://arxiv.org/abs/2310.08560", "MemGPT", "🧠💾 Memory OS")327 outline_memgpt = "Memory Hierarchy, Context Paging, Self-directed Memory Updates, Memory Editing, Memory Retrieval, Preprompt Instructions, Semantic Memory, Episodic Memory, Emotional Contextual Understanding"328 if st.button("Discuss MemGPT Features"):329 chat_with_model("Discuss the key features of MemGPT: " + outline_memgpt, "MemGPT")330 331 with col2:332 with st.expander("AutoGen 🤖🔗", expanded=False):333 link_button_with_emoji("https://arxiv.org/abs/2308.08155", "AutoGen", "🤖🔗 Multi-Agent LLM")334 outline_autogen = "Cooperative Conversations, Combining Capabilities, Complex Task Solving, Divergent Thinking, Factuality, Highly Capable Agents, Generic Abstraction, Effective Implementation"335 if st.button("Explore AutoGen Multi-Agent LLM"):336 chat_with_model("Explore the key features of AutoGen: " + outline_autogen, "AutoGen")337 338 with col3:339 with st.expander("Whisper 🔊🧑🚀", expanded=False):340 link_button_with_emoji("https://arxiv.org/abs/2212.04356", "Whisper", "🔊🧑🚀 Robust STT")341 outline_whisper = "Scaling, Deep Learning Approaches, Weak Supervision, Zero-shot Transfer Learning, Accuracy & Robustness, Pre-training Techniques, Broad Range of Environments, Combining Multiple Datasets"342 if st.button("Learn About Whisper STT"):343 chat_with_model("Learn about the key features of Whisper: " + outline_whisper, "Whisper")344 345 with col4:346 with st.expander("ChatDev 💬💻", expanded=False):347 link_button_with_emoji("https://arxiv.org/pdf/2307.07924.pdf", "ChatDev", "💬💻 Comm. Agents")348 outline_chatdev = "Effective Communication, Comprehensive Software Solutions, Diverse Social Identities, Tailored Codes, Environment Dependencies, User Manuals"349 if st.button("Deep Dive into ChatDev"):350 chat_with_model("Deep dive into the features of ChatDev: " + outline_chatdev, "ChatDev")351 352add_paper_buttons_and_links()353 354 355# Process user input is a post processor algorithm which runs after document embedding vector DB play of GPT on context of documents..356def process_user_input(user_question):357 # Check and initialize 'conversation' in session state if not present358 if 'conversation' not in st.session_state:359 st.session_state.conversation = {} # Initialize with an empty dictionary or an appropriate default value360 361 response = st.session_state.conversation({'question': user_question})362 st.session_state.chat_history = response['chat_history']363 364 for i, message in enumerate(st.session_state.chat_history):365 template = user_template if i % 2 == 0 else bot_template366 st.write(template.replace("{{MSG}}", message.content), unsafe_allow_html=True)367 368 # Save file output from PDF query results369 filename = generate_filename(user_question, 'txt')370 create_file(filename, user_question, message.content, should_save)371 372 # New functionality to create expanders and buttons373 create_expanders_and_buttons(message.content)374 375def create_expanders_and_buttons(content):376 # Split the content into paragraphs377 paragraphs = content.split("\n\n")378 for paragraph in paragraphs:379 # Identify the header and detail in the paragraph380 header, detail = extract_feature_and_detail(paragraph)381 if header and detail:382 with st.expander(header, expanded=False):383 if st.button(f"Explore {header}"):384 expanded_outline = "Expand on the feature: " + detail385 chat_with_model(expanded_outline, header)386 387def extract_feature_and_detail(paragraph):388 # Use regex to find the header and detail in the paragraph389 match = re.match(r"(.*?):(.*)", paragraph)390 if match:391 header = match.group(1).strip()392 detail = match.group(2).strip()393 return header, detail394 return None, None395 396def transcribe_audio(file_path, model):397 key = os.getenv('OPENAI_API_KEY')398 headers = {399 "Authorization": f"Bearer {key}",400 }401 with open(file_path, 'rb') as f:402 data = {'file': f}403 st.write("Read file {file_path}", file_path)404 OPENAI_API_URL = "https://api.openai.com/v1/audio/transcriptions"405 response = requests.post(OPENAI_API_URL, headers=headers, files=data, data={'model': model})406 if response.status_code == 200:407 st.write(response.json())408 chatResponse = chat_with_model(response.json().get('text'), '') # *************************************409 transcript = response.json().get('text')410 #st.write('Responses:')411 #st.write(chatResponse)412 filename = generate_filename(transcript, 'txt')413 #create_file(filename, transcript, chatResponse)414 response = chatResponse415 user_prompt = transcript416 create_file(filename, user_prompt, response, should_save)417 return transcript418 else:419 st.write(response.json())420 st.error("Error in API call.")421 return None422 423def save_and_play_audio(audio_recorder):424 audio_bytes = audio_recorder()425 if audio_bytes:426 filename = generate_filename("Recording", "wav")427 with open(filename, 'wb') as f:428 f.write(audio_bytes)429 st.audio(audio_bytes, format="audio/wav")430 return filename431 return None432 433 434 435def truncate_document(document, length):436 return document[:length]437 438def divide_document(document, max_length):439 return [document[i:i+max_length] for i in range(0, len(document), max_length)]440 441def get_table_download_link(file_path):442 with open(file_path, 'r') as file:443 try:444 data = file.read()445 except:446 st.write('')447 return file_path 448 b64 = base64.b64encode(data.encode()).decode() 449 file_name = os.path.basename(file_path)450 ext = os.path.splitext(file_name)[1] # get the file extension451 if ext == '.txt':452 mime_type = 'text/plain'453 elif ext == '.py':454 mime_type = 'text/plain'455 elif ext == '.xlsx':456 mime_type = 'text/plain'457 elif ext == '.csv':458 mime_type = 'text/plain'459 elif ext == '.htm':460 mime_type = 'text/html'461 elif ext == '.md':462 mime_type = 'text/markdown'463 else:464 mime_type = 'application/octet-stream' # general binary data type465 href = f'<a href="data:{mime_type};base64,{b64}" target="_blank" download="{file_name}">{file_name}</a>'466 return href467 468def CompressXML(xml_text):469 root = ET.fromstring(xml_text)470 for elem in list(root.iter()):471 if isinstance(elem.tag, str) and 'Comment' in elem.tag:472 elem.parent.remove(elem)473 return ET.tostring(root, encoding='unicode', method="xml")474 475def read_file_content(file,max_length):476 if file.type == "application/json":477 content = json.load(file)478 return str(content)479 elif file.type == "text/html" or file.type == "text/htm":480 content = BeautifulSoup(file, "html.parser")481 return content.text482 elif file.type == "application/xml" or file.type == "text/xml":483 tree = ET.parse(file)484 root = tree.getroot()485 xml = CompressXML(ET.tostring(root, encoding='unicode'))486 return xml487 elif file.type == "text/markdown" or file.type == "text/md":488 md = mistune.create_markdown()489 content = md(file.read().decode())490 return content491 elif file.type == "text/plain":492 return file.getvalue().decode()493 else:494 return ""495 496def extract_mime_type(file):497 # Check if the input is a string498 if isinstance(file, str):499 pattern = r"type='(.*?)'"500 match = re.search(pattern, file)501 if match:502 return match.group(1)503 else:504 raise ValueError(f"Unable to extract MIME type from {file}")505 # If it's not a string, assume it's a streamlit.UploadedFile object506 elif isinstance(file, streamlit.UploadedFile):507 return file.type508 else:509 raise TypeError("Input should be a string or a streamlit.UploadedFile object")510 511 512 513def extract_file_extension(file):514 # get the file name directly from the UploadedFile object515 file_name = file.name516 pattern = r".*?\.(.*?)$"517 match = re.search(pattern, file_name)518 if match:519 return match.group(1)520 else:521 raise ValueError(f"Unable to extract file extension from {file_name}")522 523def pdf2txt(docs):524 text = ""525 for file in docs:526 file_extension = extract_file_extension(file)527 # print the file extension528 st.write(f"File type extension: {file_extension}")529 530 # read the file according to its extension531 try:532 if file_extension.lower() in ['py', 'txt', 'html', 'htm', 'xml', 'json']:533 text += file.getvalue().decode('utf-8')534 elif file_extension.lower() == 'pdf':535 from PyPDF2 import PdfReader536 pdf = PdfReader(BytesIO(file.getvalue()))537 for page in range(len(pdf.pages)):538 text += pdf.pages[page].extract_text() # new PyPDF2 syntax539 except Exception as e:540 st.write(f"Error processing file {file.name}: {e}")541 return text542 543def txt2chunks(text):544 text_splitter = CharacterTextSplitter(separator="\n", chunk_size=1000, chunk_overlap=200, length_function=len)545 return text_splitter.split_text(text)546 547def vector_store(text_chunks):548 key = os.getenv('OPENAI_API_KEY')549 embeddings = OpenAIEmbeddings(openai_api_key=key)550 return FAISS.from_texts(texts=text_chunks, embedding=embeddings)551 552def get_chain(vectorstore):553 llm = ChatOpenAI()554 memory = ConversationBufferMemory(memory_key='chat_history', return_messages=True)555 return ConversationalRetrievalChain.from_llm(llm=llm, retriever=vectorstore.as_retriever(), memory=memory)556 557def divide_prompt(prompt, max_length):558 words = prompt.split()559 chunks = []560 current_chunk = []561 current_length = 0562 for word in words:563 if len(word) + current_length <= max_length:564 current_length += len(word) + 1 # Adding 1 to account for spaces565 current_chunk.append(word)566 else:567 chunks.append(' '.join(current_chunk))568 current_chunk = [word]569 current_length = len(word)570 chunks.append(' '.join(current_chunk)) # Append the final chunk571 return chunks572 573def create_zip_of_files(files):574 """575 Create a zip file from a list of files.576 """577 zip_name = "all_files.zip"578 with zipfile.ZipFile(zip_name, 'w') as zipf:579 for file in files:580 zipf.write(file)581 return zip_name582 583 584def get_zip_download_link(zip_file):585 """586 Generate a link to download the zip file.587 """588 with open(zip_file, 'rb') as f:589 data = f.read()590 b64 = base64.b64encode(data).decode()591 href = f'<a href="data:application/zip;base64,{b64}" download="{zip_file}">Download All</a>'592 return href593 594 595def main():596 597 # Audio, transcribe, GPT:598 filename = save_and_play_audio(audio_recorder)599 600 if filename is not None:601 try:602 transcription = transcribe_audio(filename, "whisper-1")603 except:604 st.write(' ')605 st.sidebar.markdown(get_table_download_link(filename), unsafe_allow_html=True)606 filename = None607 608 # prompt interfaces609 user_prompt = st.text_area("Enter prompts, instructions & questions:", '', height=100)610 611 # file section interface for prompts against large documents as context612 collength, colupload = st.columns([2,3]) # adjust the ratio as needed613 with collength:614 max_length = st.slider("File section length for large files", min_value=1000, max_value=128000, value=12000, step=1000)615 with colupload:616 uploaded_file = st.file_uploader("Add a file for context:", type=["pdf", "xml", "json", "xlsx", "csv", "html", "htm", "md", "txt"])617 618 619 # Document section chat620 621 document_sections = deque()622 document_responses = {}623 if uploaded_file is not None:624 file_content = read_file_content(uploaded_file, max_length)625 document_sections.extend(divide_document(file_content, max_length))626 if len(document_sections) > 0:627 if st.button("👁️ View Upload"):628 st.markdown("**Sections of the uploaded file:**")629 for i, section in enumerate(list(document_sections)):630 st.markdown(f"**Section {i+1}**\n{section}")631 st.markdown("**Chat with the model:**")632 for i, section in enumerate(list(document_sections)):633 if i in document_responses:634 st.markdown(f"**Section {i+1}**\n{document_responses[i]}")635 else:636 if st.button(f"Chat about Section {i+1}"):637 st.write('Reasoning with your inputs...')638 response = chat_with_model(user_prompt, section, model_choice)639 document_responses[i] = response640 filename = generate_filename(f"{user_prompt}_section_{i+1}", choice)641 create_file(filename, user_prompt, response, should_save)642 st.sidebar.markdown(get_table_download_link(filename), unsafe_allow_html=True)643 644 if st.button('💬 Chat'):645 st.write('Reasoning with your inputs...')646 647 # Divide the user_prompt into smaller sections648 user_prompt_sections = divide_prompt(user_prompt, max_length)649 full_response = ''650 for prompt_section in user_prompt_sections:651 # Process each section with the model652 response = chat_with_model(prompt_section, ''.join(list(document_sections)), model_choice)653 full_response += response + '\n' # Combine the responses654 response = full_response655 filename = generate_filename(user_prompt, choice)656 create_file(filename, user_prompt, response, should_save)657 st.sidebar.markdown(get_table_download_link(filename), unsafe_allow_html=True)658 659 all_files = glob.glob("*.*")660 all_files = [file for file in all_files if len(os.path.splitext(file)[0]) >= 20] # exclude files with short names661 all_files.sort(key=lambda x: (os.path.splitext(x)[1], x), reverse=True) # sort by file type and file name in descending order662 663 664 # Sidebar buttons Download All and Delete All665 colDownloadAll, colDeleteAll = st.sidebar.columns([3,3])666 with colDownloadAll:667 if st.button("⬇️ Download All"):668 zip_file = create_zip_of_files(all_files)669 st.markdown(get_zip_download_link(zip_file), unsafe_allow_html=True)670 with colDeleteAll:671 if st.button("🗑 Delete All"):672 for file in all_files:673 os.remove(file)674 st.experimental_rerun()675 676 # Sidebar of Files Saving History and surfacing files as context of prompts and responses677 file_contents=''678 next_action=''679 for file in all_files:680 col1, col2, col3, col4, col5 = st.sidebar.columns([1,6,1,1,1]) # adjust the ratio as needed681 with col1:682 if st.button("🌐", key="md_"+file): # md emoji button683 with open(file, 'r') as f:684 file_contents = f.read()685 next_action='md'686 with col2:687 st.markdown(get_table_download_link(file), unsafe_allow_html=True)688 with col3:689 if st.button("📂", key="open_"+file): # open emoji button690 with open(file, 'r') as f:691 file_contents = f.read()692 next_action='open'693 with col4:694 if st.button("🔍", key="read_"+file): # search emoji button695 with open(file, 'r') as f:696 file_contents = f.read()697 next_action='search'698 with col5:699 if st.button("🗑", key="delete_"+file):700 os.remove(file)701 st.experimental_rerun()702 703 if len(file_contents) > 0:704 if next_action=='open':705 file_content_area = st.text_area("File Contents:", file_contents, height=500)706 if next_action=='md':707 st.markdown(file_contents)708 if next_action=='search':709 file_content_area = st.text_area("File Contents:", file_contents, height=500)710 st.write('Reasoning with your inputs...')711 response = chat_with_model(user_prompt, file_contents, model_choice)712 filename = generate_filename(file_contents, choice)713 create_file(filename, user_prompt, response, should_save)714 715 st.experimental_rerun()716 717if __name__ == "__main__":718 main()719 720load_dotenv()721st.write(css, unsafe_allow_html=True)722 723st.header("Chat with documents :books:")724user_question = st.text_input("Ask a question about your documents:")725if user_question:726 process_user_input(user_question)727 728with st.sidebar:729 st.subheader("Your documents")730 docs = st.file_uploader("import documents", accept_multiple_files=True)731 with st.spinner("Processing"):732 raw = pdf2txt(docs)733 if len(raw) > 0:734 length = str(len(raw))735 text_chunks = txt2chunks(raw)736 vectorstore = vector_store(text_chunks)737 st.session_state.conversation = get_chain(vectorstore)738 st.markdown('# AI Search Index of Length:' + length + ' Created.') # add timing739 filename = generate_filename(raw, 'txt')740 create_file(filename, raw, '', should_save)