edunuke/docqa-hosp
0
1# Import libraries
2from pathlib import Path
3import fitz
4import os
5import time
6from tqdm import tqdm
7import gradio as gr
8from docx import Document as Docx
9from langchain.llms.openai import OpenAI
10from langchain.chat_models import ChatOpenAI
11from langchain.chains import RetrievalQA
12from langchain.embeddings import OpenAIEmbeddings, HuggingFaceEmbeddings
13from langchain.vectorstores import FAISS
14from langchain.schema import Document
15from langchain.document_loaders import TextLoader
16from langchain.text_splitter import RecursiveCharacterTextSplitter
17from langchain.memory import ConversationBufferMemory
18from langchain import PromptTemplate
19
20
21def read_doc(filepath):
22 filepath = filepath
23 doc = fitz.open(filepath)
24 return doc
25
26
27def read_page(doc, page_number):
28 page_index = page_number - 1
29 page = doc.load_page(page_index)
30 return page.get_text()
31
32
33def load_pdf(filepath, pr=gr.Progress(track_tqdm=True)):
34 doc = fitz.open(filepath)
35 content = []
36 for t in tqdm(range(len(doc))):
37 content.append(read_page(doc, t))
38 time.sleep(0.1)
39 return "\n".join(content)
40
41def load_docx(file, pr=gr.Progress(track_tqdm=True)):
42 doc = Docx(file)
43 content = []
44 for p in tqdm(doc.paragraphs):
45 content.append(p.text)
46 time.sleep(0.1)
47 return "\n".join(content)
48
49
50def save_temp(file, content):
51 with open(file, "w") as f:
52 f.write(content)
53
54
55def split_text(doc, chunk_size=1_500, chunk_overlap=50):
56 text_splitter = RecursiveCharacterTextSplitter(
57 chunk_size=chunk_size,
58 chunk_overlap=chunk_overlap,
59 )
60 return text_splitter.create_documents([doc])
61
62def load_to_openai(doc):
63 global qa
64 db = FAISS.from_documents(doc, embeddings)
65 retriever = db.as_retriever()
66 qa = RetrievalQA.from_chain_type(llm=llm,
67 chain_type="stuff",
68 retriever=retriever)
69
70
71# use GPT3.5 Embeddings
72embeddings = OpenAIEmbeddings()
73llm = ChatOpenAI(model_name="gpt-3.5-turbo")
74
75def ask(question):
76 global qa
77 if qa is None:
78 raise gr.exceptions.Error('No hay Documentos.Cargar Nuevo Documento')
79
80 return qa.run(question).lstrip()
81
82
83def upload_file(files, pr=gr.Progress(track_tqdm=True)):
84 clear_doc()
85 filepath = files.name
86 extension = filepath.split("/")[-1].split(".")[-1]
87 if extension == "pdf":
88 content = load_pdf(filepath)
89 content = split_text(content,
90 chunk_size=1_500,
91 chunk_overlap=50)
92 # content = [Document(page_content=doc) for doc in content]
93
94 elif extension == "docx":
95
96 content = load_docx(filepath)
97 content = split_text(content,
98 chunk_size=1_500,
99 chunk_overlap=50)
100 # content = [Document(page_content=doc) for doc in content]
101 else:
102 raise gr.exceptions.Error(f'Only .docx and .pdf formats supported but {extension} used instead.')
103 load_to_openai(content)
104
105
106def clear_doc():
107 global qa
108 qa = None
109
110def clear_bot_area():
111 return None, None
112
113
114
115template= """
116 <h2>De que se trata:</h2>
117 <p> Este es un demo de preguntas y respuestas sobre documentos en la que subes un pdf y relizas tus preguntas en lenguage natural</p>
118 <p> No subas documentos privados ni sensitivos.</p>
119 <h2>Instrucciones:</h2>
120 <ul>
121 <li>Cargar un archivo en uno de los formatos permitidos: pdf o docx.</li>
122 <li>Escribe preguntas sobre el documento en el area de consultas.</li>
123 <li>Apreta el boton de enviar consultas.</li>
124 <li>Las respuestas apareceran en el area de respuestas.</li>
125 </ul>
126 <h2>Tips para una mejor respuesta:</h2>
127 <ul>
128 <li>Incluya detalles en su consulta para obtener respuestas más relevantes</li>
129 <li>Pide a la modelo que adopte a una persona</li>
130 <li>Use delimitadores para indicar claramente distintas partes de la entrada</li>
131 <li>Especificar los pasos necesarios para completar una tarea</li>
132 <li>Proporcione ejemplos</li>
133 <li>Especifique la longitud deseada de la salida</li>
134 </ul>
135 """
136
137css ="""#chatbot{
138 overflow:scroll;
139 border:1px dotted;
140 border-style: dotted;
141 border-color: rgba(0, 0, 0, 0.5);
142 }
143 #fileupload{
144 border:1px dotted;
145 border-style: dotted;
146 border-color: rgba(0, 0, 0, 0.3);
147 }
148 #mssg{
149 border:1px dotted;
150 border-style: dotted;
151 border-color: rgba(0, 0, 0, 0.3);
152 }
153 .message-wrap > .bot{
154 border-color: rgba(0, 255, 0, 0.3);
155 background-color: rgba(0, 0, 0, 0.6);
156 }
157 """
158def respond(message, chat_history):
159 bot_message = ask(message)
160 chat_history.append((message, bot_message))
161 return "", chat_history
162
163demo = gr.Blocks(theme=gr.themes.Monochrome(), css=css)
164with demo:
165 with gr.Row():
166 gr.HTML("<h1 style='text-align: center'>Demo</h1>")
167
168 with gr.Row():
169 gr.Image("logo.jpg", label="", width=300, height=150)
170
171 with gr.Row(elem_id="fileupload"):
172
173 file_output = gr.File(type="file",
174 file_count="single",
175 file_types=[".pdf",".docx"])
176
177 file_output.upload(upload_file,
178 file_output,
179 file_output,
180 show_progress="full",
181 queue=True)
182
183 with gr.Row():
184 with gr.Column():
185 with gr.Row(elem_id="mssg"):
186 msg = gr.Textbox(label="Mis Consultas",
187 placeholder="Escribe tus consultas aqui y apreta [Enter] o [Enviar Consulta]...")
188 with gr.Row():
189 qa_button = gr.Button("Enviar Consulta")
190
191
192 with gr.Column():
193 with gr.Row(elem_id="chatbot"):
194 chatbot = gr.Chatbot()
195 qa_button.click(fn=respond,
196 inputs=[msg, chatbot],
197 outputs=[msg, chatbot],
198 show_progress="full")
199 with gr.Row():
200 clearbtn = gr.Button(value='Borrar Consultas')
201
202 clearbtn.click(clear_bot_area,
203 inputs=None,
204 outputs=[chatbot, msg])
205
206 msg.submit(respond, [msg, chatbot], [msg, chatbot])
207
208 gr.HTML(template)
209 demo.load(clear_doc, inputs=None, outputs=None)
210
211
212demo.queue(concurrency_count=3, max_size=3)
213demo.launch()