Nexialog/ESMA-GPT
0
1import json2from collections import defaultdict3import openai4import re5from config import CFG_APP6from text_embedder import SentenceTransformersTextEmbedder7from datetime import datetime8import tiktoken9 10doc_metadata = json.load(open(CFG_APP.DOC_METADATA_PATH, "r"))11# Embedding Model12if "sentence-transformers" in CFG_APP.EMBEDDING_MODEL:13 text_embedder = SentenceTransformersTextEmbedder(14 model_name=CFG_APP.EMBEDDING_MODEL,15 paragraphs_path=CFG_APP.DATA_FOLDER,16 device=CFG_APP.DEVICE,17 load_existing_index=True,18 )19else:20 raise ValueError("Embedding model not found !")21 22 23# Util Functions24def retrieve_doc_metadata(doc_metadata, doc_id):25 for meta in doc_metadata:26 if meta["id"] == doc_id:27 return meta28 29 30def get_reformulation_prompt(query: str) -> list:31 return [32 {33 "role": "user",34 "content": f"""{CFG_APP.REFORMULATION_PROMPT}35 ---36 query: {query}37 standalone question: """,38 }39 ]40 41def get_hyde_prompt(query: str) -> list:42 return [43 {44 "role": "user",45 "content": f"""{CFG_APP.HYDE_PROMPT}46 ---47 query: {query}48 output: """,49 }50 ]51 52 53def make_pairs(lst):54 """From a list of even lenght, make tupple pairs55 Args:56 lst (list): a list of even lenght57 Returns:58 list: the list as tupple pairs59 """60 assert not (l := len(lst) % 2), f"your list is of lenght {l} which is not even"61 return [(lst[i], lst[i + 1]) for i in range(0, len(lst), 2)]62 63 64def make_html_source(paragraph, meta_doc, i):65 content = paragraph["content"]66 meta_paragraph = paragraph["meta"]67 return f"""68<div class="card" id="document-{i}">69 <div class="card-content">70 <h2>Excerpts {i} - Document {meta_doc['num_doc']} - Page {meta_paragraph['page_number']}</h2>71 <p>{content}</p>72 </div>73 <div class="card-footer">74 <span>{meta_doc['short_name']}</span>75 <a href="{meta_doc['url']}#page={meta_paragraph['page_number']}" target="_blank" class="pdf-link">76 <span role="img" aria-label="Open PDF">🔗</span>77 </a>78 </div>79</div>80"""81 82def make_citations_source(citation_dic, query, Hyde: False):83 citation_list = [f'Doc {values[0]} - {keys} (excerpts {values[1]})' for keys, values in citation_dic.items()]84 85 html_output = '<div class="source">\n'86 html_output += ' <div class="title">Sources</div>\n'87 if Hyde :88 html_output += f' <div>Query used for retrieval (with the HyDE technique after no response): {query}</div>\n'89 else :90 html_output += f' <div>Query used for retrieval: {query}</div>\n'91 html_output += ' <br>\n'92 html_output += ' <ul>\n'93 94 for row in citation_list :95 html_output += f'<li>{row}</li>'96 97 html_output += ' </ul>\n'98 html_output += '</div>\n'99 100 return html_output101 102 103def preprocess_message(text: str, docs_url: dict) -> str:104 return re.sub(105 r"\[doc (\d+)\]",106 lambda match: f'<a href="{docs_url[match.group(1)]}" target="_blank" class="pdf-link">{match.group(0)}</a>',107 text,108 )109 110 111def parse_glossary(query):112 file = "glossary.json"113 glossary = json.load(open(file, "r"))114 words_query = query.split(" ")115 for i, word in enumerate(words_query):116 for key in glossary.keys():117 if word.lower() == key.lower():118 words_query[i] = words_query[i] + f" ({glossary[key]})"119 return " ".join(words_query)120 121 122def num_tokens_from_string(string: str, encoding_name: str) -> int:123 encoding = tiktoken.encoding_for_model(encoding_name)124 num_tokens = len(encoding.encode(string))125 return num_tokens126 127 128def chat(129 query: str,130 history: list,131 threshold: float = CFG_APP.THRESHOLD,132 k_total: int = CFG_APP.K_TOTAL,133) -> tuple:134 """retrieve relevant documents in the document store then query gpt-turbo135 Args:136 query (str): user message.137 history (list, optional): history of the conversation. Defaults to [system_template].138 report_type (str, optional): should be "All available" or "IPCC only". Defaults to "All available".139 threshold (float, optional): similarity threshold, don't increase more than 0.568. Defaults to 0.56.140 Yields:141 tuple: chat gradio format, chat openai format, sources used.142 """143 144 reformulated_query = openai.ChatCompletion.create(145 model=CFG_APP.MODEL_NAME,146 messages=get_reformulation_prompt(parse_glossary(query)),147 temperature=0,148 max_tokens=CFG_APP.MAX_TOKENS_REF_QUESTION,149 )150 151 reformulated_query = reformulated_query["choices"][0]["message"]["content"]152 153 if len(reformulated_query.split("\n")) == 2:154 reformulated_query, language = reformulated_query.split("\n")155 language = language.split(":")[1].strip()156 else:157 reformulated_query = reformulated_query.split("\n")[0]158 language = "English"159 160 sources, scores = text_embedder.retrieve_faiss(161 reformulated_query,162 k_total=k_total,163 threshold=threshold,164 )165 166 if CFG_APP.DEBUG == True:167 print("Scores : \n", scores)168 169 messages = history + [{"role": "user", "content": query}]170 171 docs_url = defaultdict(str)172 173 if len(sources) > 0:174 docs_string = []175 docs_html = []176 citations = {}177 178 num_tokens = num_tokens_from_string(CFG_APP.SOURCES_PROMPT, CFG_APP.MODEL_NAME)179 num_doc = 1180 181 for i, data in enumerate(sources, 1):182 meta_doc = retrieve_doc_metadata(doc_metadata, data["meta"]["document_id"])183 doc_content = f"📃 Doc {i}: \n{data['content']}"184 num_tokens_doc = num_tokens_from_string(doc_content, CFG_APP.MODEL_NAME)185 if num_tokens + num_tokens_doc > CFG_APP.MAX_TOKENS_API:186 break187 num_tokens += num_tokens_doc188 docs_string.append(doc_content)189 190 if meta_doc['short_name'] in citations.keys():191 citations[meta_doc['short_name']][1] += f', {i}'192 else :193 citations[meta_doc['short_name']] = [num_doc, f'{i}']194 num_doc += 1195 196 meta_doc["num_doc"] = citations[meta_doc['short_name']][0]197 198 docs_html.append(make_html_source(data, meta_doc, i))199 200 url_doc = f'<a href="{meta_doc["url"]}#page={data["meta"]["page_number"]}" target="_blank" class="pdf-link">'201 docs_url[i] = url_doc202 203 html_cit = [make_citations_source(citations, reformulated_query, Hyde=False)]204 205 docs_string = "\n\n".join( [f"Query used for retrieval:\n{reformulated_query}"] + docs_string)206 207 docs_html = "\n\n".join(html_cit + docs_html)208 209 messages.append(210 {211 "role": "system",212 "content": f"{CFG_APP.SOURCES_PROMPT}\n\n{docs_string}\n\nAnswer in {language}:",213 }214 )215 216 if CFG_APP.DEBUG == True:217 print(f" 👨💻 question asked by the user : {query}")218 print(f" 🕛 time : {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")219 220 print(" 🔌 messages sent to the API :")221 api_messages = [222 {"role": "system", "content": CFG_APP.INIT_PROMPT},223 {"role": "user", "content": reformulated_query},224 {225 "role": "system",226 "content": f"{CFG_APP.SOURCES_PROMPT}\n\n{docs_string}\n\nAnswer in {language}:",227 },228 ]229 for message in api_messages:230 print(231 f"length : {len(message['content'])}, content : {message['content']}"232 )233 234 response = openai.ChatCompletion.create(235 model=CFG_APP.MODEL_NAME,236 messages=[237 {"role": "system", "content": CFG_APP.INIT_PROMPT},238 {"role": "user", "content": reformulated_query},239 {240 "role": "system",241 "content": f"{CFG_APP.SOURCES_PROMPT}\n\nVery important : Answer in {language}.\n\n{docs_string}:",242 },243 ],244 temperature=0, # deterministic245 stream=True,246 max_tokens=CFG_APP.MAX_TOKENS_ANSWER,247 )248 complete_response = ""249 messages.pop()250 messages.append({"role": "assistant", "content": complete_response})251 for chunk in response:252 chunk_message = chunk["choices"][0]["delta"].get("content")253 if chunk_message:254 complete_response += chunk_message255 complete_response = preprocess_message(complete_response, docs_url)256 messages[-1]["content"] = complete_response257 gradio_format = make_pairs([a["content"] for a in messages[1:]])258 yield gradio_format, messages, docs_html259 260 else:261 reformulated_query = openai.ChatCompletion.create(262 model=CFG_APP.MODEL_NAME,263 messages=get_hyde_prompt(parse_glossary(query)),264 temperature=0,265 max_tokens=CFG_APP.MAX_TOKENS_REF_QUESTION,266 )267 268 reformulated_query = reformulated_query["choices"][0]["message"]["content"]269 270 if len(reformulated_query.split("\n")) == 2:271 reformulated_query, language = reformulated_query.split("\n")272 language = language.split(":")[1].strip()273 else:274 reformulated_query = reformulated_query.split("\n")[0]275 language = "English"276 277 sources, scores = text_embedder.retrieve_faiss(278 reformulated_query,279 k_total=k_total,280 threshold=threshold,281 )282 283 if CFG_APP.DEBUG == True:284 print("Scores : \n", scores)285 286 if len(sources) > 0 :287 docs_string = []288 docs_html = []289 citations = {}290 291 num_tokens = num_tokens_from_string(CFG_APP.SOURCES_PROMPT, CFG_APP.MODEL_NAME)292 293 num_doc = 1294 295 for i, data in enumerate(sources, 1):296 meta_doc = retrieve_doc_metadata(doc_metadata, data["meta"]["document_id"])297 doc_content = f"📃 Doc {i}: \n{data['content']}"298 num_tokens_doc = num_tokens_from_string(doc_content, CFG_APP.MODEL_NAME)299 if num_tokens + num_tokens_doc > CFG_APP.MAX_TOKENS_API:300 break301 num_tokens += num_tokens_doc302 docs_string.append(doc_content)303 304 if meta_doc['short_name'] in citations.keys():305 citations[meta_doc['short_name']][1] += f', {i}'306 else:307 citations[meta_doc['short_name']] = [num_doc, f'{i}']308 num_doc += 1309 310 meta_doc["num_doc"] = citations[meta_doc['short_name']][0]311 312 docs_html.append(make_html_source(data, meta_doc, i))313 314 url_doc = f'<a href="{meta_doc["url"]}#page={data["meta"]["page_number"]}" target="_blank" class="pdf-link">'315 docs_url[i] = url_doc316 317 html_cit = [make_citations_source(citations, reformulated_query, Hyde=True)]318 319 docs_string = "\n\n".join([f"Query used for retrieval:\n{reformulated_query}"] + docs_string)320 321 docs_html = "\n\n".join(html_cit + docs_html)322 323 messages.append(324 {325 "role": "system",326 "content": f"{CFG_APP.SOURCES_PROMPT}\n\n{docs_string}\n\nAnswer in {language}:",327 }328 )329 330 if CFG_APP.DEBUG == True:331 print(f" 👨💻 question asked by the user : {query}")332 print(f" 🕛 time : {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")333 334 print(" 🔌 messages sent to the API :")335 api_messages = [336 {"role": "system", "content": CFG_APP.INIT_PROMPT},337 {"role": "user", "content": reformulated_query},338 {339 "role": "system",340 "content": f"{CFG_APP.SOURCES_PROMPT}\n\nVery important : Answer in {language}.\n\n{docs_string}:",341 },342 ]343 for message in api_messages:344 print(345 f"length : {len(message['content'])}, content : {message['content']}"346 )347 348 response = openai.ChatCompletion.create(349 model=CFG_APP.MODEL_NAME,350 messages=[351 {"role": "system", "content": CFG_APP.INIT_PROMPT},352 {"role": "user", "content": reformulated_query},353 {354 "role": "system",355 "content": f"{CFG_APP.SOURCES_PROMPT}\n\nVery important : Answer in {language}.\n\n{docs_string}:",356 },357 ],358 temperature=0, # deterministic359 stream=True,360 max_tokens=CFG_APP.MAX_TOKENS_ANSWER,361 )362 complete_response = ""363 messages.pop()364 messages.append({"role": "assistant", "content": complete_response})365 for chunk in response:366 chunk_message = chunk["choices"][0]["delta"].get("content")367 if chunk_message:368 complete_response += chunk_message369 complete_response = preprocess_message(complete_response, docs_url)370 messages[-1]["content"] = complete_response371 gradio_format = make_pairs([a["content"] for a in messages[1:]])372 yield gradio_format, messages, docs_html373 374 else :375 docs_string = "⚠️ No relevant passages found in this report"376 complete_response = "**⚠️ No relevant passages found in this report, you may want to ask a more specific question.**"377 messages.append({"role": "assistant", "content": complete_response})378 gradio_format = make_pairs([a["content"] for a in messages[1:]])379 yield gradio_format, messages, docs_string380 381 382