Almaatla/Standard_Intelligence_Dev
0
1import numpy as np2import io3import os4import zipfile5import logging6import collections7import tempfile8from langchain.document_loaders import UnstructuredFileLoader9from langchain.text_splitter import CharacterTextSplitter10from langchain.vectorstores import FAISS11from langchain.embeddings import HuggingFaceEmbeddings12import gradio as gr13 14from langchain.document_loaders import PDFMinerPDFasHTMLLoader15from bs4 import BeautifulSoup16import re17from langchain.docstore.document import Document18 19import unstructured20from unstructured.partition.docx import partition_docx21from unstructured.partition.auto import partition22 23 24import tiktoken25#from transformers import AutoTokenizer26 27from pypdf import PdfReader28 29import pandas as pd30 31import requests32import json33 34MODEL = "thenlper/gte-base"35CHUNK_SIZE = 150036CHUNK_OVERLAP = 40037 38embeddings = HuggingFaceEmbeddings(39 model_name=MODEL,40 cache_folder=os.getenv("SENTENCE_TRANSFORMERS_HOME")41)42 43 44 45# model_id = "mistralai/Mistral-7B-Instruct-v0.1"46# access_token = os.getenv("HUGGINGFACE_SPLITFILES_API_KEY")47 48# tokenizer = AutoTokenizer.from_pretrained(49# model_id,50# padding_side="left",51# token = access_token52# )53 54 55tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo")56 57 58text_splitter = CharacterTextSplitter(59 separator = "\n",60 chunk_size = CHUNK_SIZE,61 chunk_overlap = CHUNK_OVERLAP,62 length_function = len,63)64 65 66# def update_label(label1):67# return gr.update(choices=list(df.columns))68 69def function_split_call(fi_input, dropdown, choice, chunk_size):70 if choice == "Intelligent split":71 nb_pages = chunk_size72 return split_in_df(fi_input, nb_pages)73 elif choice == "Non intelligent split":74 return non_intelligent_split(fi_input, chunk_size)75 else:76 return split_by_keywords(fi_input,dropdown)77 78def change_textbox(dropdown,radio):79 if len(dropdown) == 0 :80 dropdown = ["introduction", "objective", "summary", "conclusion"]81 if radio == "Intelligent split":82 return gr.Dropdown(dropdown, visible=False), gr.Number(label="First pages to keep (0 for all)", value=2, interactive=True, visible=True)83 elif radio == "Intelligent split by keywords":84 return gr.Dropdown(dropdown, multiselect=True, visible=True, allow_custom_value=True), gr.Number(visible=False)85 elif radio == "Non intelligent split":86 return gr.Dropdown(dropdown, visible=False),gr.Number(label="Chunk size", value=1000, interactive=True, visible=True)87 else:88 return gr.Dropdown(dropdown, visible=False),gr.Number(visible=False)89 90 91def group_text_by_font_size(content):92 cur_fs = []93 cur_text = ''94 cur_page = -195 cur_c = content[0]96 multi_fs = False97 snippets = [] # first collect all snippets that have the same font size98 for c in content:99 # print(f"c={c}\n\n")100 if c.find('a') != None and c.find('a').get('name'):101 cur_page = int(c.find('a').get('name'))102 sp_list = c.find_all('span')103 if not sp_list:104 continue105 for sp in sp_list:106 # print(f"sp={sp}\n\n")107 if not sp:108 continue109 st = sp.get('style')110 if not st:111 continue112 fs = re.findall('font-size:(\d+)px',st)113 # print(f"fs={fs}\n\n")114 if not fs:115 continue116 fs = [int(fs[0])]117 if len(cur_fs)==0:118 cur_fs = fs119 if fs == cur_fs:120 cur_text += sp.text121 elif not sp.find('br') and cur_c==c:122 cur_text += sp.text123 cur_fs.extend(fs)124 multi_fs = True125 elif sp.find('br') and multi_fs == True: # if a br tag is found and the text is in a different fs, it is the last part of the multifontsize line126 cur_fs.extend(fs)127 snippets.append((cur_text+sp.text,max(cur_fs), cur_page))128 cur_fs = []129 cur_text = ''130 cur_c = c131 multi_fs = False132 else:133 snippets.append((cur_text,max(cur_fs), cur_page))134 cur_fs = fs135 cur_text = sp.text136 cur_c = c137 multi_fs = False138 snippets.append((cur_text,max(cur_fs), cur_page))139 return snippets140 141def get_titles_fs(fs_list):142 filtered_fs_list = [item[0] for item in fs_list if item[0] > fs_list[0][0]]143 return sorted(filtered_fs_list, reverse=True)144 145def calculate_total_characters(snippets):146 font_sizes = {} #dictionary to store font-size and total characters147 148 for text, font_size, _ in snippets:149 #remove newline# and digits150 cleaned_text = text.replace('\n', '')151 #cleaned_text = re.sub(r'\d+', '', cleaned_text)152 total_characters = len(cleaned_text)153 154 #update the dictionary155 if font_size in font_sizes:156 font_sizes[font_size] += total_characters157 else:158 font_sizes[font_size] = total_characters159 #convert the dictionary into a sorted list of tuples160 size_charac_list = sorted(font_sizes.items(), key=lambda x: x[1], reverse=True)161 162 return size_charac_list163 164def create_documents(source, snippets, font_sizes):165 docs = []166 167 titles_fs = get_titles_fs(font_sizes)168 169 for snippet in snippets:170 cur_fs = snippet[1]171 if cur_fs>font_sizes[0][0] and len(snippet[0])>2:172 content = min((titles_fs.index(cur_fs)+1), 3)*"#" + " " + snippet[0].replace(" ", " ")173 category = "Title"174 else:175 content = snippet[0].replace(" ", " ")176 category = "Paragraph"177 metadata={"source":source, "filename":source.split("/")[-1], "file_directory": "/".join(source.split("/")[:-1]), "file_category":"", "file_sub-cat":"", "file_sub2-cat":"", "category":category, "filetype":source.split(".")[-1], "page_number":snippet[2]}178 categories = source.split("/")179 cat_update=""180 if len(categories)>4:181 cat_update = {"file_category":categories[1], "file_sub-cat":categories[2], "file_sub2-cat":categories[3]}182 elif len(categories)>3:183 cat_update = {"file_category":categories[1], "file_sub-cat":categories[2]}184 elif len(categories)>2:185 cat_update = {"file_category":categories[1]}186 metadata.update(cat_update)187 docs.append(Document(page_content=content, metadata=metadata))188 return docs189 190## Group Chunks docx or pdf191 192# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE193def group_chunks_by_section(chunks, min_chunk_size=64):194 filtered_chunks = [chunk for chunk in chunks if chunk.metadata['category'] != 'PageBreak']# Add more filters if needed195 #print(f"filtered = {len(filtered_chunks)} - before = {len(chunks)}")196 new_chunks = []197 seen_paragraph = False198 new_title = True #switches when there is a new paragraph to create a new chunk199 for i, chunk in enumerate(filtered_chunks):200# print(f"\n\n\n#{i}:METADATA: {chunk.metadata['category']}")201 if new_title:202 #print(f"<-- NEW title DETECTED -->")203 new_chunk = chunk204 new_title = False205 add_content = False206 new_chunk.metadata['titles'] = ""207 #print(f"CONTENT: {new_chunk.page_content}\nMETADATA: {new_chunk.metadata['category']} \n title: {new_chunk.metadata['title']}")208 209 if chunk.metadata['category'].lower() =='title':210 new_chunk.metadata['titles'] += f"{chunk.page_content} ~~ "211 else:212 #Activates when a paragraph is seen after one or more titles213 seen_paragraph = True214 215 #Avoid adding the title 2 times to the page content216 if add_content:#and chunk.page_content not in new_chunk.page_content217 new_chunk.page_content += f"\n{chunk.page_content}"218 #edit the end_page number, the last one keeps its place219 try:220 new_chunk.metadata['end_page'] = chunk.metadata['page_number']221 except:222 print("", end="")223 #print("Exception: No page number in metadata")224 225 add_content = True226 227 #If filtered_chunks[i+1] raises an error, this is probably because this is the last chunk228 try:229 #If the next chunk is a title and we have already seen a paragraph and the current chunk content is long enough, we create a new document230 if filtered_chunks[i+1].metadata['category'].lower() =="title" and seen_paragraph and len(new_chunk.page_content)>min_chunk_size:231 if 'category' in new_chunk.metadata:232 new_chunk.metadata.pop('category')233 new_chunks.append(new_chunk)234 new_title = True235 seen_paragraph = False236 #index out of range237 except:238 new_chunks.append(new_chunk)239 #print('🆘 Gone through all chunks 🆘')240 break241 return new_chunks242 243# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE244## Split documents by font245 246def split_pdf(file_path):247 loader = PDFMinerPDFasHTMLLoader(file_path)248 249 data = loader.load()[0] # entire pdf is loaded as a single Document250 soup = BeautifulSoup(data.page_content,'html.parser')251 content = soup.find_all('div')#List of all elements in div tags252 try:253 snippets = group_text_by_font_size(content)254 except Exception as e:255 print("ERROR WHILE GROUPING BY FONT SIZE", e)256 snippets = [("ERROR WHILE GROUPING BY FONT SIZE", 0, -1)]257 font_sizes = calculate_total_characters(snippets)#get the amount of characters for each font_size258 chunks = create_documents(file_path, snippets, font_sizes)259 return chunks260 261# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE262def split_docx(file_path):263 chunks_elms = partition_docx(filename=file_path)264 chunks = []265 file_categories = file_path.split("/")266 for chunk_elm in chunks_elms:267 category = chunk_elm.category268 if category == "Title":269 chunk = Document(page_content= min(chunk_elm.metadata.to_dict()['category_depth']+1, 3)*"#" + ' ' + chunk_elm.text, metadata=chunk_elm.metadata.to_dict())270 else:271 chunk = Document(page_content=chunk_elm.text, metadata=chunk_elm.metadata.to_dict())272 metadata={"source":file_path, "filename":file_path.split("/")[-1], "file_category":"", "file_sub-cat":"", "file_sub2-cat":"", "category":category, "filetype":file_path.split(".")[-1]}273 cat_update=""274 if len(file_categories)>4:275 cat_update = {"file_category":file_categories[1], "file_sub-cat":file_categories[2], "file_sub2-cat":file_categories[3]}276 elif len(file_categories)>3:277 cat_update = {"file_category":file_categories[1], "file_sub-cat":file_categories[2]}278 elif len(file_categories)>2:279 cat_update = {"file_category":file_categories[1]}280 metadata.update(cat_update)281 chunk.metadata.update(metadata)282 chunks.append(chunk)283 return chunks284 285 286def split_txt(file_path, chunk_size=700):287 with open(file_path, 'r') as file:288 content = file.read()289 words = content.split()290 chunks = [words[i:i + chunk_size] for i in range(0, len(words), chunk_size)]291 292 file_basename = os.path.basename(file_path)293 file_directory = os.path.dirname(file_path)294 source = file_path295 296 documents = []297 for i, chunk in enumerate(chunks):298 tcontent = ' '.join(chunk)299 metadata = {300 'source': source,301 "filename": file_basename,302 'file_directory': file_directory,303 "file_category": "",304 "file_sub-cat": "",305 "file_sub2-cat": "",306 "category": "",307 "filetype": source.split(".")[-1],308 "page_number": i309 }310 document = Document(page_content=tcontent, metadata=metadata)311 documents.append(document)312 313 return documents314 315# Load the index of documents (if it has already been built)316 317def rebuild_index(input_folder, output_folder):318 paths_time = []319 to_keep = set()320 print(f'number of files {len(paths_time)}')321 if len(output_folder.list_paths_in_partition()) > 0:322 with tempfile.TemporaryDirectory() as temp_dir:323 for f in output_folder.list_paths_in_partition():324 with output_folder.get_download_stream(f) as stream:325 with open(os.path.join(temp_dir, os.path.basename(f)), "wb") as f2:326 f2.write(stream.read())327 index = FAISS.load_local(temp_dir, embeddings)328 to_remove = []329 logging.info(f"{len(index.docstore._dict)} vectors loaded")330 for idx, doc in index.docstore._dict.items():331 source = (doc.metadata["source"], doc.metadata["last_modified"])332 if source in paths_time:333 # Identify documents already indexed and still present in the source folder334 to_keep.add(source)335 else:336 # Identify documents removed from the source folder337 to_remove.append(idx)338 339 docstore_id_to_index = {v: k for k, v in index.index_to_docstore_id.items()}340 341 # Remove documents that have been deleted from the source folder342 vectors_to_remove = []343 for idx in to_remove:344 del index.docstore._dict[idx]345 ind = docstore_id_to_index[idx]346 del index.index_to_docstore_id[ind]347 vectors_to_remove.append(ind)348 index.index.remove_ids(np.array(vectors_to_remove, dtype=np.int64))349 350 index.index_to_docstore_id = {351 i: ind352 for i, ind in enumerate(index.index_to_docstore_id.values())353 }354 logging.info(f"{len(to_remove)} vectors removed")355 else:356 index = None357 to_add = [path[0] for path in paths_time if path not in to_keep]358 print(f'to_keep: {to_keep}')359 print(f'to_add: {to_add}')360 return index, to_add361 362# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE363def split_chunks_by_tokens(documents, max_length=170, overlap=10):364 # Create an empty list to store the resized documents365 resized = []366 367 # Iterate through the original documents list368 for doc in documents:369 encoded = tokenizer.encode(doc.page_content)370 if len(encoded) > max_length:371 remaining_encoded = tokenizer.encode(doc.page_content)372 while len(remaining_encoded) > 0:373 split_doc = Document(page_content=tokenizer.decode(remaining_encoded[:max(10, max_length)]), metadata=doc.metadata.copy())374 resized.append(split_doc)375 remaining_encoded = remaining_encoded[max(10, max_length - overlap):]376 377 else:378 resized.append(doc)379 print(f"Number of chunks before resplitting: {len(documents)} \nAfter splitting: {len(resized)}")380 return resized381 382# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE383def split_chunks_by_tokens_period(documents, max_length=170, overlap=10, min_chunk_size=20):384 # Create an empty list to store the resized documents385 resized = []386 previous_file=""387 to_encode = ""388 skip_next = False389 # Iterate through the original documents list390 for i, doc in enumerate(documents):391 if skip_next:392 skip_next = False393 continue394 current_file = doc.metadata['source']395 if current_file != previous_file: #chunk counting396 previous_file = current_file397 chunk_counter = 0398 is_first_chunk = True # Keep track of the first chunk in the document399 to_encode += doc.page_content400 # if last chunk < min_chunk_size we add it to the previous chunk for the splitting.401 try:402 if (documents[i+1] is documents[-1] or documents[i+1].metadata['source'] != documents[i+2].metadata['source']) and len(tokenizer.encode(documents[i+1].page_content)) < min_chunk_size: # if the next doc is the last doc of the current file or the last of the corpus 403 # print('SAME DOC')404 skip_next = True405 to_encode += documents[i+1].page_content406 except Exception as e:407 print(e)408 #print(f"to_encode:\n{to_encode}")409 encoded = tokenizer.encode(to_encode)#encode the current document410 if len(encoded) < min_chunk_size and not skip_next:411 # print(f"len(encoded):{len(encoded)}<min_chunk_size:{min_chunk_size}")412 continue413 elif skip_next:414 split_doc = Document(page_content=tokenizer.decode(encoded).replace('<s> ', ''), metadata=doc.metadata.copy())415 split_doc.metadata['token_length'] = len(tokenizer.encode(split_doc.page_content))416 resized.append(split_doc)417 # print(f"Added a document of {split_doc.metadata['token_length']} tokens 1")418 to_encode = ""419 continue420 else:421 # print(f"len(encoded):{len(encoded)}>=min_chunk_size:{min_chunk_size}")422 to_encode = ""423 if len(encoded) > max_length:424 # print(f"len(encoded):{len(encoded)}>=max_length:{max_length}")425 remaining_encoded = encoded426 is_last_chunk = False427 while len(remaining_encoded) > 1 and not is_last_chunk:428 # Check for a period in the first 'overlap' tokens429 overlap_text = tokenizer.decode(remaining_encoded[:overlap])# Index by token430 period_index_b = overlap_text.find('.')# Index by character431 if len(remaining_encoded)>max_length + min_chunk_size:432 # print("len(remaining_encoded)>max_length + min_chunk_size")433 current_encoded = remaining_encoded[:max(10, max_length)]434 else:435 # print("not len(remaining_encoded)>max_length + min_chunk_size")436 current_encoded = remaining_encoded #if the last chunk is to small, concatenate it with the previous one437 is_last_chunk = True438 split_doc = Document(page_content=tokenizer.decode(current_encoded).replace('<s> ', ''), metadata=doc.metadata.copy())439 split_doc.metadata['token_length'] = len(tokenizer.encode(split_doc.page_content))440 resized.append(split_doc)441 # print(f"Added a document of {split_doc.metadata['token_length']} tokens 2")442 break443 period_index_e = -1 # an amount of character that I am sure will be greater or equal to the max lengh of a chunk, could have done len(tokenizer.decode(current_encoded))444 if len(remaining_encoded)>max_length+min_chunk_size:# If it is not the last sub chunk445 # print("len(remaining_encoded)>max_length+min_chunk_size")446 overlap_text_last = tokenizer.decode(current_encoded[-overlap:])447 period_index_last = overlap_text_last.find('.')448 if period_index_last != -1 and period_index_last < len(overlap_text_last) - 1:449 # print(f"period index last found at {period_index_last}")450 period_index_e = period_index_last - len(overlap_text_last)451 # print(f"period_index_e :{period_index_e}")452 # print(f"last :{overlap_text_last}")453 if not is_first_chunk:#starting after the period in overlap454 # print("not is_first_chunk", period_index_b)455 if period_index_b == -1:# Period not found in overlap456 # print(". not found in overlap")457 split_doc = Document(page_content=tokenizer.decode(current_encoded)[:period_index_e].replace('<s> ', ''), metadata=doc.metadata.copy()) # Keep regular splitting458 else:459 if is_last_chunk : #not the first but the last460 # print("is_last_chunk")461 split_doc = Document(page_content=tokenizer.decode(current_encoded)[period_index_b+1:].replace('<s> ', ''), metadata=doc.metadata.copy())462 #print("Should start after \".\"")463 else:464 # print("not is_last_chunk", period_index_e, len(to_encode))465 split_doc = Document(page_content=tokenizer.decode(current_encoded)[period_index_b+1:period_index_e].replace('<s> ', ''), metadata=doc.metadata.copy()) # Split at the begining and the end466 else:#first chunk467 # print("else")468 split_doc = Document(page_content=tokenizer.decode(current_encoded)[:period_index_e].replace('<s> ', ''), metadata=doc.metadata.copy()) # split only at the end if its first chunk469 if 'titles' in split_doc.metadata:470 # print("title in metadata")471 chunk_counter += 1472 split_doc.metadata['chunk_id'] = chunk_counter473 #A1 We could round chunk length in token if we ignore the '.' position in the overlap and save time of computation474 split_doc.metadata['token_length'] = len(tokenizer.encode(split_doc.page_content))475 resized.append(split_doc)476 print(f"Added a document of {split_doc.metadata['token_length']} tokens 3")477 remaining_encoded = remaining_encoded[max(10, max_length - overlap):]478 is_first_chunk = False479 # # print(len(tokenizer.encode(split_doc.page_content)), split_doc.page_content[:50], "\n-----------------")480 # print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")481 # print(split_doc.page_content[:100])482 # # print("😂😂😂😂")483 # print(split_doc.page_content[-100:])484 # print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")485 else:# len(encoded)>min_chunk_size:#ignore the chunks that are too small486 print(f"found a chunk with the perfect size:{len(encoded)}")487 #print(f"◀Document:{{ {doc.page_content} }} was not added because to short▶")488 if 'titles' in doc.metadata:#check if it was splitted by or split_docx489 chunk_counter += 1490 doc.metadata['chunk_id'] = chunk_counter491 doc.metadata['token_length'] = len(encoded)492 doc.page_content = tokenizer.decode(encoded).replace('<s> ', '')493 resized.append(doc)494 print(f"Added a document of {doc.metadata['token_length']} tokens 4")495 print(f"Number of chunks before resplitting: {len(documents)} \nAfter splitting: {len(resized)}")496 return resized497 498# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE499 500def split_doc_in_chunks(input_folder, base_folders, nb_pages):501 docs = []502 for i, filename in enumerate(input_folder):503 path = filename#os.path.join(input_folder, filename)504 print(f"Treating file {i+1}/{len(input_folder)}")505 # Select the appropriate document loader506 chunks=[]507 if path.endswith(".pdf"):508 # try:509 print("Treatment of pdf file", path)510 raw_chunks = split_pdf(path)511 for raw_chunk in raw_chunks:512 print(f"BASE zzzzz LIST : {base_folders} = i = {i}")513 raw_chunk.metadata["Base Folder"] = base_folders[i]514 sb_chunks = group_chunks_by_section(raw_chunks)515 if nb_pages > 0:516 for sb_chunk in sb_chunks:517 print(f"CHUNK PAGENUM = {sb_chunk.metadata['page_number']}")518 if int(sb_chunk.metadata["page_number"])<=nb_pages:519 chunks.append(sb_chunk)520 else:521 break522 else:523 chunks = sb_chunks524 print(f"Document splitted in {len(chunks)} chunks")525 # for chunk in chunks:526 # print(f"\n\n____\n\n\nPDF CONTENT: \n{chunk.page_content}\ntitle: {chunk.metadata['title']}\nFile Name: {chunk.metadata['filename']}\n\n")527 # except Exception as e:528 # print("Error while splitting the pdf file: ", e)529 elif path.endswith(".docx"):530 try:531 print ("Treatment of docx file", path)532 raw_chunks = split_docx(path)533 for raw_chunk in raw_chunks:534 raw_chunk.metadata["Base Folder"] = base_folders[i]535 #print(f"RAW :\n***\n{raw_chunks}")536 chunks = group_chunks_by_section(raw_chunks)537 print(f"Document splitted in {len(chunks)} chunks")538 #if "cards-Jan 2022-SP.docx" in path:539 #for chunk in chunks:540 #print(f"\n\n____\n\n\nDOCX CONTENT: \n{chunk.page_content}\ntitle: {chunk.metadata['title']}\nFile Name: {chunk.metadata['filename']}\n\n")541 except Exception as e:542 print("Error while splitting the docx file: ", e)543 elif path.endswith(".doc"):544 try:545 loader = UnstructuredFileLoader(path)546 # Load the documents and split them in chunks547 chunks = loader.load_and_split(text_splitter=text_splitter)548 counter, counter2 = collections.Counter(), collections.Counter()549 filename = os.path.basename(path)550 # Define a unique id for each chunk551 for chunk in chunks:552 chunk.metadata["filename"] = filename.split("/")[-1]553 chunk.metadata["file_directory"] = filename.split("/")[:-1]554 chunk.metadata["filetype"] = filename.split(".")[-1]555 chunk.metadata["Base Folder"] = base_folders[i]556 if "page" in chunk.metadata:557 counter[chunk.metadata['page']] += 1558 for i in range(len(chunks)):559 counter2[chunks[i].metadata['page']] += 1560 chunks[i].metadata['source'] = filename561 else:562 if len(chunks) == 1:563 chunks[0].metadata['source'] = filename564 #The file type is not supported (e.g. .xlsx)565 except Exception as e:566 print(f"An error occurred: {e}")567 elif path.endswith(".txt"):568 try:569 print ("Treatment of txt file", path)570 chunks = split_txt(path)571 for chunk in chunks:572 chunk.metadata["Base Folder"] = base_folders[i]573 print(f"Document splitted in {len(chunks)} chunks")574 except Exception as e:575 print("Error while splitting the docx file: ", e)576 try:577 if len(chunks)>0:578 docs += chunks579 except NameError as e:580 print(f"An error has occured: {e}")581 return docs582 583# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE584def resplit_by_end_of_sentence(docs, max_len, overlap, min_len):585 print("❌❌\nResplitting docs by end of sentence\n❌❌")586 resized_docs = split_chunks_by_tokens_period(docs, max_len, overlap, min_len)587 try:588 # add chunk title to all resplitted chunks #todo move this to split_chunks_by_tokens_period(inject_title = True) with a boolean parameter589 cur_source = ""590 cpt_chunk = 1591 for resized_doc in resized_docs:592 try:593 title = resized_doc.metadata['titles'].split(' ~~ ')[-2] #Getting the last title of the chunk and adding it to the content if it is not the case594 if title not in resized_doc.page_content:595 resized_doc.page_content = title + "\n" + resized_doc.page_content596 if cur_source == resized_doc.metadata["source"]:597 resized_doc.metadata['chunk_number'] = cpt_chunk598 else:599 cpt_chunk = 1600 cur_source = resized_doc.metadata["source"]601 resized_doc.metadata['chunk_number'] = cpt_chunk602 except Exception as e:#either the title was notfound or title absent in metadata603 print("An error occured: ", e)604 #print(f"METADATA:\n{resized_doc.metadata}")605 cpt_chunk += 1606 except Exception as e:607 print('AN ERROR OCCURRED: ', e)608 return resized_docs609 610# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE611def build_index(docs, index, output_folder):612 if len(docs) > 0:613 if index is not None:614 # Compute the embedding of each chunk and index these chunks615 new_index = FAISS.from_documents(docs, embeddings)616 index.merge_from(new_index)617 else:618 index = FAISS.from_documents(docs, embeddings)619 with tempfile.TemporaryDirectory() as temp_dir:620 index.save_local(temp_dir)621 for f in os.listdir(temp_dir):622 output_folder.upload_file(f, os.path.join(temp_dir, f))623 624 625def extract_zip(zip_path):626 extracted_files = []627 with zipfile.ZipFile(zip_path, 'r') as zip_ref:628 for file_info in zip_ref.infolist():629 extracted_files.append(file_info.filename)630 zip_ref.extract(file_info.filename)631 return extracted_files632 633def split_in_df(files, nb_pages):634 processed_files = []635 base_folders = []636 print("Processing zip files...")637 for file_path in files:638 if file_path.endswith('.zip'):639 extracted_files = extract_zip(file_path)640 processed_files.extend(extracted_files)641 base_folders.extend([os.path.splitext(os.path.basename(file_path))[0]] * len(extracted_files))642 else:643 processed_files.append(file_path)644 base_folders.append("")645 print(f"BASE FOLDERS LIST : {base_folders}, FILES LIST : {processed_files}")646 print("Finished processing zip files\nSplitting files into chunks...")647 documents = split_doc_in_chunks(processed_files, base_folders, nb_pages)648 re_docs = resplit_by_end_of_sentence(documents, 700, 100, 1000)649 print("Finished splitting")650 df = pd.DataFrame()651 for re_doc in re_docs:652 filename = re_doc.metadata['filename']653 content = re_doc.page_content654 655 # metadata = document.metadata656 # metadata_keys = list(metadata.keys())657 # metadata_values = list(metadata.values())658 659 doc_data = {'Filename': filename, 'Content': content}660 661 doc_data["Token_Length"] = re_doc.metadata['token_length']662 doc_data["Titles"] = re_doc.metadata['titles'] if 'titles' in re_doc.metadata else ""663 doc_data["Base Folder"] = re_doc.metadata["Base Folder"]664 665 # for key, value in zip(metadata_keys, metadata_values):666 # doc_data[key] = value667 668 df = pd.concat([df, pd.DataFrame([doc_data])], ignore_index=True)669 670 df.to_excel("dataframe.xlsx", index=False)671 672 return "dataframe.xlsx"673 674 675 676# -------------------------------------------------------------------------------- SPLIT FILES BY KEYWORDS 677 678def split_by_keywords(files, key_words, words_limit=1000):679 processed_files = []680 extracted_content = []681 tabLine = []682 683 # For each files : stock the PDF, extract the Zips and convert the Doc & Docx to PDF684 try:685 not_duplicate = True686 for f in files:687 for p in processed_files:688 if (f[:f.rfind('.')] == p[:p.rfind('.')]):689 not_duplicate = False 690 if not_duplicate: 691 if f.endswith('.zip'):692 extracted_files = extract_zip(f)693 print(f"Those are my extracted files{extracted_files}")694 695 for doc in extracted_files:696 if doc.endswith('.doc') or doc.endswith('.docx'):697 processed_files.append(transform_to_pdf(doc))698 699 if doc.endswith('.pdf'):700 processed_files.append(doc)701 702 if f.endswith('.pdf'):703 processed_files.append(f)704 705 if f.endswith('.doc') or f.endswith('.docx'):706 processed_files.append(transform_to_pdf(f))707 708 except Exception as ex:709 print(f"Error occured while processing files : {ex}")710 711 # For each processed files extract content712 for file in processed_files:713 714 try:715 file_name = file716 file = PdfReader(file)717 pdfNumberPages = len(file.pages)718 for pdfPage in range(0, pdfNumberPages):719 720 load_page = file.get_page(pdfPage)721 text = load_page.extract_text()722 lines = text.split("\n")723 sizeOfLines = len(lines) - 1724 725 for index, line in enumerate(lines):726 print(line)727 for key in key_words:728 if key in line:729 print("Found keyword")730 lineBool = True731 lineIndex = index732 previousSelectedLines = []733 stringLength = 0734 linesForSelection = lines735 loadOnce = True736 selectedPdfPage = pdfPage737 738 while lineBool:739 print(lineIndex)740 if stringLength > words_limit or lineIndex < 0:741 lineBool = False742 else:743 if lineIndex == 0:744 print(f"Line index == 0")745 746 if pdfPage == 0:747 lineBool = False748 749 else:750 try:751 selectedPdfPage -= 1752 newLoad_page = file.get_page(selectedPdfPage)753 newText = newLoad_page.extract_text()754 newLines = newText.split("\n")755 linesForSelection = newLines756 print(f"len newLines{len(newLines)}")757 lineIndex = len(newLines) - 1758 except Exception as e:759 print(f"Loading previous PDF page failed")760 lineBool = False761 762 previousSelectedLines.append(linesForSelection[lineIndex])763 stringLength += len(linesForSelection[lineIndex])764 765 lineIndex -= 1766 previousSelectedLines = ' '.join(previousSelectedLines[::-1])767 768 lineBool = True769 lineIndex = index + 1770 nextSelectedLines = ""771 linesForSelection = lines772 loadOnce = True773 selectedPdfPage = pdfPage774 775 while lineBool:776 777 if len(nextSelectedLines.split()) > words_limit:778 lineBool = False779 else:780 if lineIndex > sizeOfLines:781 lineBool = False782 783 if pdfPage == pdfNumberPages - 1:784 lineBool = False785 786 else:787 try:788 selectedPdfPage += 1789 newLoad_page = file.get_page(selectedPdfPage)790 newText = newLoad_page.extract_text()791 newLines = newText.split("\n")792 linesForSelection = newLines793 lineIndex = 0794 except Exception as e:795 print(f"Loading next PDF page failed")796 lineBool = False797 else:798 nextSelectedLines += " " + linesForSelection[lineIndex]799 lineIndex += 1800 801 print(f"Previous Lines : {previousSelectedLines}")802 print(f"Next Lines : {nextSelectedLines}")803 selectedText = previousSelectedLines + ' ' + nextSelectedLines804 print(selectedText)805 tabLine.append([file_name, selectedText, key])806 print(f"Selected line in keywords is: {line}")807 808 except Exception as ex:809 print(f"Error occured while extracting content : {ex}")810 811 for r in tabLine:812 text_joined = ''.join(r[1])813 text_joined = r[2] + " : \n " + text_joined814 extracted_content.append([r[0], text_joined])815 816 df = pd.DataFrame()817 for content in extracted_content:818 filename = content[0]819 text = content[1]820 821 # metadata = document.metadata822 # metadata_keys = list(metadata.keys())823 # metadata_values = list(metadata.values())824 825 doc_data = {'Filename': filename[filename.rfind("/")+1:], 'Content': text}826 827 # for key, value in zip(metadata_keys, metadata_values):828 # doc_data[key] = value829 830 df = pd.concat([df, pd.DataFrame([doc_data])], ignore_index=True)831 832 df.to_excel("dataframe_keywords.xlsx", index=False)833 834 return "dataframe_keywords.xlsx"835 836# -------------------------------------------------------------------------------- NON INTELLIGENT SPLIT 837 838def transform_to_pdf(doc):839 instructions = {'parts': [{'file': 'document'}]}840 841 response = requests.request(842 'POST',843 'https://api.pspdfkit.com/build',844 headers = { 'Authorization': 'Bearer pdf_live_nS6tyylSW57PNw9TIEKKL3Tt16NmLCazlQWQ9D33t0Q'},845 files = {'document': open(doc, 'rb')},846 data = {'instructions': json.dumps(instructions)},847 stream = True848 )849 850 pdf_name = doc[:doc.find(".doc")] + ".pdf"851 852 if response.ok:853 with open(pdf_name, 'wb') as fd:854 for chunk in response.iter_content(chunk_size=8096):855 fd.write(chunk)856 return pdf_name857 858 else:859 print(response.text)860 exit()861 return none862 863 864def non_intelligent_split(files, chunk_size = 1000):865 extracted_content = []866 processed_files = []867 868 869 # For each files : stock the PDF, extract the Zips and convert the Doc & Docx to PDF870 try:871 not_duplicate = True872 for f in files:873 for p in processed_files:874 if (f[:f.rfind('.')] == p[:p.rfind('.')]):875 not_duplicate = False 876 if not_duplicate: 877 if f.endswith('.zip'):878 extracted_files = extract_zip(f)879 print(f"Those are my extracted files{extracted_files}")880 881 for doc in extracted_files:882 if doc.endswith('.doc') or doc.endswith('.docx'):883 processed_files.append(transform_to_pdf(doc))884 885 if doc.endswith('.pdf'):886 processed_files.append(doc)887 888 if f.endswith('.pdf'):889 processed_files.append(f)890 891 if f.endswith('.doc') or f.endswith('.docx'):892 processed_files.append(transform_to_pdf(f))893 894 except Exception as ex:895 print(f"Error occured while processing files : {ex}")896 897 # Extract content from each processed files898 try:899 for f in processed_files:900 print(f"my filename is : {f}")901 file = PdfReader(f)902 pdfNumberPages = len(file.pages)903 selectedText = ""904 905 for pdfPage in range(0, pdfNumberPages):906 load_page = file.get_page(pdfPage)907 text = load_page.extract_text()908 lines = text.split("\n")909 sizeOfLines = 0910 911 for index, line in enumerate(lines):912 sizeOfLines += len(line)913 selectedText += " " + line914 if sizeOfLines >= chunk_size:915 textContent = (f"Page {str(pdfPage)} : {selectedText}")916 extracted_content.append([f, textContent])917 sizeOfLines = 0918 selectedText = ""919 920 textContent = (f"Page {str(pdfNumberPages)} : {selectedText}")921 extracted_content.append([f, textContent])922 except Exception as ex:923 print(f"Error occured while extracting content from processed files : {ex}")924 925 df = pd.DataFrame()926 for content in extracted_content:927 filename = content[0]928 text = content[1]929 930 doc_data = {'Filename': filename[filename.rfind("/")+1:], 'Content': text}931 932 df = pd.concat([df, pd.DataFrame([doc_data])], ignore_index=True)933 934 df.to_excel("dataframe_keywords.xlsx", index=False)935 936 return "dataframe_keywords.xlsx"