Almaatla/Standard_Intelligence_Dev
0
1import os2import requests3from bs4 import BeautifulSoup4from urllib.parse import urljoin5import pandas as pd6import numpy as np7import zipfile8import textract9import gradio as gr10import shutil11from pypdf import PdfReader12 13def browse_folder(url):14 if url.lower().endswith(('docs', 'docs/')):15 return gr.update(choices=[])16 response = requests.get(url)17 response.raise_for_status() # This will raise an exception if there's an error18 19 soup = BeautifulSoup(response.text, 'html.parser')20 21 excel_links = [a['href'] + '/' for a in soup.find_all('a', href=True) if a['href'].startswith(url)]22 23 return gr.update(choices=excel_links)24 25 26 27def extract_statuses(url):28 # Send a GET request to the webpage29 response = requests.get(url)30 31 # Parse the webpage content32 soup = BeautifulSoup(response.content, 'html.parser')33 34 # Find all links in the webpage35 links = soup.find_all('a')36 37 # Identify and download the Excel file38 for link in links:39 href = link.get('href')40 if href and (href.endswith('.xls') or href.endswith('.xlsx')):41 excel_url = href if href.startswith('http') else url + href42 excel_response = requests.get(excel_url)43 file_name = 'guide_status.xlsx' #excel_url.split('/')[-1]44 45 # Save the file46 with open(file_name, 'wb') as f:47 f.write(excel_response.content)48 49 # Read the Excel file50 df = pd.read_excel(file_name)51 52 # Check if 'TDoc Status' column exists and extract unique statuses53 if 'TDoc Status' in df.columns:54 unique_statuses = df['TDoc Status'].unique().tolist()55 print(f'Downloaded {file_name} and extracted statuses: {unique_statuses}')56 57 58 if 'withdrawn' in unique_statuses:59 unique_statuses.remove('withdrawn')60 return gr.update(choices=unique_statuses, value=unique_statuses)61 else:62 print(f"'TDoc Status' column not found in {file_name}")63 return []64 65 66import os67import requests68from bs4 import BeautifulSoup69import pandas as pd70import gradio as gr71 72 73def scrape(url, folder_name, status_list, sorted_files, progress=gr.Progress()):74 filenames = []75 status_filenames = []76 df = pd.DataFrame() # Initialize df to ensure it's always defined77 excel_file = "guide_status.xlsx"78 79 print("Downloading zip files directly from the URL...")80 response = requests.get(url)81 soup = BeautifulSoup(response.content, 'html.parser')82 83 # Select all zip files84 zip_links = [a['href'] for a in soup.find_all('a', href=True) if a['href'].endswith('.zip') ]85 86 sorted_files_tab = []87 # Check if the user selected some filters88 if len(sorted_files) != 0:89 for link in zip_links:90 for file in sorted_files:91 if file in link:92 sorted_files_tab.append(link)93 94 if len(sorted_files_tab) != 0:95 zip_links = sorted_files_tab96 97 # Construct absolute URLs for zip files98 status_filenames = [url + link if not link.startswith('http') else link for link in zip_links]99 print(f"Filenames from URL: {status_filenames}")100 101 download_directory = folder_name102 if not os.path.exists(download_directory):103 os.makedirs(download_directory)104 105 pourcentss = 0.05106 107 108 109 # Proceed with downloading files110 for file_url in status_filenames:111 112 113 filename = os.path.basename(file_url)114 save_path = os.path.join(download_directory, filename)115 progress(pourcentss, desc='Downloading')116 pourcentss += 0.4 / max(len(status_filenames), 1) # Ensure non-zero division117 118 119 120 try:121 with requests.get(file_url, stream=True) as r:122 r.raise_for_status()123 with open(save_path, 'wb') as f:124 for chunk in r.iter_content(chunk_size=8192):125 f.write(chunk)126 except requests.exceptions.HTTPError as e:127 print(f"HTTP error occurred while downloading {file_url}: {e}")128 129 return True, len(status_filenames)130 131 132 133 134 135 136def extractZip(url):137 # Répertoire où les fichiers zip sont déjà téléchargés138 nom_extract = url.split("/")[-3] + "_extraction"139 if os.path.exists(nom_extract): 140 shutil.rmtree(nom_extract)141 extract_directory = nom_extract142 143 download_directory = url.split("/")[-3] + "_downloads"144 # Répertoire où le contenu des fichiers zip sera extrait145 146 # Extraire le contenu de tous les fichiers zip dans le répertoire de téléchargement147 for zip_file in os.listdir(download_directory):148 zip_path = os.path.join(download_directory, zip_file)149 # Vérifier si le fichier est un fichier zip150 if zip_file.endswith(".zip"):151 extract_dir = os.path.join(extract_directory, os.path.splitext(zip_file)[0]) # Supprimer l'extension .zip152 153 # Vérifier si le fichier zip existe154 if os.path.exists(zip_path):155 # Créer un répertoire pour extraire le contenu s'il n'existe pas156 if not os.path.exists(extract_dir):157 os.makedirs(extract_dir)158 159 # Extraire le contenu du fichier zip160 try:161 with zipfile.ZipFile(zip_path, 'r') as zip_ref:162 zip_ref.extractall(extract_dir)163 164 print(f"Extraction terminée pour {zip_file}")165 except:166 print(f"Erreur: Extraction {zip_file}")167 else:168 print(f"Fichier zip {zip_file} introuvable")169 170 print("Toutes les extractions sont terminées !")171 172 173def excel3gpp(url):174 response = requests.get(url)175 response.raise_for_status() # This will raise an exception if there's an error176 177 # Use BeautifulSoup to parse the HTML content178 soup = BeautifulSoup(response.text, 'html.parser')179 180 # Look for Excel file links; assuming they have .xlsx or .xls extensions181 excel_links = [a['href'] for a in soup.find_all('a', href=True) if a['href'].endswith(('.xlsx', '.xls'))]182 183 # Download the first Excel file found (if any)184 if excel_links:185 excel_url = excel_links[0] # Assuming you want the first Excel file186 if not excel_url.startswith('http'):187 excel_url = os.path.join(url, excel_url) # Handle relative URLs188 189 # Download the Excel file190 excel_response = requests.get(excel_url)191 excel_response.raise_for_status()192 193 # Define the path where you want to save the file194 # Replace 'path_to_save_directory' with your desired path195 196 # Write the content of the Excel file to a local file197 # Write the content of the Excel file to a local file named 'guide.xlsx'198 199 nom_guide = 'guide.xlsx' # Directly specify the filename200 if os.path.exists(nom_guide):201 os.remove(nom_guide)202 filepath = nom_guide203 204 205 with open(filepath, 'wb') as f:206 f.write(excel_response.content)207 print(f'Excel file downloaded and saved as: {filepath}')208 209 210 211def replace_line_breaks(text):212 return text.replace("\n", "/n")213 214def remod_text(text):215 return text.replace("/n", "\n")216 217def update_excel(data, excel_file, url):218 new_df_columns = ["URL", "File", "Type", "Title", "Source", "Related WIs", "Status", "Content"]219 temp_df = pd.DataFrame(data, columns=new_df_columns)220 221 try:222 # Check if the Excel file already exists and append data to it223 if os.path.exists(excel_file):224 old_df = pd.read_excel(excel_file)225 df = pd.concat([old_df, temp_df], axis=0, ignore_index=True)226 else:227 df = temp_df228 229 # Save the updated data back to the Excel file230 df.to_excel(excel_file, index=False)231 except Exception as e:232 print(f"Error updating Excel file: {e}")233 234def extractionPrincipale(url, excel_file=None, status_list=None, progress=gr.Progress()):235 nom_download = url.split("/")[-3] + "_downloads"236 if os.path.exists(nom_download): 237 shutil.rmtree(nom_download)238 folder_name = nom_download239 240 nom_status = url.split("/")[-3] + "_status.xlsx"241 if os.path.exists(nom_status):242 os.remove(nom_status)243 temp_excel = nom_status244 245 progress(0.0,desc='Downloading')246 247 #Sorting files, downloading only files which have the status selected by the user248 sorted_files = []249 250 try:251 guide_file = 'guide.xlsx'252 if os.path.exists(guide_file):253 dfStatus = pd.read_excel(guide_file)254 255 # Look if the user selected some filter status256 if len(dfStatus['TDoc Status'].unique().tolist()) != len (status_list):257 258 259 keys_statuses_filename = dfStatus['TDoc'].tolist()260 values_unique_statuses = dfStatus['TDoc Status'].tolist()261 262 doc_statuses = dict(zip(keys_statuses_filename, values_unique_statuses))263 for key in doc_statuses.keys():264 if doc_statuses[key] in status_list:265 sorted_files.append(key)266 267 print(sorted_files)268 except Exception as e:269 print(f"Not able to retrieve informations from 'guide.xlsx' ")270 271 result, count = scrape(url, folder_name, status_list, sorted_files)272 if result:273 print("Success")274 else:275 return(None)276 277 progress(0.4,desc='Extraction')278 extractZip(url)279 progress(0.5,desc='Extraction 2')280 excel3gpp(url)281 progress(0.6,desc='Creating Excel File')282 283 284 extract_directory = url.split("/")[-3] + "_extraction"285 TabCategories = ["URL", "File", "Title", "Source", "Related WIs", "Content"]286 categories = {287 "Other": TabCategories,288 "CR": TabCategories,289 "pCR": TabCategories,290 "LS": TabCategories,291 "WID": TabCategories,292 "SID": TabCategories,293 "DISCUSSION": TabCategories,294 "pdf": TabCategories,295 "ppt": TabCategories,296 "pptx": TabCategories297 }298 299 pourcents2=0.6300 data = []301 errors_count = 0302 processed_count = 0 # Counter for processed files303 304 pre_title_section = None305 306 try:307 df = pd.read_excel(temp_excel)308 except Exception as e:309 print(f"Initializing a new DataFrame because: {e}")310 df = pd.DataFrame(columns=["URL", "File", "Type", "Title", "Source", "Related WIs","Status", "Content"])311 312 for folder in os.listdir(extract_directory):313 folder_path = os.path.join(extract_directory, folder)314 if os.path.isdir(folder_path):315 for file in os.listdir(folder_path):316 progress(min(pourcents2,0.99),desc='Creating Excel File')317 pourcents2+=0.4/count318 319 320 if file == "__MACOSX":321 continue322 file_path = os.path.join(folder_path, file)323 if file.endswith((".pptx", ".ppt", ".pdf", ".docx", ".doc", ".DOCX")):324 try:325 text = textract.process(file_path).decode('utf-8')326 if file.endswith((".pdf")):327 pdfReader = PdfReader(file_path)328 except Exception as e:329 print(f"Error processing {file_path}: {e}")330 errors_count += 1331 continue332 333 cleaned_text_lines = text.split('\n')334 cleaned_text = '\n'.join([line.strip('|').strip() for line in cleaned_text_lines if line.strip()])335 336 title = ""337 debut = ""338 sections = cleaned_text.split("Title:")339 if len(sections) > 1:340 pre_title_section = sections[0].strip().split()341 title = sections[1].strip().split("\n")[0].strip()342 debut = sections[0].strip()343 344 category = "Other"345 if file.endswith(".pdf"):346 category = "pdf"347 elif file.endswith((".ppt", ".pptx")):348 category = "ppt" # assuming all ppt and pptx files go into the same category349 elif "CHANGE REQUEST" in debut:350 category = "CR"351 elif "Discussion" in title:352 category = "DISCUSSION"353 elif "WID" in title:354 category = "WID"355 elif "SID" in title:356 category = "SID"357 elif "LS" in title:358 category = "LS"359 elif pre_title_section and pre_title_section[-1] == 'pCR':360 category = "pCR"361 elif "Pseudo-CR" in title:362 category = "pCR"363 364 365 contenu = "" # This will hold the concatenated content for 'Contenu' column366 if category in categories:367 columns = categories[category]368 extracted_content = []369 if category == "CR":370 reason_for_change = ""371 summary_of_change = ""372 if len(sections) > 1:373 reason_for_change = sections[1].split("Reason for change", 1)[-1].split("Summary of change")[0].strip()374 summary_of_change = sections[1].split("Summary of change", 1)[-1].split("Consequences if not")[0].strip()375 extracted_content.append(f"Reason for change: {reason_for_change}")376 extracted_content.append(f"Summary of change: {summary_of_change}")377 elif category == "pCR":378 if len(sections) > 1:# Handle 'pCR' category-specific content extraction379 pcr_specific_content = sections[1].split("Introduction", 1)[-1].split("First Change")[0].strip()380 extracted_content.append(f"Introduction: {pcr_specific_content}")381 elif category == "LS":382 overall_review = ""383 if len(sections) > 1:384 overall_review = sections[1].split("Overall description", 1)[-1].strip()385 extracted_content.append(f"Overall review: {overall_review}")386 elif category in ["WID", "SID"]:387 objective = ""388 start_index = cleaned_text.find("Objective")389 end_index = cleaned_text.find("Expected Output and Time scale")390 if start_index != -1 and end_index != -1:391 objective = cleaned_text[start_index + len("Objective"):end_index].strip()392 extracted_content.append(f"Objective: {objective}")393 elif category == "DISCUSSION":394 Discussion = ""395 extracted_text = replace_line_breaks(cleaned_text)396 start_index_doc_for = extracted_text.find("Document for:")397 if start_index_doc_for != -1:398 start_index_word_after_doc_for = start_index_doc_for + len("Document for:")399 end_index_word_after_doc_for = start_index_word_after_doc_for + extracted_text[start_index_word_after_doc_for:].find("/n")400 word_after_doc_for = extracted_text[start_index_word_after_doc_for:end_index_word_after_doc_for].strip()401 result_intro = ''402 result_conclusion = ''403 result_info = ''404 if word_after_doc_for.lower() == "discussion":405 start_index_intro = extracted_text.find("Introduction")406 end_index_intro = extracted_text.find("Discussion", start_index_intro)407 408 intro_text = ""409 if start_index_intro != -1 and end_index_intro != -1:410 intro_text = extracted_text[start_index_intro + len("Introduction"):end_index_intro].strip()411 result_intro = remod_text(intro_text) # Convert back line breaks412 else:413 result_intro = "Introduction section not found."414 415 # Attempt to find "Conclusion"416 start_index_conclusion = extracted_text.find("Conclusion", end_index_intro)417 end_index_conclusion = extracted_text.find("Proposal", start_index_conclusion if start_index_conclusion != -1 else end_index_intro)418 419 conclusion_text = ""420 if start_index_conclusion != -1 and end_index_conclusion != -1:421 conclusion_text = extracted_text[start_index_conclusion + len("Conclusion"):end_index_conclusion].strip()422 result_conclusion = remod_text(conclusion_text)423 elif start_index_conclusion == -1: # Conclusion not found, look for Proposal directly424 start_index_proposal = extracted_text.find("Proposal", end_index_intro)425 if start_index_proposal != -1:426 end_index_proposal = len(extracted_text) # Assuming "Proposal" section goes till the end if present427 proposal_text = extracted_text[start_index_proposal + len("Proposal"):end_index_proposal].strip()428 result_conclusion = remod_text(proposal_text) # Using "Proposal" content as "Conclusion"429 else:430 result_conclusion = "Conclusion/Proposal section not found."431 else:432 # Handle case where "Conclusion" exists but no "Proposal" to mark its end433 conclusion_text = extracted_text[start_index_conclusion + len("Conclusion"):].strip()434 result_conclusion = remod_text(conclusion_text)435 Discussion=f"Introduction: {result_intro}\nConclusion/Proposal: {result_conclusion}"436 elif word_after_doc_for.lower() == "information":437 start_index_info = extracted_text.find(word_after_doc_for)438 if start_index_info != -1:439 info_to_end = extracted_text[start_index_info + len("Information"):].strip()440 result_info = remod_text(info_to_end)441 Discussion = f"Discussion:{result_info}"442 else:443 Discussion = "The word after 'Document for:' is not 'Discussion', 'DISCUSSION', 'Information', or 'INFORMATION'."444 else:445 Discussion = "The phrase 'Document for:' was not found."446 # Since DISCUSSION category handling requires more specific processing, adapt as necessary447 # Here's a simplified example448 discussion_details = Discussion449 extracted_content.append(discussion_details)450 451 elif category == "pdf":452 try:453 tabLine = []454 file = pdfReader455 pdfNumberPages = len(file.pages)456 words_limit = 1000457 for pdfPage in range(0, pdfNumberPages):458 459 load_page = file.get_page(pdfPage)460 text = load_page.extract_text()461 lines = text.split("\n")462 sizeOfLines = len(lines) - 1463 keyword = ["objective", "introduction", "summary", "scope"]464 465 for index, line in enumerate(lines):466 print(line)467 for key in keyword:468 line = line.lower()469 470 if key in line:471 print("Found keyword")472 lineBool = True473 lineIndex = index474 previousSelectedLines = []475 stringLength = 0476 linesForSelection = lines477 loadOnce = True478 selectedPdfPage = pdfPage479 480 while lineBool:481 print(lineIndex)482 if stringLength > words_limit or lineIndex < 0:483 lineBool = False484 else:485 if lineIndex == 0:486 print(f"Line index == 0")487 488 if pdfPage == 0:489 lineBool = False490 491 else:492 try:493 selectedPdfPage -= 1494 newLoad_page = file.get_page(selectedPdfPage)495 newText = newLoad_page.extract_text()496 newLines = newText.split("\n")497 linesForSelection = newLines498 print(f"len newLines{len(newLines)}")499 lineIndex = len(newLines) - 1500 except Exception as e:501 print(f"Loading previous PDF page failed")502 lineBool = False503 504 previousSelectedLines.append(linesForSelection[lineIndex])505 stringLength += len(linesForSelection[lineIndex])506 507 lineIndex -= 1508 previousSelectedLines = ' '.join(previousSelectedLines[::-1])509 510 lineBool = True511 lineIndex = index + 1512 nextSelectedLines = ""513 linesForSelection = lines514 loadOnce = True515 selectedPdfPage = pdfPage516 517 while lineBool:518 519 if len(nextSelectedLines.split()) > words_limit:520 lineBool = False521 else:522 if lineIndex > sizeOfLines:523 lineBool = False524 525 if pdfPage == pdfNumberPages - 1:526 lineBool = False527 528 else:529 try:530 selectedPdfPage += 1531 newLoad_page = file.get_page(selectedPdfPage)532 newText = newLoad_page.extract_text()533 newLines = newText.split("\n")534 linesForSelection = newLines535 lineIndex = 0536 except Exception as e:537 print(f"Loading next PDF page failed")538 lineBool = False539 else:540 nextSelectedLines += " " + linesForSelection[lineIndex]541 lineIndex += 1542 543 print(f"Previous Lines : {previousSelectedLines}")544 print(f"Next Lines : {nextSelectedLines}")545 selectedText = previousSelectedLines + ' ' + nextSelectedLines546 print(selectedText)547 tabLine.append([pdfPage, selectedText, key])548 print(f"Selected line in keywords is: {line}")549 550 for r in tabLine:551 extracted_content.append(f'PDF Page number {r[0]} extracted text from the KEYWORD {r[2]} : \n')552 extracted_content.append(''.join(r[1])) 553 except Exception as e:554 print(f"Error occured while extracting PDF content : {e}")555 # Add more categories as needed556 contenu = "\n".join(extracted_content)557 558 # Assuming 'source' needs to be filled from the guide.xlsx mapping559 # Placeholder for source value calculation560 source = "" # Update this with actual source determination logic561 RelatedWIs = ""562 status = ""563 data.append([url+ "/" + folder + '.zip', folder , category, title, source, RelatedWIs, status, contenu])564 565 guide_file = 'guide.xlsx'566 if os.path.exists(guide_file):567 # If guide.xlsx exists, proceed with operations that require it568 try:569 guide_df = pd.read_excel(guide_file, usecols=['Source', 'TDoc', 'Related WIs', 'TDoc Status'])570 # Continue with the operations that require guide.xlsx571 # For example, reading the file, processing the data, etc.572 tdoc_source_map = {row['TDoc']: row['Source'] for index, row in guide_df.iterrows()}573 tdoc_relatedWIs_map = {row['TDoc']: row['Related WIs'] for index, row in guide_df.iterrows()}574 tdoc_status_map = {row['TDoc']: row['TDoc Status'] for index, row in guide_df.iterrows()}575 # Update the 'Source' in your data based on matching 'Nom du fichier' with 'TDoc'576 for item in data:577 nom_du_fichier = item[1] # Assuming 'Nom du fichier' is the first item in your data list578 if nom_du_fichier in tdoc_source_map:579 item[4] = tdoc_source_map[nom_du_fichier] # Update the 'Source' field, assuming it's the fourth item580 item[5] = tdoc_relatedWIs_map[nom_du_fichier]581 item[6] = tdoc_status_map[nom_du_fichier]582 # Your code that depends on guide.xlsx goes here583 584 except Exception as e:585 print(f"An error occurred while processing {guide_file}: {e}")586 # Handle any errors that arise during processing587 else:588 print(f"File {guide_file} not found. Skipping operations that require this file.")589 # Since guide.xlsx is not found, skip the related operations590 591 592 593 594 595 processed_count += 1596 597 # Check if it's time to update the Excel file598 if processed_count % 20 == 0:599 update_excel(data, temp_excel, url)600 print(f"Updated after processing {processed_count} files.")601 data = [] # Clear the data list after updating602 603 if data:604 # This final call ensures that any remaining data is processed and saved.605 update_excel(data, temp_excel, url)606 print(f"Final update after processing all files.")607 608 file_name = temp_excel609 # Save the updated DataFrame to Excel610 return file_name611 612