OrganizedProgrammers/FastAPI_Neo4j
0
1import os2import requests3from contextlib import asynccontextmanager4from bs4 import BeautifulSoup5from fastapi import FastAPI, HTTPException6from neo4j import GraphDatabase, basic_auth7import google.generativeai as genai8import logging # Import logging module9 10# --- Logging Configuration ---11# Basic logger configuration to display INFO messages and above.12logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')13logger = logging.getLogger(__name__) # Create a logger instance for this module14 15# --- Environment Variable Configuration ---16NEO4J_URI = os.getenv("NEO4J_URI")17NEO4J_USER = os.getenv("NEO4J_USER")18NEO4J_PASSWORD = os.getenv("NEO4J_PASSWORD")19 20# Validation of essential configurations21if not NEO4J_URI or not NEO4J_USER or not NEO4J_PASSWORD:22 logger.critical("CRITICAL ERROR: NEO4J_URI, NEO4J_USER, and NEO4J_PASSWORD environment variables must be set.")23 24# --- Application Lifecycle (Startup/Shutdown) ---25@asynccontextmanager26async def lifespan(app: FastAPI):27 """Handles startup and shutdown events."""28 # Initialize Gemini Client29 logger.info("Initializing Gemini client...")30 if genai:31 try:32 # Assuming GEMINI_API_KEY is set in environment or loaded via settings33 api_key = os.getenv("GEMINI_API_KEY") or getattr(settings, "GEMINI_API_KEY", None)34 if not api_key:35 raise ValueError("GEMINI_API_KEY not found in environment or settings.")36 else:37 genai.configure(api_key=api_key)38 logger.info("Gemini client configured successfully.")39 except Exception as e:40 logger.error(f"Failed to configure Gemini client: {e}", exc_info=True)41 else:42 logger.warning("Gemini library not imported. Endpoints requiring Gemini will not work.")43 44 yield # API runs here45 46 # --- Shutdown ---47 logger.info("API shutting down...")48 logger.info("API shutdown complete.")49 50# Initialize FastAPI application51app = FastAPI(52 title="Neo4j Importer",53 description="API to fetch documents, summarize it with Gemini, and add it to Neo4j.",54 version="1.0.0",55 lifespan=lifespan56)57 58# --- Utility Functions (Adapted from your script) ---59 60def get_content(number: str, node_type: str) -> str:61 """Fetches raw HTML content from Arxiv or other sources."""62 redirect_links = {63 "Patent": f"https://patents.google.com/patent/{number}/en",64 "ResearchPaper": f"https://arxiv.org/abs/{number}"65 }66 67 url = redirect_links.get(node_type)68 if not url:69 logger.warning(f"Unknown node type: {node_type} for number {number}")70 return ""71 72 try:73 response = requests.get(url)74 response.raise_for_status() # Raises HTTPError for bad responses (4XX or 5XX)75 return response.content.decode('utf-8', errors='replace').replace("\n", "")76 except requests.exceptions.RequestException as e:77 logger.error(f"Request error for {node_type} number: {number} at URL {url}: {e}")78 return ""79 except Exception as e:80 logger.error(f"An unexpected error occurred in get_content for {number}: {e}")81 return ""82 83def extract_arxiv(rp_number: str, node_type: str = "ResearchPaper") -> dict:84 """Extracts information from an Arxiv research paper and generates a summary."""85 86 rp_data = {87 "document": f"Arxiv {rp_number}", # ID for the paper88 "title": "Error fetching content or content not found",89 "abstract": "Error fetching content or content not found",90 "summary": "Summary not yet generated" # Default summary91 }92 93 raw_content = get_content(rp_number, node_type)94 95 if not raw_content:96 logger.warning(f"No content fetched for Arxiv ID: {rp_number}")97 return rp_data # Returns default error data98 99 try:100 soup = BeautifulSoup(raw_content, 'html.parser')101 102 # Extract Title103 title_tag = soup.find('h1', class_='title')104 if title_tag and title_tag.find('span', class_='descriptor'):105 title_text = title_tag.find('span', class_='descriptor').next_sibling106 if title_text and isinstance(title_text, str):107 rp_data["title"] = title_text.strip()108 else: 109 rp_data["title"] = title_tag.get_text(separator=" ", strip=True).replace("Title:", "").strip()110 elif title_tag : # Fallback if the span descriptor is not there but h1.title exists111 rp_data["title"] = title_tag.get_text(separator=" ", strip=True).replace("Title:", "").strip()112 113 114 # Extract Abstract115 abstract_tag = soup.find('blockquote', class_='abstract')116 if abstract_tag:117 abstract_text = abstract_tag.get_text(strip=True)118 if abstract_text.lower().startswith('abstract'): # Check if "abstract" (case-insensitive) is at the beginning119 # Find the first occurrence of ':' after "abstract" or just remove "abstract" prefix120 prefix_end = abstract_text.lower().find('abstract') + len('abstract')121 if prefix_end < len(abstract_text) and abstract_text[prefix_end] == ':':122 prefix_end += 1 # Include the colon in removal123 abstract_text = abstract_text[prefix_end:].strip()124 rp_data["abstract"] = abstract_text125 126 # Mark if title or abstract are still not found127 if rp_data["title"] == "Error fetching content or content not found" and not title_tag:128 rp_data["title"] = "Title not found on page"129 if rp_data["abstract"] == "Error fetching content or content not found" and not abstract_tag:130 rp_data["abstract"] = "Abstract not found on page"131 132 except Exception as e:133 logger.error(f"Failed to parse content for Arxiv ID {rp_number}: {e}")134 135 # Generate summary with Gemini API if available and abstract exists136 if rp_data["abstract"] and \137 not rp_data["abstract"].startswith("Error fetching content") and \138 not rp_data["abstract"].startswith("Abstract not found"):139 140 prompt = f"""You are a 3GPP standardization expert. Summarize the key information in the provided document in simple technical English relevant to identifying potential Key Issues.141 Focus on challenges, gaps, or novel aspects.142 Here is the document: <document>{rp_data['abstract']}<document>"""143 144 try:145 model = genai.GenerativeModel("gemini-2.5-flash-preview-05-20")146 response = model.generate_content(prompt)147 148 rp_data["summary"] = response.text149 logger.info(f"Summary generated for Arxiv ID: {rp_number}")150 except Exception as e:151 logger.error(f"Error generating summary with Gemini for Arxiv ID {rp_number}: {e}")152 rp_data["summary"] = "Error generating summary (API failure)"153 else:154 rp_data["summary"] = "Summary not generated (Abstract unavailable or problematic)"155 return rp_data156 157def extract_google_patents(patent_number: str, node_type: str = "Patent"):158 """159 Extracts information from a Google Patents page with robust error handling.160 """161 # Initialize a dictionary with default error messages for consistency.162 patent_data = {163 "number": f"{patent_number}",164 "title": "Error fetching content or content not found",165 "description": "Error fetching content or content not found",166 "claim": "Error fetching content or content not found",167 "summary": "Summary not yet generated" # Default summary168 }169 170 # Use the generic get_content function to fetch the raw page content.171 raw_content = get_content(patent_number, node_type)172 173 if not raw_content:174 logger.warning(f"No content fetched for Patent ID: {patent_number}")175 return patent_data # Return the dictionary with default error messages.176 177 try:178 # Let BeautifulSoup handle the decoding from raw bytes.179 soup = BeautifulSoup(raw_content, 'html.parser')180 181 # --- Extract Title ---182 title_tag = soup.find('meta', attrs={'name': 'DC.title'})183 if title_tag and title_tag.get('content'):184 patent_data["title"] = title_tag['content'].strip()185 else:186 # Fallback to finding the title in an <h1> tag.187 title_h1 = soup.find('h1', id='title')188 if title_h1:189 patent_data["title"] = title_h1.get_text(strip=True)190 191 # --- Extract Description ---192 description_section = soup.find('section', itemprop='description')193 if description_section:194 # Remove unnecessary nested spans to clean the output.195 for src_text in description_section.find_all('span', class_='google-src-text'):196 src_text.decompose()197 patent_data["description"] = description_section.get_text(separator=' ', strip=True)198 199 # --- Extract Claims ---200 claims_section = soup.find('section', itemprop='claims')201 if claims_section:202 # Remove unnecessary nested spans here as well.203 for src_text in claims_section.find_all('span', class_='google-src-text'):204 src_text.decompose()205 patent_data["claim"] = claims_section.get_text(separator=' ', strip=True)206 207 # Update status message if specific sections were not found on the page.208 if patent_data["title"] == "Error fetching content or content not found":209 patent_data["title"] = "Title not found on page"210 if patent_data["description"] == "Error fetching content or content not found":211 patent_data["description"] = "Description not found on page"212 if patent_data["claim"] == "Error fetching content or content not found":213 patent_data["claim"] = "Claim not found on page"214 215 except Exception as e:216 # Catch any unexpected errors during the parsing process.217 logger.error(f"Failed to parse content for Patent ID {patent_number}: {e}")218 219 # Generate summary with Gemini API if available and abstract exists220 if patent_data["description"] and \221 not patent_data["description"].startswith("Error fetching content") and \222 not patent_data["description"].startswith("Description not found"):223 224 prompt = f"""You are a 3GPP standardization expert. Summarize the key information in the provided document in simple technical English relevant to identifying potential Key Issues.225 Focus on challenges, gaps, or novel aspects.226 Here is the document: <document>{patent_data['description']}<document>"""227 228 try:229 model = genai.GenerativeModel("gemini-2.5-flash-preview-05-20")230 response = model.generate_content(prompt)231 232 patent_data["summary"] = response.text233 logger.info(f"Summary generated for Patent ID: {patent_number}")234 except Exception as e:235 logger.error(f"Error generating summary with Gemini for Patent ID {patent_number}: {e}")236 patent_data["summary"] = "Error generating summary (API failure)"237 else:238 rp_data["summary"] = "Summary not generated (Description unavailable or problematic)"239 return patent_data240 241def add_nodes_to_neo4j(driver, data_list: list, node_type: str):242 """Adds a list of nodes to Neo4j in a single transaction."""243 if not data_list:244 logger.warning("No data provided to add_nodes_to_neo4j.")245 return 0246 247 query = (248 "UNWIND $data as properties "249 f"CREATE (n:{node_type}) "250 "SET n = properties"251 )252 253 # query = (254 # f"UNWIND $data as properties "255 # f"MERGE (n:{node_type} {{arxiv_id: properties.arxiv_id}}) " # Use MERGE for idempotency256 # f"ON CREATE SET n = properties "257 # f"ON MATCH SET n += properties" # Update properties if the node already exists258 # )259 260 try:261 with driver.session(database="neo4j") as session: # Specify database if not default262 result = session.execute_write(lambda tx: tx.run(query, data=data_list).consume())263 nodes_created = result.counters.nodes_created264 265 if nodes_created > 0:266 logger.info(f"{nodes_created} new {node_type} node(s) added successfully.")267 268 return nodes_created # Return the number of nodes actually created269 except Exception as e:270 logger.error(f"Neo4j Error - Failed to add/update {node_type} nodes: {e}")271 raise HTTPException(status_code=500, detail=f"Neo4j database error: {e}")272 273 274# --- FastAPI Endpoint ---275# API state check route276@app.get("/")277def read_root():278 return {"status": "ok"}279 280@app.post("/add_research_paper/{arxiv_id}", status_code=201) # 201 Created for successful creation281async def add_single_research_paper(arxiv_id: str):282 """283 Fetches a research paper from Arxiv by its ID, extracts information,284 generates a summary, and adds/updates it as a 'ResearchPaper' node in Neo4j.285 """286 node_type = "ResearchPaper"287 logger.info(f"Processing request for Arxiv ID: {arxiv_id}")288 289 if not NEO4J_URI or not NEO4J_USER or not NEO4J_PASSWORD:290 logger.error("Neo4j database connection details are not configured on the server.")291 raise HTTPException(status_code=500, detail="Neo4j database connection details are not configured on the server.")292 293 # Step 1: Extract paper data294 paper_data = extract_arxiv(arxiv_id, node_type)295 296 if paper_data["title"].startswith("Error fetching content") or paper_data["title"] == "Title not found on page":297 logger.warning(f"Could not fetch or parse content for Arxiv ID {arxiv_id}. Title: {paper_data['title']}")298 raise HTTPException(status_code=404, detail=f"Could not fetch or parse content for Arxiv ID {arxiv_id}. Title: {paper_data['title']}")299 300 # Step 2: Add/Update in Neo4j301 driver_instance = None # Initialize for the finally block302 try:303 auth_token = basic_auth(NEO4J_USER, NEO4J_PASSWORD)304 driver_instance = GraphDatabase.driver(NEO4J_URI, auth=auth_token)305 driver_instance.verify_connectivity()306 logger.info("Successfully connected to Neo4j.")307 308 nodes_created_count = add_nodes_to_neo4j(driver_instance, [paper_data], node_type)309 310 if nodes_created_count > 0 :311 logger.info(f"Research paper {arxiv_id} was successfully added to Neo4j.")312 status_code_response = 201 # Created313 314 # Note: FastAPI uses the status_code from the decorator or HTTPException.315 # This custom status_code_response is for the JSON body if needed, but the actual HTTP response status316 # will be 201 (from decorator) unless an HTTPException overrides it or we change the decorator based on logic.317 # For simplicity here, we'll return it in the body and let the decorator's 201 stand if no error.318 # A more advanced setup might change the response status dynamically.319 320 return {"data": paper_data}321 322 except HTTPException as e: # Re-raise HTTPExceptions323 logger.error(f"HTTPException during Neo4j operation for {arxiv_id}: {e.detail}")324 raise e325 except Exception as e:326 logger.error(f"An unexpected error occurred during Neo4j operation for {arxiv_id}: {e}", exc_info=True)327 raise HTTPException(status_code=500, detail=f"An unexpected server error occurred: {e}")328 finally:329 if driver_instance:330 driver_instance.close()331 logger.info("Neo4j connection closed.")332 333 334@app.post("/add_patent/{patent_id}", status_code=201) # 201 Created for successful creation335async def add_single_patent(patent_id: str):336 """337 Fetches a patent from Google Patents by its ID, extracts information,338 generates a summary, and adds/updates it as a 'Patent' node in Neo4j.339 """340 node_type = "Patent"341 logger.info(f"Processing request for Patent ID: {patent_id}")342 343 if not NEO4J_URI or not NEO4J_USER or not NEO4J_PASSWORD:344 logger.error("Neo4j database connection details are not configured on the server.")345 raise HTTPException(status_code=500, detail="Neo4j database connection details are not configured on the server.")346 347 # Step 1: Extract patent data348 patent_data = extract_google_patents(patent_id, node_type)349 350 if patent_data["title"].startswith("Error fetching content") or patent_data["title"] == "Title not found on page":351 logger.warning(f"Could not fetch or parse content for Patent ID {patent_id}. Title: {patent_data['title']}")352 raise HTTPException(status_code=404, detail=f"Could not fetch or parse content for Patent ID {patent_id}. Title: {patent_data['title']}")353 354 # Step 2: Add/Update in Neo4j355 driver_instance = None # Initialize for the finally block356 try:357 auth_token = basic_auth(NEO4J_USER, NEO4J_PASSWORD)358 driver_instance = GraphDatabase.driver(NEO4J_URI, auth=auth_token)359 driver_instance.verify_connectivity()360 logger.info("Successfully connected to Neo4j.")361 362 nodes_created_count = add_nodes_to_neo4j(driver_instance, [patent_data], node_type)363 364 if nodes_created_count > 0 :365 logger.info(f"Patent {patent_id} was successfully added to Neo4j.")366 status_code_response = 201 # Created367 368 # Note: FastAPI uses the status_code from the decorator or HTTPException.369 # This custom status_code_response is for the JSON body if needed, but the actual HTTP response status370 # will be 201 (from decorator) unless an HTTPException overrides it or we change the decorator based on logic.371 # For simplicity here, we'll return it in the body and let the decorator's 201 stand if no error.372 # A more advanced setup might change the response status dynamically.373 374 return {"data": patent_data}375 376 except HTTPException as e: # Re-raise HTTPExceptions377 logger.error(f"HTTPException during Neo4j operation for {patent_id}: {e.detail}")378 raise e379 except Exception as e:380 logger.error(f"An unexpected error occurred during Neo4j operation for {patent_id}: {e}", exc_info=True)381 raise HTTPException(status_code=500, detail=f"An unexpected server error occurred: {e}")382 finally:383 if driver_instance:384 driver_instance.close()385 logger.info("Neo4j connection closed.")