andrewammann/AddressScrap_API_5

1from fastapi import FastAPI2import uvicorn3 4 5 6import pandas as pd7import numpy as np8import requests9from urllib.parse import urlparse, quote10import re11from bs4 import BeautifulSoup12import time13from joblib import Parallel, delayed14from nltk import ngrams15from googlesearch import search16 17 18app = FastAPI()19 20 21#Endpoints22#Root endpoints23@app.get("/")24def root():25    return {"API": "Adress Scrap"}26 27def normalize_string(string):28    normalized_string = string.lower()29    normalized_string = re.sub(r'[^\w\s]', '', normalized_string)30    31    return normalized_string32 33 34def jaccard_similarity(string1, string2,n = 2, normalize=True):35    try:36        if normalize:37           string1,string2= normalize_string(string1),normalize_string(string2)38           39        grams1 = set(ngrams(string1, n))40        grams2 = set(ngrams(string2, n))41        similarity = len(grams1.intersection(grams2)) / len(grams1.union(grams2))42    except:43        similarity=044        45    if string2=='did not extract address':46        similarity=047        48    return similarity49 50def jaccard_sim_split_word_number(string1,string2):51    numbers1 = ' '.join(re.findall(r'\d+', string1))52    words1 = ' '.join(re.findall(r'\b[A-Za-z]+\b', string1))53    54    numbers2 = ' '.join(re.findall(r'\d+', string2))55    words2 = ' '.join(re.findall(r'\b[A-Za-z]+\b', string2))   56    57    number_similarity=jaccard_similarity(numbers1,numbers2)58    words_similarity=jaccard_similarity(words1,words2)59    return (number_similarity+words_similarity)/260 61def extract_website_domain(url):62    parsed_url = urlparse(url)63    return parsed_url.netloc64 65 66def google_address(address):     67    all_data=[i for i in search(address, ssl_verify=False, advanced=True,68                                num_results=11)]69    70    71    df=pd.DataFrame({'Title':[i.title for i in all_data],72                     'Link':[i.url for i in all_data],73                     'Description':[i.description for i in all_data],})74    75    df=df.query("Title==Title")76    df['Link']=df['Link'].str.replace('/www.','https://www.')77    78    # df['Description']=df['Description'].bfill()79    df['Address Output']=df['Title'].str.extract(r'(.+? \d{5})').fillna("**DID NOT EXTRACT ADDRESS**")80    81    df['Link']=[i[7:i.find('&sa=')] for i in df['Link']]82    df['Website'] = df['Link'].apply(extract_website_domain)83    84    df['Square Footage']=df['Description'].str.extract(r"((\d+) Square Feet|(\d+) sq. ft.|(\d+) sqft|(\d+) Sq. Ft.|(\d+) sq|(\d+(?:,\d+)?) Sq\. Ft\.|(\d+(?:,\d+)?) sq)")[0]85    try:86        df['Square Footage']=df['Square Footage'].replace({',':''},regex=True).str.replace(r'\D', '')87    except:88        pass89    df['Beds']=df['Description'].replace({'-':' ','total':''},regex=True).str.extract(r"(\d+) bed")90    91    92    df['Baths']=df['Description'].replace({'-':' ','total':''},regex=True).str.extract(r"((\d+) bath|(\d+(?:\.\d+)?) bath)")[0]93    df['Baths']=df['Baths'].str.extract(r'([\d.]+)').astype(float)94    95    df['Year Built']=df['Description'].str.extract(r"built in (\d{4})")96    97    df['Match Percent']=[jaccard_sim_split_word_number(address,i)*100 for i in df['Address Output']]98    df['Google Search Result']=[*range(1,df.shape[0]+1)]99   100    # df_final=df[df['Address Output'].notnull()]101    # df_final=df_final[(df_final['Address Output'].str.contains(str(address_number))) & (df_final['Address Output'].str.contains(str(address_zip)))]102 103    df.insert(0,'Address Input',address)104    105    return df106 107   108 109@app.get('/Address_Scrap')110async def predict(address: str):111    try: 112        results= google_address(address)113        results=results[['Address Input', 'Address Output','Match Percent','Website','Square Footage', 'Beds', 'Baths', 'Year Built',114                     'Link','Google Search Result', 'Description' ]]115    except:116        results= pd.DataFrame({'Address Input':[address]})117        118    return results.to_json()119   120