mattritchey/AddressScrap2

1import streamlit as st2import pandas as pd3import numpy as np4import requests5from urllib.parse import urlparse, quote6import re7from bs4 import BeautifulSoup8import time9from joblib import Parallel, delayed10from nltk import ngrams11 12@st.cache_data13def convert_df(df):14    return df.to_csv()15 16def normalize_string(string):17    normalized_string = string.lower()18    normalized_string = re.sub(r'[^\w\s]', '', normalized_string)19    20    return normalized_string21 22 23def jaccard_similarity(string1, string2,n = 2, normalize=True):24    try:25        if normalize:26           string1,string2= normalize_string(string1),normalize_string(string2)27           28        grams1 = set(ngrams(string1, n))29        grams2 = set(ngrams(string2, n))30        similarity = len(grams1.intersection(grams2)) / len(grams1.union(grams2))31    except:32        similarity=033        34    if string2=='did not extract address':35        similarity=036        37    return similarity38 39def jaccard_sim_split_word_number(string1,string2):40    numbers1 = ' '.join(re.findall(r'\d+', string1))41    words1 = ' '.join(re.findall(r'\b[A-Za-z]+\b', string1))42    43    numbers2 = ' '.join(re.findall(r'\d+', string2))44    words2 = ' '.join(re.findall(r'\b[A-Za-z]+\b', string2))   45    46    number_similarity=jaccard_similarity(numbers1,numbers2)47    words_similarity=jaccard_similarity(words1,words2)48    return (number_similarity+words_similarity)/249 50def extract_website_domain(url):51    parsed_url = urlparse(url)52    return parsed_url.netloc53 54 55def google_address(address):     56    # address_number = re.findall(r'\b\d+\b', address)[0]57    # address_zip =re.search(r'(\d{5})$', address).group()[:2]58    59    search_query = quote(address)60    url=f'https://www.google.com/search?q={search_query}'61    response = requests.get(url)62    soup = BeautifulSoup(response.content, "html.parser")63    64    texts_links = []65    for link in soup.find_all("a"):66        t,l=link.get_text(), link.get("href")67        if (l[:11]=='/url?q=http') and (len(t)>20 ):68            texts_links.append((t,l))69    70    text = soup.get_text()71    72    texts_links_des=[]73    for i,t_l in enumerate(texts_links):74        start=text.find(texts_links[i][0][:50])75        try:76            end=text.find(texts_links[i+1][0][:50])77        except:78            end=text.find('Related searches')79            80        description=text[start:end]81        texts_links_des.append((t_l[0],t_l[1],description))82    83    df=pd.DataFrame(texts_links_des,columns=['Title','Link','Description'])84    df['Description']=df['Description'].bfill()85    df['Address Output']=df['Title'].str.extract(r'(.+? \d{5})').fillna("**DID NOT EXTRACT ADDRESS**")86    87    df['Link']=[i[7:i.find('&sa=')] for i in df['Link']]88    df['Website'] = df['Link'].apply(extract_website_domain)89    90    df['Square Footage']=df['Description'].str.extract(r"((\d+) Square Feet|(\d+) sq. ft.|(\d+) sqft|(\d+) Sq. Ft.|(\d+) sq|(\d+(?:,\d+)?) Sq\. Ft\.|(\d+(?:,\d+)?) sq)")[0]91    try:92        df['Square Footage']=df['Square Footage'].replace({',':''},regex=True).str.replace(r'\D', '')93    except:94        pass95    df['Beds']=df['Description'].replace({'-':' ','total':''},regex=True).str.extract(r"(\d+) bed")96    97    98    df['Baths']=df['Description'].replace({'-':' ','total':''},regex=True).str.extract(r"((\d+) bath|(\d+(?:\.\d+)?) bath)")[0]99    df['Baths']=df['Baths'].str.extract(r'([\d.]+)').astype(float)100    101    df['Year Built']=df['Description'].str.extract(r"built in (\d{4})")102    103    df['Match Percent']=[jaccard_sim_split_word_number(address,i)*100 for i in df['Address Output']]104    df['Google Search Result']=[*range(1,df.shape[0]+1)]105   106    # df_final=df[df['Address Output'].notnull()]107    # df_final=df_final[(df_final['Address Output'].str.contains(str(address_number))) & (df_final['Address Output'].str.contains(str(address_zip)))]108 109    df.insert(0,'Address Input',address)110    111    return df112 113    114def catch_errors(addresses):115    try: 116        return google_address(addresses)117    except:118        return pd.DataFrame({'Address Input':[addresses]})119 120@st.cache_data121def process_multiple_address(addresses):122    results=Parallel(n_jobs=32, prefer="threads")(delayed(catch_errors)(i) for i in addresses)123    return results124    125  126st.set_page_config(layout="wide")127st.header("Google Data Scrap")  128 129address = st.sidebar.text_input("Single Address:", "190 Pebble Creek Dr Etna, OH 43062")130uploaded_file = st.sidebar.file_uploader("Upload Multiple Addresses:")131return_top_1 = st.sidebar.radio('Return Only Top Results',('No', 'Yes'))132match_percent = st.sidebar.selectbox('Address Match Percentage At Least:',(70, 80, 90, 100, 0))133return_sq = st.sidebar.radio('Return Only Results with Square Footage',('No', 'Yes'))134 135if uploaded_file is not None:136    try:137        df = pd.read_csv(uploaded_file)138    except:139        try:140            df = pd.read_excel(uploaded_file)141        except:142            df = pd.read_parquet(uploaded_file)143     144    address_cols=list(df.columns[:4])145    df[address_cols[-1]]=df[address_cols[-1]].astype(str).str[:5].astype(int).astype(str)146    df[address_cols[-1]]=df[address_cols[-1]].apply(lambda x: x.zfill(5))147    148    df['Address All']=df[address_cols[0]]+', '+df[address_cols[1]]+', '+df[address_cols[2]]+' '+df[address_cols[3]]149    150    results= process_multiple_address(df['Address All'].values)151    results=pd.concat(results).reset_index(drop=1)152    # results.index=results.index+1153 154else:    155    results=google_address(address).reset_index(drop=1)156    # results.index=results.index+1157 158    159results=results[['Address Input', 'Address Output','Match Percent','Website','Square Footage', 'Beds', 'Baths', 'Year Built',160 'Link','Google Search Result', 'Description' ]]161results=results.query(f"`Match Percent`>={match_percent}")162 163if return_sq=='Yes':164    results=results.query("`Square Footage`==`Square Footage`").reset_index(drop=1)165    # results.index=results.index+1166    167if return_top_1=='Yes':168    results=results.query("`Google Search Result`==1").reset_index(drop=1)169 170 171with st.container():172 173    st.dataframe(174        results,175        column_config={176       177            "Link": st.column_config.LinkColumn("Link"),178            'Match Percent': st.column_config.NumberColumn(format='%.2f %%'),179        },180        hide_index=True,181        # height=500,182        # width=500,183    )184 185csv2 = convert_df(results)186st.download_button(187    label="Download Results as CSV",188    data=csv2,189    file_name=f'download_scrap.csv',190    mime='text/csv')191 192 193st.markdown(""" <style>194#MainMenu {visibility: hidden;}195footer {visibility: hidden;}196</style> """, unsafe_allow_html=True)