dejanseo/CTR

1import pandas as pd2import numpy as np3import xgboost as xgb4import streamlit as st5import requests6from bs4 import BeautifulSoup7from gensim.models import FastText8import joblib9 10# Load the trained FastText model11try:12    fasttext_model = FastText.load('fasttext_model.bin')13except FileNotFoundError:14    st.error("The FastText model file was not found. Please ensure 'fasttext_model.bin' and its associated files are in the correct directory.")15    st.stop()16 17# Load the trained XGBoost model for the combined features18try:19    model = joblib.load('model.pkl')20except FileNotFoundError:21    st.error("The XGBoost model file was not found. Please ensure 'model.pkl' is in the correct directory.")22    st.stop()23 24def tokenize(text):25    if isinstance(text, str):26        return text.split()27    else:28        return []29 30def embed_text(text_series, fasttext_model):31    embeddings = []32    for text in text_series:33        tokens = tokenize(text)34        vectors = [fasttext_model.wv[token] for token in tokens if token in fasttext_model.wv]35        if vectors:36            embeddings.append(np.mean(vectors, axis=0))37        else:38            embeddings.append(np.zeros(fasttext_model.vector_size))39    return np.array(embeddings)40 41def preprocess_input(query, title, description, url, fasttext_model):42    query = str(query) if pd.notna(query) else ''43    title = str(title) if pd.notna(title) else ''44    description = str(description) if pd.notna(description) else ''45    url = str(url) if pd.notna(url) else ''46 47    query_ft = embed_text(pd.Series([query]), fasttext_model)48    title_ft = embed_text(pd.Series([title]), fasttext_model)49    description_ft = embed_text(pd.Series([description]), fasttext_model)50    url_ft = embed_text(pd.Series([url]), fasttext_model)51 52    combined_features = np.hstack([query_ft, title_ft, description_ft, url_ft])53 54    dmatrix = xgb.DMatrix(combined_features)55    return dmatrix56 57def extract_title_description(url):58    headers = {59        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.81 Safari/537.36'60    }61    try:62        response = requests.get(url, headers=headers)63        soup = BeautifulSoup(response.text, 'html.parser')64        title = soup.title.string if soup.title else 'No title found'65        description_tag = soup.find('meta', attrs={'name': 'description'})66        description = description_tag['content'] if description_tag else 'No description found'67        return title, description68    except Exception as e:69        return 'Error extracting title', 'Error extracting description'70 71def predict(query, title, description, url, fasttext_model):72    dmatrix = preprocess_input(query, title, description, url, fasttext_model)73    probability = model.predict(dmatrix, validate_features=False)[0]74    binary_prediction = int(probability >= 0.5)75    return binary_prediction, probability76 77# Streamlit interface78st.title('CTR Prediction Inference')79 80tab1, tab2, tab3 = st.tabs(["Single Entry", "Batch Entry", "A/B Test"])81 82with tab1:83    st.header('Single Entry Inference')84    85    query = st.text_input('Query')86    url = st.text_input('URL')87 88    if st.button('Predict'):89        title, description = extract_title_description(url)90        st.write(f'Extracted Title: {title}')91        st.write(f'Extracted Description: {description}')92        93        if query and url:94            binary_result, confidence = predict(query, title, description, url, fasttext_model)95            st.write(f'Predicted +/-: {binary_result}')96            st.write(f'Conf.: {confidence:.2%}')97            confidence_percentage = int(confidence * 100)98            st.progress(confidence_percentage)99        else:100            st.write('Please enter both a query and a URL.')101 102with tab2:103    st.header('Batch Entry Inference')104    105    uploaded_file = st.file_uploader("Upload CSV", type="csv")106    107    if uploaded_file is not None:108        df = pd.read_csv(uploaded_file)109        required_columns = ['Query', 'Title', 'Description', 'URL']110        111        if set(required_columns).issubset(df.columns):112            predictions = []113            confidences = []114            for index, row in df.iterrows():115                binary_result, confidence = predict(row['Query'], row['Title'], row['Description'], row['URL'], fasttext_model)116                predictions.append(binary_result)117                confidences.append(confidence)118            119            df['+/-'] = predictions120            df['Conf.'] = [f"{conf:.2%}" for conf in confidences]121            122            cols = ['+/-', 'Conf.'] + [col for col in df.columns if col not in ['+/-', 'Conf.']]123            df = df[cols]124            125            st.write(df)126            st.download_button("Download Predictions", df.to_csv(index=False), "predictions.csv")127        else:128            st.write('CSV must contain Query, Title, Description, and URL columns.')129 130with tab3:131    st.header('A/B Test Inference')132 133    query = st.text_input('Query for A/B Test')134    url = st.text_input('URL for A/B Test')135 136    if 'step' not in st.session_state:137        st.session_state.step = 0138 139    if st.button('Scrape A/B'):140        title_A, description_A = extract_title_description(url)141        st.session_state['title_A'] = title_A142        st.session_state['description_A'] = description_A143        st.session_state.step = 1144 145    if st.session_state.step == 1:146        title_B = st.text_input('Title B', value=st.session_state.get('title_A', ''))147        description_B = st.text_area('Description B', value=st.session_state.get('description_A', ''))148 149        if st.button('Predict A/B'):150            if query and url:151                binary_result_A, confidence_A = predict(query, st.session_state['title_A'], st.session_state['description_A'], url, fasttext_model)152                binary_result_B, confidence_B = predict(query, title_B, description_B, url, fasttext_model)153                154                st.write(f'Results for A: Predicted +/-: {binary_result_A}, Conf.: {confidence_A:.2%}')155                st.write(f'Results for B: Predicted +/-: {binary_result_B}, Conf.: {confidence_B:.2%}')156                157                if binary_result_A == 1 and binary_result_B == 0:158                    st.write("B is worse than A")159                elif binary_result_A == 0 and binary_result_B == 1:160                    st.write("B is better than A")161                else:162                    st.write("B is the same as A")163