Neprox/reddit-dashboard

1import os2import hopsworks3import pandas as pd4import streamlit as st5import seaborn as sns6import matplotlib.pyplot as plt7from warnings import warn8 9is_local=False10if is_local:11    from dotenv import load_dotenv12    load_dotenv()13 14MODEL_VERSION=2215 16@st.experimental_memo17def load_data():18    project = hopsworks.login()19    fs = project.get_feature_store()20 21    try:22        posts_fg = fs.get_feature_group("reddit_posts", version=os.getenv("POSTS_FG_VERSION", default=1))23        users_fg = fs.get_feature_group("reddit_users", version=os.getenv("USERS_FG_VERSION", default=1))24        subreddits_fg = fs.get_feature_group("reddit_subreddits", version=os.getenv("SUBREDDITS_FG_VERSION", default=1))25        full_join = posts_fg.select(features=["post_id", "snapshot_time", "num_likes", "upvote_ratio"]).join(26                            users_fg.select(features=["user_id", "snapshot_time"]), on=["user_id", "snapshot_time"]).join(27                                subreddits_fg.select(features=["subreddit_id", "snapshot_time"]), on=["subreddit_id", "snapshot_time"])28        df = full_join.read()29    except Exception as e:30        warn("Could not load data from feature store (most likely due to Port issues with Hopsworks). Trying to load the data from the model registry instead. Full exception:")31        warn(str(e))32        df = None33 34    # Load model including the generated images and evaluation scores35    mr = project.get_model_registry()36    model_hsfs = mr.get_model("reddit_predict", version=MODEL_VERSION)37    model_dir = model_hsfs.download()38    print("Model directory: {}".format(model_dir))39 40    metric_rows = {}41    metrics_avail = [m.replace("_likes","") for m in model_hsfs.training_metrics if "_likes" in m]42    for target in ["likes", "upvote_ratio"]:43        metric_rows[target] = []44        for metric in metrics_avail:45            metric_rows[target].append(model_hsfs.training_metrics[f"{metric}_{target}"])46    df_metrics = pd.DataFrame(metric_rows, index=metrics_avail)47    48    if df is None:49        try:50            df = pd.read_pickle(os.path.join(model_dir, "df_dashboard.pkl"))51        except:52            warn("Failed to load data from both the feature store and the model directory. Please upload the data to the model directory manually.")53 54    plots = {55        "predictions": plt.imread(f"{model_dir}/prediction_error.png"),56        "predictions_logscale": plt.imread(f"{model_dir}/prediction_error_logscale.png"),57        "confusion_matrix": plt.imread(f"{model_dir}/confusion_matrix.png"),58        "shap_numlikes": plt.imread(f"{model_dir}/shap_summary_plot_num_likes.png"),59        "shap_upvote_ratio": plt.imread(f"{model_dir}/shap_summary_plot_upvote_ratio.png"),60        "shap_numlikes_compact": plt.imread(f"{model_dir}/shap_summary_plot_compact.png")61    }62 63    return df, plots, df_metrics64 65 66df, plots, df_metrics = load_data()67 68if df is None:69    st.error("Could not load data from feature store or model directory as Huggingface has compatibility issues with parts of the data read API from Hopsworks.")70    st.stop()71 72# create a distribution plot of the number of likes using seaborn73st.title("Like It or Not")74st.markdown("This is the dashboard for the Like It Or Not model that predict the number of likes and the upvote ratio that a Reddit post is going to get.")75 76# Data stats77st.markdown("## Data Statistics")78col1, col2, col3 = st.columns(3)79col1.metric("Unqiue Posts", str(df["post_id"].nunique()))80col2.metric("Unique Users", str(df["user_id"].nunique()))81col3.metric("Unique Subreddits", str(df["subreddit_id"].nunique()))82 83# Distribution of the target variables84col1, col2 = st.columns(2)85col1.markdown("### Distribution of Number of Likes")86col2.markdown("### Distribution of Upvote Ratio")87col1, col2 = st.columns(2)88fig, ax = plt.subplots()89sns.histplot(df["num_likes"], ax=ax)90ax.set_ylabel("Number of posts")91ax.set_xlabel("Number of likes (log scale)")92ax.set_xscale("log")93plt.tight_layout()94col1.pyplot(fig)95 96fig2, ax = plt.subplots()97sns.distplot(df["upvote_ratio"], ax=ax, kde=False)98ax.set_ylabel("Number of posts")99plt.tight_layout()100col2.pyplot(fig2)101 102# Performance metrics103st.markdown("## Performance Metrics")104st.markdown("The model achieved the below scores on the test set. Please keep the effect of the sample weights in mind as explained in the Github repository. These reduce for example the R2 score from 0.75 to roughly 0.05. However, despite these low scores, the model is more useful in practice as it provides a meaningful lower bound estimate of the likes to be received as opposed to overestimating every post by up to 1500")105st.dataframe(df_metrics)106 107# Prediction error plots108st.markdown("## Prediction Error Plots")109st.markdown("The green line indicates the perfect prediction while the blue lines show point densities. Every point represents a prediction. The model is optimized for the number of likes and provides an estimate for the minimum number of likes expected. The upvote ratio does not perform well and would profit from dedicated modeling with another objective function if it is important.")110st.markdown("### Linear Scale")111st.image(plots["predictions"])112st.markdown("### Log Scale")113st.image(plots["predictions_logscale"])114 115# Confusion matrix116st.markdown("## Confusion Matrix")117st.markdown("After mapping the predicted number of likes to categories, the following confusion matrix can be obtained:")118st.image(plots["confusion_matrix"])119 120# Shap plots121st.markdown("## Shap Evaluation")122st.markdown("Shap values are an approach to machine learning explainability where the magnitude and kind of impact (positive / negative) of all features is computed." +123"Below, you see a beeswarm plot obtained on the predictions on the test data where every point represents a sample, its color tells if the feature had a high or low value " +124"and its position tells if the feature had a positive or negative impact on the prediction.")125st.image(plots["shap_numlikes"])126 127st.markdown("In addition, it is possible to sum up and average the absolute impact of all features over all samples. " +128            "The result can be interpreted as the feature importance. For the embedding features, we summed the values of the individual dimensions.")129st.image(plots["shap_numlikes_compact"])130