HuggingFaceTB/inspect_web_clusters

1import streamlit as st2from datasets import load_dataset3import os 4 5HF_TOKEN = os.environ.get("HF_TOKEN", None)6 7st.set_page_config(page_title="Web Clusters inspection", layout="wide")8st.title("Web clusters inspection")9 10st.markdown("""11We clustered 100k web samples using [text-clustering](https://github.com/huggingface/text-clustering). 12 13Our approach involved prompting Mixtral to evaluate whether the topics in each cluster are educational or could be considered college material using a score from 1 to 10. \14Technically, we provide it with 10 random examples from the cluster in the prompt and ask it to judge their topics.15 16Additionally, the model was tasked with finding the topic of each cluster (based on the 10 random examples). 17""")18 19 20@st.cache_data21def load_data(min_score=1, max_score=10, show_special=False):22    # HuggingFaceTB/FW_clusters_free_topics23    ds = load_dataset("HuggingFaceTB/FW_clusters_100k_145_topics", split="train", token=HF_TOKEN, num_proc=2)24    def filter_func(x):25        try:26            score = int(x['educational_score'])27            value = False if show_special else min_score <= score <= max_score28            return value29        except (ValueError, TypeError):30            # Return True if show_special is checked and educational_score is None or ''31            return show_special32 33    ds = ds.filter(filter_func)34    return ds35 36st.subheader("Cluster information")37col_1, col_2, col_3 = st.columns(3)38with col_1:39    show_special = st.checkbox('Show only clusters with undefined educational score', False)40with col_2:41    min_value = st.slider('Select minimum educational score', 1, 10, 1, key='min_score')42with col_3:43    max_value = st.slider('Select maximum educational score', 1, 10, 10, key='max_score')44    45# Load data based on slider values and checkbox status46ds = load_data(min_value, max_value, show_special)47categories = list(set(ds["category"]))48selected_category = st.selectbox("Select a topic", categories)49selected_cluster = ds.filter(lambda x: x['category'] == selected_category)50 51# Select sample index52n_samples = len(selected_cluster)53if n_samples > 0:54    col_1, col_2 = st.columns(2)55    with col_1:56        index_cluster = st.number_input(f"Found {len(selected_cluster)} clusters, choose one",  min_value=0, max_value=len(selected_cluster)-1, value=0, step=1)57 58    files = selected_cluster[index_cluster]["examples"]59 60    with col_2:61        index_example = st.number_input(f"Found {len(files)} files in the cluster, choose one",  min_value=0, max_value=len(files)-1, value=0, step=1)62 63    sample = files[index_example]64    st.markdown(f"**Educational score of the cluster**: {selected_cluster[index_cluster]['educational_score']}")65    st.markdown(sample)66else:67    st.markdown("No files found, change the cluster.")