Array3022/PubMedTM

1import streamlit as st2import pandas as pd3import datetime4from top2vec import Top2Vec5from matplotlib import pyplot as plt6 7st.set_page_config(layout='wide')8 9def create_frequency_dataframe(dataframe, default, specific, year_start=1976, year_stop=2000):10    '''Produces a dataframe with the number of articles in each topic in a each year.11    12    Parameters13        dataframe: Pandas dataframe containing the articles, abstracts, and dates.14        default: String indicating which topics to include in the dataframe. Selected by user.15        specific: String of additional topic numbers to include in the dataframe. Selected by user. Added to default.16        year_start: Integer indicating the first year to include in the dataframe. Selected by user.17        year_stop: Integer indicating the last year to include in the dataframe. Selected by user.18        19    Returns20        date_df: Pandas dataframe with the number of articles in each topic in a each year.21        topic_columns: List of topic numbers included in the dataframe.22    '''23    topic_sizes, topic_nums = model.get_topic_sizes(reduced=True)24    topic_columns = []25    if default == 'Top 5':26        topic_columns = topic_nums[:5]27    elif default == 'Top 10':28        topic_columns = topic_nums[:10]29    elif default == 'Bottom 5':30        topic_columns = topic_nums[-5:]31    elif default == 'Bottom 10':32        topic_columns = topic_nums[-10:]33 34    topic_columns= list(topic_columns) + [int(i) for i in specific.split() if len(i) > 0] # Break input text into list of ints then merge lists35    36    # Format df for number of articles in each topic in a given year37    date_df = pd.DataFrame(index=range(year_start, year_stop + 1), columns=topic_columns)38    for col in date_df.columns:39        date_df[col].values[:] = 0  # Covert values from null to 040 41    #Year/index num correspondence42    # index_dict = {}43    # for i, year in enumerate(date_df.index.values):44    #     index_dict[year] = i45 46    # Get doc_ids for each requested topic47    for topic in topic_columns:48        _, document_ids = model.search_documents_by_topic(topic_num=topic, reduced=True, num_docs=topic_sizes[topic]) # Works because topic number corresponds to its order in the topic size list49        for id in document_ids:50            if dataframe.at[id, 'DP'] >= datetime.date(year_start,1,1).year and dataframe.at[id, 'DP'] < datetime.date(year_stop + 1,1,1).year:51                date_df.at[dataframe.at[id, 'DP'], topic] += 152 53    return date_df, topic_columns54 55def create_prevalence_dataframe(date_df):56    date_df = date_df.join(date_df.apply(sum, axis=1).reindex_like(date_df.index.to_series()).rename('Sums'))  # Sum the total number of articles in each year57    # Find the percentage of total number of articles represented by each topic in each year58    for i, row in zip(date_df.index.values, date_df['Sums']):59        if row != 0:60            date_df.loc[i, :] = (date_df.loc[i, :] / date_df.loc[i, 'Sums']) * 10061    date_df = date_df.drop(columns='Sums')62    return date_df63 64def create_plot(date_df, ylabel, title):65    fig, ax = plt.subplots(figsize=(10, 5))66    for topic in date_df.columns:67        ax.plot(date_df.index.values, date_df.loc[:, topic].values, label=f'Topic {topic}')68    ax.xaxis.set_major_locator(plt.MaxNLocator(integer=True)) # Only show whole numbers on x-axis69    ax.set_xlabel('Year')70    ax.set_ylabel(ylabel)71    ax.set_title(title)72    ax.legend(bbox_to_anchor=(1, 1))73    ax.grid(True)74    return fig75 76@st.cache_resource77def load_model():78    name = '2024-02-14_22_19_09_reduced__model_medline0[test,mincount_250,topicmergedelta_0.15,ngram_True,ngramvocab_args_250_.4_80M_npmi_connector].json'79    with open(name, 'rb') as f:80        return Top2Vec.load(f)81 82@st.cache_data83def load_data():84    df = pd.read_pickle('medline_0_df_nlp.pkl.bz2', compression='bz2')85    return df86 87# Notification of loading88start_container = st.empty()89if 'first' not in st.session_state:90    st.session_state['first'] = True91    92if st.session_state['first'] == True:93    start_container.text('Loading model and data... This may take two to three minutes on the initial run.')94    st.session_state['first'] = False95 96# Load the model and data97model = load_model()98df = load_data()99 100# Remove loading notification101start_container.empty()102 103# User Interface104with st.sidebar.form('Topic Prevalence', clear_on_submit=False):105    default = st.radio('Default Topics', ['Top 5', 'Top 10', 'Bottom 5', 'Bottom 10', 'None'])106    specific = st.text_input('Topic Numbers', help='''Enter a series of topic numbers to graph. 107                                                    The numbers should be separated by a space.108                                                    Topic numbers are assigned in order of decreasing frequency across the corpus.109                                                    In other words, Topic 0 is the most frequent topic, Topic 1 is the second most frequent, etc.''')110    year_start_tp, year_stop_tp = st.slider('Year Range', 111                                    min_value=1975, 112                                    max_value=2000, 113                                    value=[1975, 2000])114    submitted_ed = st.form_submit_button("Submit")115 116    st.markdown('### Glossary')117    st.markdown('**Proportion of Articles:** The percentage of the total number of articles displayed that was in a given topic.')118    st.markdown('**Topic Frequency**: The number of articles in a given topic.')119    120if submitted_ed:121    frequency_df, topics = create_frequency_dataframe(dataframe = df, 122                                                    default = default, 123                                                    specific = specific, 124                                                    year_start = year_start_tp, 125                                                    year_stop = year_stop_tp)126    frequency_plot = create_plot(frequency_df, 'Number of Articles in Topic', 'Topic Frequency by Year')127    prevalence_df = create_prevalence_dataframe(frequency_df)128    prevalence_plot = create_plot(prevalence_df, ylabel='Percent of all Articles (%)', title='Proportion of Articles in Topics by Year')129    130    # Display data131    col1, col2 = st.columns([3, 1])132    133    with col1:134        st.header('Proportion of Articles in Topics by Year - Graph')135        st.pyplot(prevalence_plot)136        st.divider()137 138        st.header('Proportion of Articles in Topics by Year - Table')139        st.dataframe(prevalence_df.T)140        st.divider()141    142        st.header('Topic Frequency by Year - Graph')143        st.pyplot(frequency_plot)144        st.divider()145        146        st.header('Topic Frequency by Year - Table')147        st.dataframe(frequency_df.T)148        st.divider()149    150    with col2:151        st.header('Topic Information')152        topic_words, _, topic_nums = model.get_topics(reduced=True)153        for topic in topics:154            st.write(f'Topic {topic}:', ', '.join(topic_words[topic][:10]))155            st.divider()156