nazneen/error-analysis
4
1## LIBRARIES ###2## Data3import numpy as np4import pandas as pd5import torch6import math7from tqdm import tqdm8from math import floor9from collections import defaultdict10from transformers import AutoTokenizer11pd.options.display.float_format = '${:,.2f}'.format12 13# Analysis14# from gensim.models.doc2vec import Doc2Vec15# from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score16import nltk17from nltk.cluster import KMeansClusterer18import scipy.spatial.distance as sdist19from scipy.spatial import distance_matrix20# nltk.download('punkt') #make sure that punkt is downloaded21 22# App & Visualization23import streamlit as st24import altair as alt25import plotly.graph_objects as go26from streamlit_vega_lite import altair_component27 28 29 30# utils31from random import sample32from error_analysis import utils as ut33 34 35def down_samp(embedding):36 """Down sample a data frame for altiar visualization """37 # total number of positive and negative sentiments in the class38 #embedding = embedding.groupby('slice').apply(lambda x: x.sample(frac=0.3))39 total_size = embedding.groupby(['slice','label'], as_index=False).count()40 41 user_data = 042 # if 'Your Sentences' in str(total_size['slice']):43 # tmp = embedding.groupby(['slice'], as_index=False).count()44 # val = int(tmp[tmp['slice'] == "Your Sentences"]['source'])45 # user_data = val46 47 max_sample = total_size.groupby('slice').max()['content']48 49 # # down sample to meeting altair's max values50 # # but keep the proportional representation of groups51 down_samp = 1/(sum(max_sample.astype(float))/(1000-user_data))52 53 max_samp = max_sample.apply(lambda x: floor(x*down_samp)).astype(int).to_dict()54 max_samp['Your Sentences'] = user_data55 56 # # sample down for each group in the data frame57 embedding = embedding.groupby('slice').apply(lambda x: x.sample(n=max_samp.get(x.name))).reset_index(drop=True)58 59 # # order the embedding60 return(embedding)61 62 63def data_comparison(df):64 selection = alt.selection_multi(fields=['cluster','label'])65 color = alt.condition(alt.datum.slice == 'high-loss', alt.Color('cluster:N', scale = alt.Scale(domain=df.cluster.unique().tolist())), alt.value("lightgray"))66 opacity = alt.condition(selection, alt.value(0.7), alt.value(0.25))67 68 # basic chart69 scatter = alt.Chart(df).mark_point(size=100, filled=True).encode(70 x=alt.X('x:Q', axis=None),71 y=alt.Y('y:Q', axis=None),72 color=color,73 shape=alt.Shape('label:N', scale=alt.Scale(range=['circle', 'diamond'])),74 tooltip=['cluster:N','slice:N','content:N','label:N','pred:O'],75 opacity=opacity76 ).properties(77 width=1000,78 height=80079 ).interactive()80 81 legend = alt.Chart(df).mark_point(size=100, filled=True).encode(82 x=alt.X("label:N"),83 y=alt.Y('cluster:N', axis=alt.Axis(orient='right'), sort='descending', title=''),84 shape=alt.Shape('label:N', scale=alt.Scale(85 range=['circle', 'diamond']), legend=None),86 color=color,87 ).add_selection(88 selection89 )90 layered = scatter | legend91 layered = layered.configure_axis(92 grid=False93 ).configure_view(94 strokeOpacity=095 )96 return layered97 98def quant_panel(embedding_df):99 """ Quantitative Panel Layout"""100 all_metrics = {}101 st.warning("**Error slice visualization**")102 with st.expander("How to read this chart:"):103 st.markdown("* Each **point** is an input example.")104 st.markdown("* Gray points have low-loss and the colored have high-loss. High-loss instances are clustered using **kmeans** and each color represents a cluster.")105 st.markdown("* The **shape** of each point reflects the label category -- positive (diamond) or negative sentiment (circle).")106 #st.altair_chart(data_comparison(down_samp(embedding_df)), use_container_width=True)107 st.altair_chart(data_comparison(embedding_df), use_container_width=True)108 109 110def frequent_tokens(data, tokenizer, loss_quantile=0.95, top_k=200, smoothing=0.005):111 unique_tokens = []112 tokens = []113 for row in tqdm(data['content']):114 tokenized = tokenizer(row,padding=True, return_tensors='pt')115 tokens.append(tokenized['input_ids'].flatten())116 unique_tokens.append(torch.unique(tokenized['input_ids']))117 losses = data['loss'].astype(float)118 high_loss = losses.quantile(loss_quantile)119 loss_weights = (losses > high_loss)120 loss_weights = loss_weights / loss_weights.sum()121 token_frequencies = defaultdict(float)122 token_frequencies_error = defaultdict(float)123 124 weights_uniform = np.full_like(loss_weights, 1 / len(loss_weights))125 126 num_examples = len(data)127 for i in tqdm(range(num_examples)):128 for token in unique_tokens[i]:129 token_frequencies[token.item()] += weights_uniform[i]130 token_frequencies_error[token.item()] += loss_weights[i]131 132 token_lrs = {k: (smoothing+token_frequencies_error[k]) / (smoothing+token_frequencies[k]) for k in token_frequencies}133 tokens_sorted = list(map(lambda x: x[0], sorted(token_lrs.items(), key=lambda x: x[1])[::-1]))134 135 top_tokens = []136 for i, (token) in enumerate(tokens_sorted[:top_k]):137 top_tokens.append(['%10s' % (tokenizer.decode(token)), '%.4f' % (token_frequencies[token]), '%.4f' % (138 token_frequencies_error[token]), '%4.2f' % (token_lrs[token])])139 return pd.DataFrame(top_tokens, columns=['Token', 'Freq', 'Freq error slice', 'Ratio w/ smoothing'])140 141 142@st.cache(ttl=600)143def get_data(inference, emb):144 preds = inference.outputs.numpy()145 losses = inference.losses.numpy()146 embeddings = pd.DataFrame(emb, columns=['x', 'y'])147 num_examples = len(losses)148 # dataset_labels = [dataset[i]['label'] for i in range(num_examples)]149 return pd.concat([pd.DataFrame(np.transpose(np.vstack([dataset[:num_examples]['content'], 150 dataset[:num_examples]['label'], preds, losses])), columns=['content', 'label', 'pred', 'loss']), embeddings], axis=1)151 152def clustering(data,num_clusters):153 X = np.array(data['embedding'].tolist())154 kclusterer = KMeansClusterer(155 num_clusters, distance=nltk.cluster.util.cosine_distance,156 repeats=25,avoid_empty_clusters=True)157 assigned_clusters = kclusterer.cluster(X, assign_clusters=True)158 data['cluster'] = pd.Series(assigned_clusters, index=data.index).astype('int')159 data['centroid'] = data['cluster'].apply(lambda x: kclusterer.means()[x])160 return data, assigned_clusters161 162def kmeans(df, num_clusters=3):163 #data_hl = df.loc[df['slice'] == 'high-loss']164 data_kmeans,clusters = clustering(df,num_clusters)165 #merged = pd.merge(df, data_kmeans, left_index=True, right_index=True, how='outer', suffixes=('', '_y'))166 #merged.drop(merged.filter(regex='_y$').columns.tolist(),axis=1,inplace=True)167 #merged['cluster'] = merged['cluster'].fillna(num_clusters).astype('int')168 return data_kmeans169 170def distance_from_centroid(row):171 return sdist.norm(row['embedding'] - row['centroid'].tolist())172 173@st.cache(ttl=600)174def topic_distribution(weights, smoothing=0.01):175 topic_frequencies = defaultdict(float)176 topic_frequencies_error= defaultdict(float)177 weights_uniform = np.full_like(weights, 1 / len(weights))178 num_examples = len(weights)179 for i in range(num_examples):180 example = dataset[i]181 category = example['title']182 topic_frequencies[category] += weights_uniform[i]183 topic_frequencies_error[category] += weights[i]184 185 topic_ratios = {c: (smoothing + topic_frequencies_error[c]) / (186 smoothing + topic_frequencies[c]) for c in topic_frequencies}187 188 categories_sorted = map(lambda x: x[0], sorted(189 topic_ratios.items(), key=lambda x: x[1], reverse=True))190 191 topic_distr = []192 for category in categories_sorted:193 topic_distr.append(['%.3f' % topic_frequencies[category], '%.3f' %194 topic_frequencies_error[category], '%.2f' % topic_ratios[category], '%s' % category])195 196 return pd.DataFrame(topic_distr, columns=['Overall frequency', 'Error frequency', 'Ratio', 'Category'])197 198def populate_session(dataset,model):199 data_df = read_file_to_df('./assets/data/'+dataset+ '_'+ model+'.parquet')200 if model == 'albert-base-v2-yelp-polarity':201 tokenizer = AutoTokenizer.from_pretrained('textattack/'+model)202 else:203 tokenizer = AutoTokenizer.from_pretrained(model)204 if "user_data" not in st.session_state:205 st.session_state["user_data"] = data_df206 if "selected_slice" not in st.session_state:207 st.session_state["selected_slice"] = None208 209@st.cache(allow_output_mutation=True)210def read_file_to_df(file):211 return pd.read_parquet(file)212 213if __name__ == "__main__":214 ### STREAMLIT APP CONGFIG ###215 st.set_page_config(layout="wide", page_title="Interactive Error Analysis")216 217 ut.init_style()218 219 lcol, rcol = st.columns([2, 2])220 # ******* loading the mode and the data221 #st.sidebar.mardown("<h4>Interactive Error Analysis</h4>", unsafe_allow_html=True)222 223 dataset = st.sidebar.selectbox(224 "Dataset",225 ["amazon_polarity", "yelp_polarity"],226 index = 1227 )228 229 model = st.sidebar.selectbox(230 "Model",231 ["distilbert-base-uncased-finetuned-sst-2-english",232 "albert-base-v2-yelp-polarity"],233 )234 235 ### LOAD DATA AND SESSION VARIABLES ###236 ##uncomment the next next line to run dynamically and not from file237 #populate_session(dataset, model)238 data_df = read_file_to_df('./assets/data/'+dataset+ '_'+ model+'.parquet')239 loss_quantile = st.sidebar.slider(240 "Loss Quantile", min_value=0.5, max_value=1.0,step=0.01,value=0.99241 )242 data_df = data_df.drop(data_df[data_df.pred == data_df.label].index) #drop rows that are not errors243 data_df['loss'] = data_df['loss'].astype(float)244 losses = data_df['loss']245 high_loss = losses.quantile(loss_quantile)246 data_df['slice'] = 'high-loss'247 data_df['slice'] = data_df['slice'].where(data_df['loss'] > high_loss, 'low-loss') 248 data_hl = data_df.drop(data_df[data_df['slice'] == 'low-loss'].index) #drop rows that are not hl249 data_ll = data_df.drop(data_df[data_df['slice'] == 'high-loss'].index) 250 df_list = [d for _, d in data_hl.groupby(['label'])] # this is to allow clustering over each error type. fp, fn for binary classification251 252 with lcol:253 st.markdown('<h3>Error Slices</h3>',unsafe_allow_html=True)254 with st.expander("How to read the table:"):255 st.markdown("* *Error slice* refers to the subset of evaluation dataset the model performs poorly on.")256 st.markdown("* The table displays model error slices on the evaluation dataset, sorted by loss.")257 st.markdown("* Each row is an input example that includes the label, model pred, loss, and error cluster.")258 with st.spinner(text='loading error slice...'):259 dataframe=read_file_to_df('./assets/data/'+dataset+ '_'+ model+'_error-slices.parquet')260 #uncomment the next next line to run dynamically and not from file261 # dataframe = merged[['content', 'label', 'pred', 'loss', 'cluster']].sort_values(262 # by=['loss'], ascending=False)263 # table_html = dataframe.to_html(264 # columns=['content', 'label', 'pred', 'loss', 'cluster'], max_rows=50)265 # table_html = table_html.replace("<th>", '<th align="left">') # left-align the headers266 st.write(dataframe,width=900, height=300)267 268 with rcol:269 with st.spinner(text='loading...'):270 st.markdown('<h3>Word Distribution in Error Slice</h3>', unsafe_allow_html=True)271 #uncomment the next two lines to run dynamically and not from file272 #commontokens = frequent_tokens(data_df, tokenizer, loss_quantile=loss_quantile)273 commontokens = read_file_to_df('./assets/data/'+dataset+ '_'+ model+'_commontokens.parquet')274 with st.expander("How to read the table:"):275 st.markdown("* The table displays the most frequent tokens in error slices, relative to their frequencies in the val set.")276 st.write(commontokens)277 278 run_kmeans = st.sidebar.radio("Cluster error slice?", ('True', 'False'), index=0)279 280 num_clusters = st.sidebar.slider("# clusters", min_value=1, max_value=20, step=1, value=3)281 282 if run_kmeans == 'True':283 with st.spinner(text='running kmeans...'):284 merged = pd.DataFrame()285 ind=0286 for df in df_list:287 #num_clusters= int(math.sqrt(len(df)/2))288 kmeans_df = kmeans(df,num_clusters=num_clusters)289 #print(kmeans_df.loc[kmeans_df['cluster'].idxmax()])290 kmeans_df['cluster'] = kmeans_df['cluster'] + ind*num_clusters291 ind = ind+1292 merged = pd.concat([merged, kmeans_df])293 merged = pd.concat([merged, data_ll])294 295 with st.spinner(text='loading visualization...'):296 quant_panel(merged)