HuggingFaceM4/IDEFICS_Data_Measurement_Tool
2
1# Copyright 2021 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import logging16 17import gradio as gr18import numpy as np19import pandas as pd20from matplotlib.figure import Figure21import seaborn as sns22import statistics23import streamlit as st24import utils25import utils.dataset_utils as ds_utils26#from st_aggrid import AgGrid, GridOptionsBuilder --> commenting out to fix local build error27from utils.dataset_utils import HF_DESC_FIELD, HF_FEATURE_FIELD, HF_LABEL_FIELD28 29logs = utils.prepare_logging(__file__)30st.set_option('deprecation.showPyplotGlobalUse', False)31 32# Note: Make sure to consider colorblind-friendly colors for your images! Ex:33# ["#332288", "#117733", "#882255", "#AA4499", "#CC6677", "#44AA99", "#DDCC77",34# "#88CCEE"]35 36pd.options.display.float_format = "{:,.3f}".format # '{:20,.2f}'.format37 38def subheader():39 gr.Markdown("""This demo showcases the 40 [dataset metrics as we develop them](https://huggingface.co/blog/data-measurements-tool).41 Right now this has:42 - dynamic loading of datasets in the lib43 - fetching config and info without downloading the dataset44 - propose the list of candidate text and label features to select.45 """)46 47 48def get_label_names(dataset_name: str, config_name: str, ds_name_to_dict):49 label_field, label_names = (50 ds_name_to_dict[dataset_name][config_name][HF_FEATURE_FIELD][51 HF_LABEL_FIELD][0]52 if len(53 ds_name_to_dict[dataset_name][config_name][HF_FEATURE_FIELD][54 HF_LABEL_FIELD]55 ) > 056 else ((), [])57 )58 return label_field, label_names59 60 61 62def update_dataset(dataset_name: str, ds_name_to_dict):63 # choose a config to analyze64 ds_configs = ds_name_to_dict[dataset_name]65 # special handling for the largest-by-far dataset, C466 if dataset_name == "c4":67 config_names = ['en', 'en.noblocklist', 'realnewslike']68 else:69 config_names = list(ds_configs.keys())70 71 config_name = config_names[0]72 ds_config = ds_configs[config_name]73 74 text_features = ds_config[HF_FEATURE_FIELD]["string"]75 text_features = [('text',)] if dataset_name == "c4" else [tp for tp in text_features if tp[0] != "id"]76 feature = str(text_features[0])77 text_features = [str(f) for f in text_features]78 79 avail_splits = list(ds_config["splits"].keys())80 split = avail_splits[0]81 82 return [(config_names, config_name), (text_features, feature), (avail_splits, split)]83 84 85def update_config(dataset_name: str, config_name: str, ds_name_to_dict):86 ds_config = ds_name_to_dict[dataset_name][config_name]87 88 text_features = ds_config[HF_FEATURE_FIELD]["string"]89 text_features = [('text',)] if dataset_name == "c4" else [tp for tp in text_features if tp[0] != "id"]90 feature = str(text_features[0])91 text_features = [str(f) for f in text_features]92 93 avail_splits = list(ds_config["splits"].keys())94 split = avail_splits[0]95 96 return [(text_features, feature), (avail_splits, split)]97 98 99def sidebar_selection(ds_name_to_dict, column_id=""):100 ds_names = list(ds_name_to_dict.keys())101 with gr.Accordion(f"Choose dataset and field {column_id}", open=True):102 subheader()103 # choose a dataset to analyze104 ds_name = gr.Dropdown(105 label=f"Choose dataset to explore{column_id}:",106 choices=ds_names,107 value="hate_speech18",108 )109 # choose a config to analyze110 ds_configs = ds_name_to_dict[ds_name.value]111 # special handling for the largest-by-far dataset, C4112 if ds_name == "c4":113 config_names = ['en', 'en.noblocklist', 'realnewslike']114 else:115 config_names = list(ds_configs.keys())116 config_name = gr.Dropdown(117 label=f"Choose configuration{column_id}:",118 choices=config_names,119 value=config_names[0],120 )121 # choose a subset of num_examples122 ds_config = ds_configs[config_name.value]123 text_features = ds_config[HF_FEATURE_FIELD]["string"]124 # TODO @yacine: Explain what this is doing and why eg tp[0] could = "id"125 text = f"Which text feature from the {column_id} dataset would you like to analyze?"126 choices = [('text',)] if ds_name == "c4" else [tp for tp in text_features if tp[0] != "id"]127 text_field = gr.Dropdown(128 label=text,129 choices=[str(f) for f in choices],130 value=str(choices[0])131 )132 # Choose a split and dataset size133 avail_splits = list(ds_config["splits"].keys())134 # 12.Nov note: Removing "test" because those should not be examined135 # without discussion of pros and cons, which we haven't done yet.136 if "test" in avail_splits:137 avail_splits.remove("test")138 split = gr.Dropdown(139 label=f"Which split from the{column_id} dataset would you like to analyze?",140 choices=avail_splits,141 value=avail_splits[0],142 )143 label_field, label_names = get_label_names(ds_name.value, config_name.value, ds_name_to_dict)144 calculate_btn = gr.Button(value="Calculate", variant="primary")145 return {146 "dset_name": ds_name,147 "dset_config": config_name,148 "split_name": split,149 "text_field": text_field,150 "label_field": label_field,151 "label_names": label_names,152 "calculate_btn": calculate_btn153 }154 155 156def expander_header(dstats, ds_name_to_dict, column_id=""):157 with st.expander(f"Dataset Description{column_id}"):158 st.markdown(159 ds_name_to_dict[dstats.dset_name][dstats.dset_config][HF_DESC_FIELD]160 )161 st.dataframe(dstats.dset_peek)162 163 164def expander_general_stats(dstats, column_id=""):165 with gr.Accordion(f"General Text Statistics{column_id}"):166 st.caption(167 "Use this widget to check whether the terms you see most "168 "represented in the dataset make sense for the goals of the dataset."169 )170 st.markdown("There are {0} total words".format(str(dstats.total_words)))171 st.markdown(172 "There are {0} words after removing closed "173 "class words".format(str(dstats.total_open_words))174 )175 st.markdown(176 "The most common "177 "[open class words](https://dictionary.apa.org/open-class-words) "178 "and their counts are: "179 )180 st.dataframe(dstats.sorted_top_vocab_df)181 st.markdown(182 "There are {0} missing values in the dataset.".format(183 str(dstats.text_nan_count)184 )185 )186 if dstats.dups_frac > 0:187 st.markdown(188 "The dataset is {0}% duplicates. "189 "For more information about the duplicates, "190 "click the 'Duplicates' tab below.".format(191 str(round(dstats.dups_frac * 100, 2)))192 )193 else:194 st.markdown("There are 0 duplicate items in the dataset. ")195 196 197def expander_label_distribution(dstats, column_id=""):198 with st.expander(f"Label Distribution{column_id}", expanded=False):199 st.caption(200 "Use this widget to see how balanced the labels in your dataset are."201 )202 if dstats.fig_labels:203 st.plotly_chart(dstats.fig_labels, use_container_width=True)204 else:205 st.markdown("No labels were found in the dataset")206 207 208def expander_text_lengths(dstats, column_id=""):209 _TEXT_LENGTH_CAPTION = (210 "Use this widget to identify outliers, particularly suspiciously long "211 "outliers."212 )213 with st.expander(f"Text Lengths{column_id}", expanded=False):214 st.caption(_TEXT_LENGTH_CAPTION)215 st.markdown(216 "Below, you can see how the lengths of the text instances in your "217 "dataset are distributed."218 )219 st.markdown(220 "Any unexpected peaks or valleys in the distribution may help to "221 "identify instances you want to remove or augment."222 )223 st.markdown(224 "### Here is the count of different text lengths in "225 "your dataset:"226 )227 # When matplotlib first creates this, it's a Figure.228 # Once it's saved, then read back in,229 # it's an ndarray that must be displayed using st.image230 # (I know, lame).231 if isinstance(dstats.length_obj.fig_lengths, Figure):232 st.pyplot(dstats.length_obj.fig_lengths, use_container_width=True)233 else:234 try:235 st.image(dstats.length_obj.fig_lengths)236 except Exception as e:237 logs.exception("Hit exception for lengths figure:")238 logs.exception(e)239 st.markdown(240 "The average length of text instances is **"241 + str(round(dstats.length_obj.avg_length, 2))242 + " words**, with a standard deviation of **"243 + str(round(dstats.length_obj.std_length, 2))244 + "**."245 )246 if dstats.length_obj.lengths_df is not None:247 start_id_show_lengths = st.selectbox(248 "Show examples of length:",249 np.sort(dstats.length_obj.lengths_df["length"].unique())[::-1].tolist(),250 key=f"select_show_length_{column_id}",251 )252 st.table(253 dstats.length_obj.lengths_df[254 dstats.length_obj.lengths_df["length"] == start_id_show_lengths255 ].set_index("length")256 )257 258 259def expander_text_duplicates(dstats, column_id=""):260 with st.expander(f"Text Duplicates{column_id}", expanded=False):261 st.caption(262 "Use this widget to identify text strings that appear more than "263 "once."264 )265 st.markdown(266 "A model's training and testing may be negatively affected by "267 "unwarranted duplicates "268 "([Lee et al., 2021](https://arxiv.org/abs/2107.06499))."269 )270 st.markdown("------")271 st.write(272 "### Here is the list of all the duplicated items and their counts "273 "in the dataset."274 )275 if not dstats.duplicates_results:276 st.write("There are no duplicates in this dataset! 🥳")277 else:278 st.write("The fraction of the data that is a duplicate is:")279 st.write(str(round(dstats.dups_frac, 4)))280 # TODO: Check if this is slow when the size is large --281 # Should we store as dataframes?282 # Dataframes allow this to be interactive.283 st.dataframe(ds_utils.counter_dict_to_df(dstats.dups_dict))284 285 286def expander_text_perplexities(dstats, column_id=""):287 with st.expander(f"Text Perplexities{column_id}", expanded=False):288 st.caption(289 "Use this widget to identify text perplexities from GPT-2."290 )291 st.markdown(292 """293 Outlier perplexities, especially very high values, could highlight 294 an issue with an example. Smaller variations should be interpreted 295 with more care, as they indicate how similar to the GPT-2 training 296 corpus the examples are rather than being reflective of general 297 linguistic properties.298 For more information on GPT-2, 299 see its [model card](https://hf.co/gpt2).300 """301 )302 st.markdown("------")303 st.write(304 "### Here is the list of the examples in the dataset, sorted by "305 "GPT-2 perplexity:"306 )307 if dstats.perplexities_df is None or dstats.perplexities_df.empty:308 st.write(309 "Perplexities have not been computed yet for this dataset, or "310 "this dataset is too large for the UI (> 1,000,000 examples).")311 else:312 st.dataframe(dstats.perplexities_df.reset_index(drop=True))313 314 315def expander_npmi_description(min_vocab):316 _NPMI_CAPTION = (317 "Use this widget to identify problematic biases and stereotypes in "318 "your data."319 )320 _NPMI_CAPTION1 = """321 nPMI scores for a word help to identify potentially322 problematic associations, ranked by how close the association is."""323 _NPMI_CAPTION2 = """324 nPMI bias scores for paired words help to identify how word325 associations are skewed between the selected selected words326 ([Aka et al., 2021](https://arxiv.org/abs/2103.03417)).327 """328 329 st.caption(_NPMI_CAPTION)330 st.markdown(_NPMI_CAPTION1)331 st.markdown(_NPMI_CAPTION2)332 st.markdown(" ")333 st.markdown(334 "You can select from gender and sexual orientation "335 "identity terms that appear in the dataset at least %s "336 "times." % min_vocab337 )338 st.markdown(339 "The resulting ranked words are those that co-occur with both "340 "identity terms. "341 )342 st.markdown(343 "The more *positive* the score, the more associated the word is with "344 "the first identity term. "345 "The more *negative* the score, the more associated the word is with "346 "the second identity term."347 )348 349 350def expander_zipf(dstats, column_id=""):351 z = dstats.z352 zipf_fig = dstats.zipf_fig353 with st.expander(354 f"Vocabulary Distribution{column_id}: Zipf's Law Fit", expanded=False355 ):356 try:357 _ZIPF_CAPTION = """This shows how close the observed language is to an ideal358 natural language distribution following [Zipf's law](https://en.wikipedia.org/wiki/Zipf%27s_law),359 calculated by minimizing the [Kolmogorov-Smirnov (KS) statistic](https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test)."""360 361 powerlaw_eq = r"""p(x) \propto x^{- \alpha}"""362 zipf_summary = (363 "The optimal alpha based on this dataset is: **"364 + str(round(z.alpha, 2))365 + "**, with a KS distance of: **"366 + str(round(z.ks_distance, 2))367 )368 zipf_summary += (369 "**. This was fit with a minimum rank value of: **"370 + str(int(z.xmin))371 + "**, which is the optimal rank *beyond which* the scaling regime of the power law fits best."372 )373 374 alpha_warning = "Your alpha value is a bit on the high side, which means that the distribution over words in this dataset is a bit unnatural. This could be due to non-language items throughout the dataset."375 xmin_warning = "The minimum rank for this fit is a bit on the high side, which means that the frequencies of your most common words aren't distributed as would be expected by Zipf's law."376 fit_results_table = pd.DataFrame.from_dict(377 {378 r"Alpha:": [str("%.2f" % z.alpha)],379 "KS distance:": [str("%.2f" % z.ks_distance)],380 "Min rank:": [str("%s" % int(z.xmin))],381 },382 columns=["Results"],383 orient="index",384 )385 fit_results_table.index.name = column_id386 st.caption(387 "Use this widget for the counts of different words in your dataset, measuring the difference between the observed count and the expected count under Zipf's law."388 )389 st.markdown(_ZIPF_CAPTION)390 st.write(391 """392 A Zipfian distribution follows the power law: $p(x) \propto x^{-α}$393 with an ideal α value of 1."""394 )395 st.markdown(396 "In general, an alpha greater than 2 or a minimum rank greater than 10 (take with a grain of salt) means that your distribution is relativaly _unnatural_ for natural language. This can be a sign of mixed artefacts in the dataset, such as HTML markup."397 )398 st.markdown(399 "Below, you can see the counts of each word in your dataset vs. the expected number of counts following a Zipfian distribution."400 )401 st.markdown("-----")402 st.write("### Here is your dataset's Zipf results:")403 st.dataframe(fit_results_table)404 st.write(zipf_summary)405 # TODO: Nice UI version of the content in the comments.406 # st.markdown("\nThe KS test p-value is < %.2f" % z.ks_test.pvalue)407 # if z.ks_test.pvalue < 0.01:408 # st.markdown(409 # "\n Great news! Your data fits a powerlaw with a minimum KS " "distance of %.4f" % z.distance)410 # else:411 # st.markdown("\n Sadly, your data does not fit a powerlaw. =(")412 # st.markdown("Checking the goodness of fit of our observed distribution")413 # st.markdown("to the hypothesized power law distribution")414 # st.markdown("using a Kolmogorov–Smirnov (KS) test.")415 st.plotly_chart(zipf_fig, use_container_width=True)416 if z.alpha > 2:417 st.markdown(alpha_warning)418 if z.xmin > 5:419 st.markdown(xmin_warning)420 except:421 st.write("Under construction!")422 423 424def npmi_widget(dstats, column_id=""):425 """426 Part of the UI, but providing for interaction.427 :param column_id:428 :param dstats:429 :return:430 """431 min_vocab = dstats.min_vocab_count432 npmi_stats = dstats.npmi_obj433 available_terms = npmi_stats.avail_identity_terms434 with st.expander(f"Word Association{column_id}: nPMI", expanded=False):435 if npmi_stats and len(available_terms) > 0:436 expander_npmi_description(min_vocab)437 st.markdown("-----")438 term1 = st.selectbox(439 f"What is the first term you want to select?{column_id}",440 available_terms,441 )442 term2 = st.selectbox(443 f"What is the second term you want to select?{column_id}",444 reversed(available_terms),445 )446 try:447 joint_npmi_df = npmi_stats.get_display(term1, term2)448 npmi_show(joint_npmi_df)449 except Exception as e:450 logs.exception(e)451 st.markdown(452 "**WARNING!** The nPMI for these terms has not been"453 " pre-computed, please re-run caching."454 )455 else:456 st.markdown("No words found co-occurring with both of the selected identity"457 " terms.")458 459 460def npmi_show(paired_results):461 if paired_results.empty:462 st.markdown(463 "No words that co-occur enough times for results! Or there's a 🐛."464 " Or we're still computing this one. 🤷")465 else:466 logs.debug("Results to be shown in streamlit are")467 logs.debug(paired_results)468 s = pd.DataFrame(469 paired_results.sort_values(paired_results.columns[0], ascending=True))470 s.index.name = "word"471 bias_col = s.filter(like="bias").columns472 #count_cols = s.filter(like="count").columns473 # Keep the dataframe from being crazy big.474 if s.shape[0] > 10000:475 bias_thres = max(abs(s[s[0]][5000]),476 abs(s[s[0]][-5000]))477 logs.info(f"filtering with bias threshold: {bias_thres}")478 s_filtered = s[s[0].abs() > bias_thres]479 else:480 s_filtered = s481 cm = sns.palplot(sns.diverging_palette(270, 36, s=99, l=48, n=16))482 out_df = s_filtered.style.background_gradient(subset=bias_col, cmap=cm).format(formatter="{:,.3f}").set_properties(**{"align": "center", "width":"100em"}).set_caption("nPMI scores between the selected identity terms and the words they both co-occur with")483 #set_properties(subset=count_cols, **{"width": "10em", "text-align": "center"}).484 # .format(subset=count_cols, formatter=int).485 #.format(subset=bias_col, formatter="{:,.3f}")486 st.write("### Here is your dataset's bias results:")487 st.dataframe(out_df)488