HuggingFaceM4/IDEFICS_Data_Measurement_Tool
2
1# Copyright 2021 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import argparse16import ast17import gradio as gr18from os.path import isdir19from data_measurements.dataset_statistics import DatasetStatisticsCacheClass as dmt_cls20import utils21from utils import dataset_utils22from utils import gradio_utils as gr_utils23import widgets24 25logs = utils.prepare_logging(__file__)26 27# Utility for sidebar description and selection of the dataset28DATASET_NAME_TO_DICT = dataset_utils.get_dataset_info_dicts()29 30 31def get_load_prepare_list(dstats):32 """33 # Get load_or_prepare functions for the measurements we will display34 """35 # Measurement calculation:36 # Add any additional modules and their load-prepare function here.37 load_prepare_list = [("general stats", dstats.load_or_prepare_general_stats),38 ("label distribution", dstats.load_or_prepare_labels),39 ("text_lengths", dstats.load_or_prepare_text_lengths),40 ("duplicates", dstats.load_or_prepare_text_duplicates),41 ("npmi", dstats.load_or_prepare_npmi),42 ("zipf", dstats.load_or_prepare_zipf)]43 44 return load_prepare_list45 46 47def get_ui_widgets():48 """Get the widgets that will be displayed in the UI."""49 return [widgets.DatasetDescription(DATASET_NAME_TO_DICT),50 widgets.GeneralStats(),51 widgets.LabelDistribution(),52 widgets.TextLengths(),53 widgets.Duplicates(),54 widgets.Npmi(),55 widgets.Zipf()]56 57 58def get_widgets():59 """60 # A measurement widget requires 2 things:61 # - A load or prepare function62 # - A display function63 # We define these in two separate functions get_load_prepare_list and get_ui_widgets;64 # any widget can be added by modifying both functions and the rest of the app logic will work.65 # get_load_prepare_list is a function since it requires a DatasetStatisticsCacheClass which will66 # not be created until dataset and config values are selected in the ui67 """68 return get_load_prepare_list, get_ui_widgets()69 70 71def get_title(dstats):72 title_str = f"### Showing: {dstats.dset_name} - {dstats.dset_config} - {dstats.split_name} - {'-'.join(dstats.text_field)}"73 logs.info("showing header")74 return title_str75 76 77def display_initial_UI():78 """Displays the header in the UI"""79 # Extract the selected arguments80 dataset_args = gr_utils.sidebar_selection(DATASET_NAME_TO_DICT)81 return dataset_args82 83 84def load_or_prepare_widgets(dstats, load_prepare_list, show_perplexities, live=True, pull_cache_from_hub=False):85 """86 Takes the dataset arguments from the GUI and uses them to load a dataset from the Hub or, if87 a cache for those arguments is available, to load it from the cache.88 Widget data is loaded only when the system is live (deployed for users).89 Otherwise, the data is prepared if it doesn't yet exist.90 Args:91 ds_args (dict): the dataset arguments defined via the streamlit app GUI92 load_prepare_list (list): List of (widget_name, widget_load_or_prepare_function)93 show_perplexities (Bool): whether perplexities should be loaded and displayed for this dataset94 live (Bool): Whether the system is deployed for live use by users.95 pull_cache_from_hub (Bool): Whether the cache should be pulled from the hub (vs locally)96 Returns:97 dstats: the computed dataset statistics (from the dataset_statistics class)98 """99 100 # When we're "live" (tool is being used by users on our servers),101 # cache is used and the f'ns are instructed to only try to load cache,102 # not to prepare/compute anything anew.103 if live:104 # Only use what's cached; don't prepare anything105 load_only = True106 logs.info("Only using cache.")107 else:108 # Prepare things anew and cache them if we're not live.109 load_only = False110 logs.info("Making new calculations if cache is not there.")111 if pull_cache_from_hub:112 dataset_utils.pull_cache_from_hub(dstats.cache_path, dstats.dataset_cache_dir)113 114 # Data common across DMT:115 # Includes the dataset text/requested feature column,116 # the dataset tokenized, and the vocabulary117 dstats.load_or_prepare_text_dataset(load_only=load_only)118 # Just a snippet of the dataset119 dstats.load_or_prepare_dset_peek(load_only=load_only)120 # Tokenized dataset121 dstats.load_or_prepare_tokenized_df(load_only=load_only)122 # Vocabulary (uses tokenized dataset)123 dstats.load_or_prepare_vocab(load_only=load_only)124 # Custom widgets125 for widget_tuple in load_prepare_list:126 widget_name = widget_tuple[0]127 widget_fn = widget_tuple[1]128 try:129 widget_fn(load_only=load_only)130 except Exception as e:131 logs.warning("Issue with %s." % widget_name)132 logs.exception(e)133 # TODO: If these are cached, can't we just show them by default?134 # It won't take up computation time.135 if show_perplexities:136 try:137 dstats.load_or_prepare_text_perplexities(load_only=load_only)138 except Exception as e:139 logs.warning("Issue with %s." % "perplexities")140 logs.exception(e)141 return dstats142 143 144def show_column(dstats, display_list, show_perplexities, column_id=""):145 """146 Function for displaying the elements in the streamlit app.147 Args:148 dstats (class): The dataset_statistics.py DatasetStatisticsCacheClass149 display_list (list): List of tuples for (widget_name, widget_display_function)150 show_perplexities (Bool): Whether perplexities should be loaded and displayed for this dataset151 column_id (str): Which column of the dataset the analysis is done on [DEPRECATED for v1]152 """153 154 # start showing stuff155 gr_utils.expander_header(dstats, DATASET_NAME_TO_DICT)156 for widget_tuple in display_list:157 widget_type = widget_tuple[0]158 widget_fn = widget_tuple[1]159 logs.info("showing %s." % widget_type)160 try:161 widget_fn(dstats, column_id)162 except Exception as e:163 logs.warning("Jk jk jk. There was an issue with %s:" % widget_type)164 logs.exception(e)165 # TODO: Fix how this is a weird outlier.166 if show_perplexities:167 gr_utils.expander_text_perplexities(dstats, column_id)168 logs.info("Have finished displaying the widgets.")169 170 171def create_demo(live: bool, pull_cache_from_hub: bool):172 with gr.Blocks() as demo:173 state = gr.State()174 with gr.Row():175 with gr.Column(scale=1):176 dataset_args = display_initial_UI()177 get_load_prepare_list_fn, widget_list = get_widgets()178 # # TODO: Make this less of a weird outlier.179 # Doesn't do anything right now180 show_perplexities = gr.Checkbox(label="Show text perplexities")181 with gr.Column(scale=4):182 gr.Markdown("# Data Measurements Tool")183 title = gr.Markdown()184 for widget in widget_list:185 widget.render()186 187 def update_ui(dataset: str, config: str, split: str, feature: str):188 feature = ast.literal_eval(feature)189 label_field, label_names = gr_utils.get_label_names(dataset, config, DATASET_NAME_TO_DICT)190 dstats = dmt_cls(dset_name=dataset, dset_config=config, split_name=split, text_field=feature,191 label_field=label_field, label_names=label_names, use_cache=True)192 load_prepare_list = get_load_prepare_list_fn(dstats)193 dstats = load_or_prepare_widgets(dstats, load_prepare_list, show_perplexities=False,194 live=live, pull_cache_from_hub=pull_cache_from_hub)195 output = {title: get_title(dstats), state: dstats}196 for widget in widget_list:197 output.update(widget.update(dstats))198 return output199 200 def update_dataset(dataset: str):201 new_values = gr_utils.update_dataset(dataset, DATASET_NAME_TO_DICT)202 config = new_values[0][1]203 feature = new_values[1][1]204 split = new_values[2][1]205 new_dropdown = {206 dataset_args["dset_config"]: gr.Dropdown.update(choices=new_values[0][0], value=config),207 dataset_args["text_field"]: gr.Dropdown.update(choices=new_values[1][0], value=feature),208 dataset_args["split_name"]: gr.Dropdown.update(choices=new_values[2][0], value=split),209 }210 return new_dropdown211 212 def update_config(dataset: str, config: str):213 new_values = gr_utils.update_config(dataset, config, DATASET_NAME_TO_DICT)214 215 feature = new_values[0][1]216 split = new_values[1][1]217 new_dropdown = {218 dataset_args["text_field"]: gr.Dropdown.update(choices=new_values[0][0], value=feature),219 dataset_args["split_name"]: gr.Dropdown.update(choices=new_values[1][0], value=split)220 }221 return new_dropdown222 223 measurements = [comp for output in widget_list for comp in output.output_components]224 demo.load(update_ui,225 inputs=[dataset_args["dset_name"], dataset_args["dset_config"], dataset_args["split_name"], dataset_args["text_field"]],226 outputs=[title, state] + measurements)227 228 for widget in widget_list:229 widget.add_events(state)230 #dataset_args["text_field"] --> the text that could be returned231 dataset_args["dset_name"].change(update_dataset,232 inputs=[dataset_args["dset_name"]],233 outputs=[dataset_args["dset_config"],234 dataset_args["split_name"], dataset_args["text_field"],235 title, state] + measurements)236 237 dataset_args["dset_config"].change(update_config,238 inputs=[dataset_args["dset_name"], dataset_args["dset_config"]],239 outputs=[dataset_args["split_name"], dataset_args["text_field"],240 title, state] + measurements)241 242 dataset_args["calculate_btn"].click(update_ui,243 inputs=[dataset_args["dset_name"], dataset_args["dset_config"],244 dataset_args["split_name"], dataset_args["text_field"]],245 outputs=[title, state] + measurements)246 return demo247 248 249def main():250 parser = argparse.ArgumentParser()251 parser.add_argument(252 "--live", default=False, required=False, action="store_true", help="Flag to specify that this is not running live.")253 parser.add_argument(254 "--pull_cache_from_hub", default=False, required=False, action="store_true", help="Flag to specify whether to look in the hub for measurements caches. If you are using this option, you must have HUB_CACHE_ORGANIZATION=<the organization you've set up on the hub to store your cache> and HF_TOKEN=<your hf token> on separate lines in a file named .env at the root of this repo.")255 arguments = parser.parse_args()256 live = arguments.live257 pull_cache_from_hub = arguments.pull_cache_from_hub258 259 # Create and initialize the demo260 demo = create_demo(live, pull_cache_from_hub)261 262 demo.launch()263 264if __name__ == "__main__":265 main()266