GEM/DatasetCardForm
9
1import json2import streamlit as st3 4from os.path import join as pjoin5 6from .streamlit_utils import (7 make_multiselect,8 make_selectbox,9 make_text_area,10 make_text_input,11 make_radio,12)13 14N_FIELDS_WHERE = 915N_FIELDS_LANGUAGES = 816N_FIELDS_CREDIT = 517N_FIELDS_STRUCTURE = 718 19N_FIELDS = N_FIELDS_WHERE + N_FIELDS_LANGUAGES + N_FIELDS_CREDIT + N_FIELDS_STRUCTURE20 21 22languages_bcp47 = [23 x24 for x in json.load(open(pjoin("resources", "bcp47.json"), encoding="utf-8"))[25 "subtags"26 ]27 if x["type"] == "language"28]29 30license_list = json.load(open(pjoin("resources", "licenses.json"), encoding="utf-8"))31 32 33def overview_page():34 st.session_state.card_dict["overview"] = st.session_state.card_dict.get(35 "overview", {}36 )37 with st.expander("What is this dataset?", expanded=True):38 key_pref = ["overview", "what"]39 st.session_state.card_dict["overview"]["what"] = st.session_state.card_dict[40 "overview"41 ].get("what", {})42 make_text_area(43 label="Provide a summary of this dataset in 3-4 sentences.",44 key_list=key_pref + ["dataset"],45 help="[free text]",46 )47 with st.expander("Where to find the data and its documentation", expanded=False):48 key_pref = ["overview", "where"]49 st.session_state.card_dict["overview"]["where"] = st.session_state.card_dict[50 "overview"51 ].get("where", {})52 make_text_input(53 label="What is the webpage for the dataset (if it exists)?",54 key_list=key_pref + ["website"],55 help="[URL]",56 )57 make_text_input(58 label="What is the link to where the original dataset is hosted?",59 key_list=key_pref + ["data-url"],60 help="[URL]",61 )62 make_text_input(63 label="What is the link to the paper describing the dataset (open access preferred)?",64 key_list=key_pref + ["paper-url"],65 help="[URL]",66 )67 make_text_area(68 label="Provide the BibTex-formatted reference for the dataset. Please use the correct published version (ACL anthology, etc.) instead of google scholar created Bibtex.",69 key_list=key_pref + ["paper-bibtext"],70 help="[free text]",71 )72 make_radio(73 label="Does the dataset have an active leaderboard?",74 options=["no", "yes"],75 key_list=key_pref + ["has-leaderboard"],76 help="If no, enter N/A for the following two fields",77 )78 if st.session_state.card_dict["overview"]["where"]["has-leaderboard"] == "yes":79 make_text_input(80 label="Provide a link to the leaderboard.",81 key_list=key_pref + ["leaderboard-url"],82 help="[URL] or N/A",83 )84 make_text_area(85 label="Briefly describe how the leaderboard evaluates models.",86 key_list=key_pref + ["leaderboard-description"],87 help="[free text; a paragraph] or N/A",88 )89 else:90 st.session_state.card_dict["overview"]["where"]["leaderboard-url"] = "N/A"91 st.session_state.card_dict["overview"]["where"]["leaderboard-description"] = "N/A"92 make_text_input(93 label="If known, provide the name of at least one person the reader can contact for questions about the dataset.",94 key_list=key_pref + ["contact-name"],95 help="[free text]",96 )97 make_text_input(98 label="If known, provide the email of at least one person the reader can contact for questions about the dataset.",99 key_list=key_pref + ["contact-email"],100 help="[free text]",101 )102 with st.expander("Languages and Intended Use", expanded=False):103 key_pref = ["overview", "languages"]104 st.session_state.card_dict["overview"][105 "languages"106 ] = st.session_state.card_dict["overview"].get("languages", {})107 make_radio(108 label="Is the dataset multilingual?",109 options=["no", "yes"],110 key_list=key_pref + ["is-multilingual"],111 help="More than one language present in all of the text fields",112 )113 make_multiselect(114 label="What languages/dialects are covered in the dataset?",115 key_list=key_pref + ["language-names"],116 options=[", ".join(x["description"]) for x in languages_bcp47],117 help="This is a comprehensive list of languages obtained from the BCP-47 standard list.",118 )119 make_text_area(120 label="What dialects are covered? Are there multiple dialects per language?",121 key_list=key_pref + ["language-dialects"],122 help="[free text, paragraphs] - Describe the dialect(s) as appropriate.",123 )124 make_text_area(125 label="Whose language is in the dataset?",126 key_list=key_pref + ["language-speakers"],127 help="[free text, paragraphs] - Provide locally appropriate demographic information about the language producers, if available. Use ranges where reasonable in order to protect individuals’ privacy.",128 )129 make_text_area(130 label="What is the intended use of the dataset?",131 key_list=key_pref + ["intended-use"],132 help="[free text, paragraphs] - Describe how the dataset creators describe its purpose and intended use.",133 )134 make_selectbox(135 label="What is the license of the dataset?",136 key_list=key_pref + ["license"],137 options=license_list,138 help="select `other` if missing from list, `unkown` if not provided.",139 )140 if "other" in st.session_state.card_dict["overview"]["languages"].get("license", []):141 make_text_input(142 label="What is the 'other' license of the dataset?",143 key_list=key_pref + ["license-other"],144 help="[free text]",145 )146 else:147 st.session_state.card_dict["overview"]["languages"]["license-other"] = "N/A"148 149 150 make_selectbox(151 label="What primary task does the dataset support?",152 key_list=key_pref + ["task"],153 options=[154 "", # default needs to be invalid value to make sure people actually fill in155 "Content Transfer",156 "Data-to-Text",157 "Dialog Response Generation",158 "Paraphrasing",159 "Question Generation",160 "Reasoning",161 "Simplification",162 "Style Transfer",163 "Summarization",164 "Text-to-Slide",165 "Other"166 ],167 help="Select `other` if the task is not included in the list.",168 )169 if "Other" in st.session_state.card_dict["overview"]["languages"].get("task", []):170 make_text_input(171 label="What is the primary task?",172 key_list=key_pref + ["task-other"],173 help="[free text]",174 )175 else:176 st.session_state.card_dict["overview"]["languages"]["task-other"] = "N/A"177 178 make_text_area(179 label="Provide a short description of the communicative goal of a model trained for this task on this dataset.",180 key_list=key_pref + ["communicative"],181 help="[free text, a paragraph] (e.g., describe a restaurant from a structured representation of its attributes)",182 )183 with st.expander("Credit", expanded=False):184 key_pref = ["overview", "credit"]185 st.session_state.card_dict["overview"][186 "credit"187 ] = st.session_state.card_dict["overview"].get("credit", {})188 make_multiselect(189 label="In what kind of organization did the dataset curation happen?",190 options=["industry", "academic", "independent", "other"],191 key_list=key_pref + ["organization-type"],192 )193 make_text_input(194 label="Name the organization(s).",195 key_list=key_pref + ["organization-names"],196 help="comma-separated",197 )198 make_text_input(199 label="Who created the original dataset? List the people involved in collecting the dataset and their affiliation(s).",200 key_list=key_pref + ["creators"],201 help="name (affiliation); comma-separated",202 )203 make_text_input(204 label="Who funded the data creation?",205 key_list=key_pref + ["funding"],206 help="[free text] enter N/A if unkown",207 )208 make_text_input(209 label="Who contributed to the data card and adding the dataset to GEM? List the people+affiliations involved in creating this data card and who helped integrate this dataset into GEM.",210 key_list=key_pref + ["gem-added-by"],211 help="name (affiliation); comma-separated",212 )213 with st.expander("Structure", expanded=False):214 key_pref = ["overview", "structure"]215 st.session_state.card_dict["overview"]["structure"] = st.session_state.card_dict[216 "overview"217 ].get("structure", {})218 data_fields_help = """219 [free text; paragraphs]220 - Mention their data type, and whether and how they are used as part of the generation pipeline.221 - Describe each fields' attributes, such as whether they are at the character level or word level, whether they are contiguous or not, etc.222 - If the datasets contain example IDs, state whether they have an inherent meaning, such as a mapping to other datasets or pointing to relationships between data points.223 """224 make_text_area(225 label="List and describe the fields present in the dataset.",226 key_list=key_pref + ["data-fields"],227 help=data_fields_help,228 )229 make_text_area(230 label="How was the dataset structure determined?",231 key_list=key_pref + ["structure-description"],232 help="[free text; paragraph]",233 )234 make_text_area(235 label="How were the labels chosen?",236 key_list=key_pref + ["structure-labels"],237 help="[free text; paragraph]",238 )239 make_text_area(240 label="Provide a JSON formatted example of a typical instance in the dataset.",241 key_list=key_pref + ["structure-example"],242 help="[JSON]",243 )244 make_text_area(245 label="Describe and name the splits in the dataset if there are more than one.",246 key_list=key_pref + ["structure-splits"],247 help="[free text, paragraphs] - As appropriate, provide any descriptive statistics for the features, such as size, average lengths of input and output.",248 )249 make_text_area(250 label="Describe any criteria for splitting the data, if used. If there are differences between the splits (e.g., if the training annotations are machine-generated and the dev and test ones are created by humans, or if different numbers of annotators contributed to each example), describe them here.",251 key_list=key_pref + ["structure-splits-criteria"],252 help="[free text, paragraphs]",253 )254 make_text_area(255 label="What does an outlier of the dataset in terms of length/perplexity/embedding look like?",256 key_list=key_pref + ["structure-outlier"],257 help="[free text + json formatted text/file for an example]",258 )259 260 261def overview_summary():262 total_filled = sum(263 [len(dct) for dct in st.session_state.card_dict.get("overview", {}).values()]264 )265 with st.expander(266 f"Dataset Overview Completion - {total_filled} of {N_FIELDS}", expanded=False267 ):268 completion_markdown = ""269 completion_markdown += (270 f"- **Overall completion:**\n - {total_filled} of {N_FIELDS} fields\n"271 )272 completion_markdown += f"- **Sub-section - Where to find:**\n - {len(st.session_state.card_dict.get('overview', {}).get('where', {}))} of {N_FIELDS_WHERE} fields\n"273 completion_markdown += f"- **Sub-section - Languages and Intended Use:**\n - {len(st.session_state.card_dict.get('overview', {}).get('languages', {}))} of {N_FIELDS_LANGUAGES} fields\n"274 completion_markdown += f"- **Sub-section - Credit:**\n - {len(st.session_state.card_dict.get('overview', {}).get('credit', {}))} of {N_FIELDS_CREDIT} fields\n"275 completion_markdown += f"- **Sub-section - Structure:**\n - {len(st.session_state.card_dict.get('overview', {}).get('structure', {}))} of {N_FIELDS_STRUCTURE} fields\n"276 st.markdown(completion_markdown)277