GEM/DatasetCardForm

1from argparse import ArgumentParser2from json import load3import pathlib4import os5 6 7def multi_grep(d, l1, l2, l3):8  return d.get(l1, {}).get(l2, {}).get(l3, "[Needs More Information]")9 10def multi_grep2(d, l1, l2, l3):11  return d.get(l1, {}).get(l2, {}).get(l3, ["unknown"])12 13def sanitize_md_url(s):14  """Strip out MD fragments if they exist."""15  if len(s.split("](")) > 1:16    return s.split("](")[1].replace(")", "")17  else:18    return s19 20# ---21# annotations_creators:22# - expert-generated23# language_creators:24# - found25# languages:26# - en27# licenses:28# - unknown29# multilinguality:30# - monolingual31# pretty_name: FairytaleQA32# size_categories:33# - 10K<n<100K34# source_datasets:35# - original36# task_categories:37# - question-generation38# task_ids:39# - abstractive-qg40# ---41 42def construct_preamble(data, name):43  pre = "---\n"44  pre += "annotations_creators:\n"45  # - expert-generated46  s = multi_grep(data, "curation", "annotations", "origin")47  if s == "[Needs More Information]":48    pre += "- unknown\n"49  else:50    pre += "- " + s.replace(" ", "-") + "\n"51 52  pre += "language_creators:\n- unknown\n"53  pre += "languages:"54  languages = multi_grep2(data, "overview", "languages", "language_names")55  for l in languages:56    pre += f"\n- {l}"57  pre += "\nlicenses:\n"58 59  s = multi_grep(data, "overview", "languages", "license")60  if s == "[Needs More Information]":61    pre += "- unknown\n"62  else:63    pre += "- " + s.split(":")[0] + "\n"64 65  pre += "multilinguality:\n"66  if languages == ["unknown"]:67    pre += "- unknown"68  elif len(languages) == 1:69    pre += "- monolingual"70  else:71    pre += "- multilingual"72 73  # - monolingual74  pre += f"\npretty_name: {name}\n"75  pre += "size_categories:\n- unknown\n"76  pre += "source_datasets:\n- original\n"77  pre += "task_categories:\n"78 79  s = multi_grep(data, "overview", "languages", "task")80  if s == "[Needs More Information]":81    pre += "- unknown\n"82  else:83    pre += "- " + "-".join(s.lower().split(" ")) + "\n"84  # - question-generation85  pre += "task_ids:\n- unknown\n"86  # - abstractive-qg87 88  pre += "---\n\n"89  return pre90 91 92 93## Table of Contents94# - [Dataset Description](#dataset-description)95#   - [Dataset Summary](#dataset-summary)96#   - [Supported Tasks](#supported-tasks-and-leaderboards)97#   - [Languages](#languages)98# - [Dataset Structure](#dataset-structure)99#   - [Data Instances](#data-instances)100#   - [Data Fields](#data-instances)101#   - [Data Splits](#data-instances)102# - [Dataset Creation](#dataset-creation)103#   - [Curation Rationale](#curation-rationale)104#   - [Source Data](#source-data)105#   - [Annotations](#annotations)106#   - [Personal and Sensitive Information](#personal-and-sensitive-information)107# - [Considerations for Using the Data](#considerations-for-using-the-data)108#   - [Social Impact of Dataset](#social-impact-of-dataset)109#   - [Discussion of Biases](#discussion-of-biases)110#   - [Other Known Limitations](#other-known-limitations)111# - [Additional Information](#additional-information)112#   - [Dataset Curators](#dataset-curators)113#   - [Licensing Information](#licensing-information)114#   - [Citation Information](#citation-information)115 116def construct_toc(data):117  pass118 119def construct_links(data):120 121  links = "## Dataset Description\n\n"122 123  s = sanitize_md_url(multi_grep(data, "overview", "where", "website"))124  links += f"- **Homepage:** {s}\n"125 126  s = sanitize_md_url(multi_grep(data, "overview", "where", "data-url"))127  links += f"- **Repository:** {s}\n"128 129  s = sanitize_md_url(multi_grep(data, "overview", "where", "paper-url"))130  links += f"- **Paper:** {s}\n"131 132  s = sanitize_md_url(multi_grep(data, "overview", "where", "leaderboard-url"))133  links += f"- **Leaderboard:** {s}\n"134 135  s = multi_grep(data, "overview", "where", "contact-name")136  links += f"- **Point of Contact:** {s}\n\n"137 138  return links139 140 141def json_to_markdown(filename, original_json_path):142    json = load(open(filename))143    original_json = load(open(original_json_path))144    dataset_name = pathlib.Path(original_json_path).stem145 146 147    preamble = construct_preamble(original_json, dataset_name)148    markdown = preamble149 150    markdown += f'# Dataset Card for GEM/{json["name"]}\n\n'151 152    # ToC here.153 154    markdown += construct_links(original_json)155 156    markdown += "### Link to Main Data Card\n\n"157    markdown += f'You can find the main data card on the [GEM Website](https://gem-benchmark.com/data_cards/{dataset_name}).\n\n'158 159    markdown += "### Dataset Summary \n\n"160    markdown += json['summary'] + '\n\n'161 162    for key in json:163        if key not in ('name', 'summary', 'sections'):164            markdown += f'#### {key}\n{json[key]}\n\n'165 166    markdown += '\n'.join(section_to_markdown(section) \167                          for section in json['sections'])168 169    readme_path = os.path.join(pathlib.Path(original_json_path).parents[0], "README.md")170 171    with open(readme_path, 'w') as f:172        f.write(markdown)173 174 175def section_to_markdown(section):176    markdown = f'{"#" * section["level"]} {section["title"]}\n\n'177    markdown += '\n'.join(subsection_to_markdown(subsection) \178                          for subsection in section['subsections'])179 180    return markdown + '\n'181 182 183def subsection_to_markdown(subsection):184    markdown = f'{"#" * subsection["level"]} {subsection["title"]}\n\n'185    markdown += '\n'.join(field_to_markdown(field) \186                          for field in subsection['fields'])187 188    return markdown + '\n'189 190 191def field_to_markdown(field):192    markdown = f'{"#" * field["level"]} {field["title"]}\n\n'193 194    if 'flags' in field and 'quick' in field['flags']:195        markdown += f'<!-- quick -->\n'196 197    if field.get('info', False):198        markdown += f'<!-- info: {field["info"]} -->\n'199 200    if field.get('scope', False):201        markdown += f'<!-- scope: {field["scope"]} -->\n'202 203    markdown += field.get('content', '')204 205    return markdown + '\n'206 207 208# def main():209#     """Converts JSON output from `reformat_json.py`210#        to Markdown input for Data Cards Labs."""211#     args = parse_args()212#     for filename in args.input:213#         if filename[-5:] == '.json':214#             json_to_markdown(filename)215 216if __name__ == "__main__":217 218  for dataset in os.listdir("../../../GEMv2"):219    data_card_path = f"../../../GEMv2/{dataset}/{dataset}.json"220    if os.path.exists(data_card_path):221      print(f"Now processing {dataset}.")222      # This script assumes you have run reformat_json.py223      new_path = f"datacards/{dataset}.json"224 225      md_string = json_to_markdown(new_path, data_card_path)226 227    else:228      print(f"{dataset} has no data card!")229