tau/zero_scrolls

1# coding=utf-82# Lint as: python33"""The ZeroSCROLLS benchmark."""4 5import json6import os7import datasets8 9_ZERO_SCROLLS_CITATION = """10@inproceedings{shaham-etal-2023-zeroscrolls,11    title = "{Z}ero{SCROLLS}: A Zero-Shot Benchmark for Long Text Understanding",12    author = "Shaham, Uri  and13      Ivgi, Maor  and14      Efrat, Avia  and15      Berant, Jonathan  and16      Levy, Omer",17    editor = "Bouamor, Houda  and18      Pino, Juan  and19      Bali, Kalika",20    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2023",21    month = dec,22    year = "2023",23    address = "Singapore",24    publisher = "Association for Computational Linguistics",25    url = "https://aclanthology.org/2023.findings-emnlp.536",26    doi = "10.18653/v1/2023.findings-emnlp.536",27    pages = "7977--7989"28}29Note that each ZeroSCROLLS task has its own citation. Please see the source to30get the correct citation for each one.31"""32 33_ZERO_SCROLLS_DESCRIPTION = """34ZeroSCROLLS: Zero-Shot CompaRison Over Long Language Sequences.35A zero shot benchmark for long text reasoning.36https://zero.scrolls-benchmark.com/37"""38 39_SCROLLS_CITATION = """40@inproceedings{shaham-etal-2022-scrolls,41    title = "{SCROLLS}: Standardized {C}ompa{R}ison Over Long Language Sequences",42    author = "Shaham, Uri  and43      Segal, Elad  and44      Ivgi, Maor  and45      Efrat, Avia  and46      Yoran, Ori  and47      Haviv, Adi  and48      Gupta, Ankit  and49      Xiong, Wenhan  and50      Geva, Mor  and51      Berant, Jonathan  and52      Levy, Omer",53    booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing",54    month = dec,55    year = "2022",56    address = "Abu Dhabi, United Arab Emirates",57    publisher = "Association for Computational Linguistics",58    url = "https://aclanthology.org/2022.emnlp-main.823",59    pages = "12007--12021",60}61"""62 63_SCROLLS_DESCRIPTION = """64SCROLLS: Standardized CompaRison Over Long Language Sequences.65A suite of natural language datasets that require reasoning over long texts.66https://scrolls-benchmark.com/67"""68 69_SUMM_SCREEN_DESCRIPTION = """70SummScreenFD (Chen  et al., 2022) is a summarization dataset in the domain of TV shows (e.g. Friends, Game of Thrones).71Given a transcript of a specific episode, the goal is to produce the episode's recap.72The original dataset is divided into two complementary subsets, based on the source of its community contributed transcripts. 73For SCROLLS, we use the ForeverDreaming (FD) subset, as it incorporates 88 different shows, 74making it a more diverse alternative to the TV MegaSite (TMS) subset, which has only 10 shows. 75Community-authored recaps for the ForeverDreaming transcripts were collected from English Wikipedia and TVMaze."""76 77_QASPER_DESCRIPTION = """78Qasper (Dasigi et al., 2021) is a question answering dataset over NLP papers filtered from the Semantic Scholar Open Research Corpus (S2ORC).79Questions were written by NLP practitioners after reading only the title and abstract of the papers, 80while another set of NLP practitioners annotated the answers given the entire document.81Qasper contains abstractive, extractive, and yes/no questions, as well as unanswerable ones."""82 83_QMSUM_DESCRIPTION = """84QMSum (Zhong et al., 2021) is a query-based summarization dataset, consisting of 232 meetings transcripts from multiple domains. 85The corpus covers academic group meetings at the International Computer Science Institute and their summaries, industrial product meetings for designing a remote control, 86and committee meetings of the Welsh and Canadian Parliaments, dealing with a variety of public policy issues.87Annotators were tasked with writing queries about the broad contents of the meetings, as well as specific questions about certain topics or decisions, 88while ensuring that the relevant text for answering each query spans at least 200 words or 10 turns."""89 90_NARRATIVE_QA_DESCRIPTION = """91NarrativeQA (Kočiský et al., 2018) is an established question answering dataset over entire books from Project Gutenberg and movie scripts from different websites.92Annotators were given summaries of the books and scripts obtained from Wikipedia, and asked to generate question-answer pairs, 93resulting in about 30 questions and answers for each of the 1,567 books and scripts.94They were encouraged to use their own words rather then copying, and avoid asking yes/no questions or ones about the cast.95Each question was then answered by an additional annotator, providing each question with two reference answers (unless both answers are identical).."""96 97_GOV_REPORT_DESCRIPTION = """98GovReport (Huang et al., 2021) is a summarization dataset of reports addressing various national policy issues published by the 99Congressional Research Service and the U.S. Government Accountability Office, where each document is paired with a hand-written executive summary.100The reports and their summaries are longer than their equivalents in other popular long-document summarization datasets; 101for example, GovReport's documents are approximately 1.5 and 2.5 times longer than the documents in Arxiv and PubMed, respectively."""102 103_QUALITY_DESCRIPTION = """104QuALITY (Pang et al., 2022) is a multiple-choice question answering dataset over articles and stories sourced from Project Gutenberg, 105the Open American National Corpus, and more.106Experienced writers wrote questions and distractors, and were incentivized to write answerable, unambiguous questions such that in order to correctly answer them, 107human annotators must read large portions of the given document. 108Reference answers were then calculated using the majority vote between of the annotators and writer's answers.109To measure the difficulty of their questions, Pang et al. conducted a speed validation process, 110where another set of annotators were asked to answer questions given only a short period of time to skim through the document.111As a result, 50% of the questions in QuALITY are labeled as hard, i.e. the majority of the annotators in the speed validation setting chose the wrong answer."""112 113_SQUALITY_DESCRIPTION = """114SQuALITY (Wang et al., 2022) is a question-focused summarization dataset, where given a story from Project Gutenberg, 115the task is to produce a summary of the story or aspects of it based on a guiding question.116The questions and summaries are original and crowdsourced; experienced writers were guided to design questions that require reading significant parts of the story to answer correctly.117"""118 119_MUSIQUE_DESCRIPTION = """120MuSiQue (Trivedi et al,. 2022) is a multi-hop question answering dataset, where the inputs are 20 Wikipedia paragraphs and a question that requires multiple hops between different paragraphs.121In the original dataset, each question also has an unanswerable twin question, where the correct answer is not present in the paragraphs.122"""123 124_SPACE_DIGEST_DESCRIPTION = """125SpaceDigest is a new sentiment aggregation task.126Given 50 hotel reviews (without their ratings) from the Space dataset (Angelidis et al., 2021), the task is to determine the percentage of positive reviews.127"""128 129_BOOK_SUM_DESCRIPTION = """130BookSumSort is a new task based on the BookSum dataset (Kry ́sci ́nski et al., 2022), which contains summaries of chapters (or parts) of novels, plays, and long poems from various sources. 131Given a shuffled list of chapter summaries, the task is to reorder them according to the original order of summaries in BookSum.132"""133 134_SUMM_SCREEN_CITATION = r"""135@inproceedings{chen-etal-2022-summscreen,136    title = "{S}umm{S}creen: A Dataset for Abstractive Screenplay Summarization",137    author = "Chen, Mingda  and138      Chu, Zewei  and139      Wiseman, Sam  and140      Gimpel, Kevin",141    booktitle = "Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",142    month = may,143    year = "2022",144    address = "Dublin, Ireland",145    publisher = "Association for Computational Linguistics",146    url = "https://aclanthology.org/2022.acl-long.589",147    doi = "10.18653/v1/2022.acl-long.589",148    pages = "8602--8615",149    abstract = "We introduce SummScreen, a summarization dataset comprised of pairs of TV series transcripts and human written recaps. The dataset provides a challenging testbed for abstractive summarization for several reasons. Plot details are often expressed indirectly in character dialogues and may be scattered across the entirety of the transcript. These details must be found and integrated to form the succinct plot descriptions in the recaps. Also, TV scripts contain content that does not directly pertain to the central plot but rather serves to develop characters or provide comic relief. This information is rarely contained in recaps. Since characters are fundamental to TV series, we also propose two entity-centric evaluation metrics. Empirically, we characterize the dataset by evaluating several methods, including neural models and those based on nearest neighbors. An oracle extractive approach outperforms all benchmarked models according to automatic metrics, showing that the neural models are unable to fully exploit the input transcripts. Human evaluation and qualitative analysis reveal that our non-oracle models are competitive with their oracle counterparts in terms of generating faithful plot events and can benefit from better content selectors. Both oracle and non-oracle models generate unfaithful facts, suggesting future research directions.",150}"""151 152_QASPER_CITATION = r"""153@inproceedings{dasigi-etal-2021-dataset,154    title = "A Dataset of Information-Seeking Questions and Answers Anchored in Research Papers",155    author = "Dasigi, Pradeep  and156      Lo, Kyle  and157      Beltagy, Iz  and158      Cohan, Arman  and159      Smith, Noah A.  and160      Gardner, Matt",161    booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",162    month = jun,163    year = "2021",164    address = "Online",165    publisher = "Association for Computational Linguistics",166    url = "https://aclanthology.org/2021.naacl-main.365",167    doi = "10.18653/v1/2021.naacl-main.365",168    pages = "4599--4610"169    }"""170 171_QMSUM_CITATION = r"""@inproceedings{zhong-etal-2021-qmsum,172    title = "{QMS}um: A New Benchmark for Query-based Multi-domain Meeting Summarization",173    author = "Zhong, Ming  and174      Yin, Da  and175      Yu, Tao  and176      Zaidi, Ahmad  and177      Mutuma, Mutethia  and178      Jha, Rahul  and179      Awadallah, Ahmed Hassan  and180      Celikyilmaz, Asli  and181      Liu, Yang  and182      Qiu, Xipeng  and183      Radev, Dragomir",184    booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",185    month = jun,186    year = "2021",187    address = "Online",188    publisher = "Association for Computational Linguistics",189    url = "https://aclanthology.org/2021.naacl-main.472",190    doi = "10.18653/v1/2021.naacl-main.472",191    pages = "5905--5921"192}"""193 194_NARRATIVE_QA_CITATION = r"""195@article{kocisky-etal-2018-narrativeqa,196    title = "The {N}arrative{QA} Reading Comprehension Challenge",197    author = "Ko{\v{c}}isk{\'y}, Tom{\'a}{\v{s}}  and198      Schwarz, Jonathan  and199      Blunsom, Phil  and200      Dyer, Chris  and201      Hermann, Karl Moritz  and202      Melis, G{\'a}bor  and203      Grefenstette, Edward",204    journal = "Transactions of the Association for Computational Linguistics",205    volume = "6",206    year = "2018",207    address = "Cambridge, MA",208    publisher = "MIT Press",209    url = "https://aclanthology.org/Q18-1023",210    doi = "10.1162/tacl_a_00023",211    pages = "317--328"212}"""213 214_GOV_REPORT_CITATION = r"""215@inproceedings{huang-etal-2021-efficient,216    title = "Efficient Attentions for Long Document Summarization",217    author = "Huang, Luyang  and218      Cao, Shuyang  and219      Parulian, Nikolaus  and220      Ji, Heng  and221      Wang, Lu",222    booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",223    month = jun,224    year = "2021",225    address = "Online",226    publisher = "Association for Computational Linguistics",227    url = "https://aclanthology.org/2021.naacl-main.112",228    doi = "10.18653/v1/2021.naacl-main.112",229    pages = "1419--1436"230}"""231 232_QUALITY_CITATION = """\233@inproceedings{pang-etal-2022-quality,234    title = "{Q}u{ALITY}: Question Answering with Long Input Texts, Yes!",235    author = "Pang, Richard Yuanzhe  and236      Parrish, Alicia  and237      Joshi, Nitish  and238      Nangia, Nikita  and239      Phang, Jason  and240      Chen, Angelica  and241      Padmakumar, Vishakh  and242      Ma, Johnny  and243      Thompson, Jana  and244      He, He  and245      Bowman, Samuel",246    booktitle = "Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",247    month = jul,248    year = "2022",249    address = "Seattle, United States",250    publisher = "Association for Computational Linguistics",251    url = "https://aclanthology.org/2022.naacl-main.391",252    doi = "10.18653/v1/2022.naacl-main.391",253    pages = "5336--5358"254}255"""256_SQUALITY_CITATION = """\257@inproceedings{wang-etal-2022-squality,258    title = "{SQ}u{ALITY}: Building a Long-Document Summarization Dataset the Hard Way",259    author = "Wang, Alex  and260      Pang, Richard Yuanzhe  and261      Chen, Angelica  and262      Phang, Jason  and263      Bowman, Samuel R.",264    booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing",265    month = dec,266    year = "2022",267    address = "Abu Dhabi, United Arab Emirates",268    publisher = "Association for Computational Linguistics",269    url = "https://aclanthology.org/2022.emnlp-main.75",270    pages = "1139--1156"271}272"""273_MUSIQUE_CITATION = """\274@article{trivedi-etal-2022-musique,275    title = "♫ {M}u{S}i{Q}ue: Multihop Questions via Single-hop Question Composition",276    author = "Trivedi, Harsh  and277      Balasubramanian, Niranjan  and278      Khot, Tushar  and279      Sabharwal, Ashish",280    journal = "Transactions of the Association for Computational Linguistics",281    volume = "10",282    year = "2022",283    address = "Cambridge, MA",284    publisher = "MIT Press",285    url = "https://aclanthology.org/2022.tacl-1.31",286    doi = "10.1162/tacl_a_00475",287    pages = "539--554"288}289"""290 291_SPACE_CITATION = """\292@article{angelidis-etal-2021-extractive,293    title = "Extractive Opinion Summarization in Quantized Transformer Spaces",294    author = "Angelidis, Stefanos  and295      Amplayo, Reinald Kim  and296      Suhara, Yoshihiko  and297      Wang, Xiaolan  and298      Lapata, Mirella",299    journal = "Transactions of the Association for Computational Linguistics",300    volume = "9",301    year = "2021",302    address = "Cambridge, MA",303    publisher = "MIT Press",304    url = "https://aclanthology.org/2021.tacl-1.17",305    doi = "10.1162/tacl_a_00366",306    pages = "277--293"307}308"""309_BOOK_SUM_CITATION = """\310@inproceedings{kryscinski-etal-2022-booksum,311    title = "{BOOKSUM}: A Collection of Datasets for Long-form Narrative Summarization",312    author = "Kryscinski, Wojciech  and313      Rajani, Nazneen  and314      Agarwal, Divyansh  and315      Xiong, Caiming  and316      Radev, Dragomir",317    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2022",318    month = dec,319    year = "2022",320    address = "Abu Dhabi, United Arab Emirates",321    publisher = "Association for Computational Linguistics",322    url = "https://aclanthology.org/2022.findings-emnlp.488",323    pages = "6536--6558"324}325"""326 327FEATURE_TO_TYPE = {328    "id": "string",329    "pid": "string",330    "input": "string",331    "output": "string",332    "document_start_index": "int32",333    "document_end_index": "int32",334    "query_start_index": "int32",335    "query_end_index": "int32",336    "truncation_seperator": "string"337}338 339 340class ZeroScrollsConfig(datasets.BuilderConfig):341    """BuilderConfig for SCROLLS."""342 343    def __init__(self, features, data_url, citation, url, **kwargs):344        """BuilderConfig for SCROLLS.345        Args:346          features: `list[string]`, list of the features that will appear in the347            feature dict. Should not include "label".348          data_url: `string`, url to download the zip file from.349          citation: `string`, citation for the data set.350          url: `string`, url for information about the data set.351          label_classes: `list[string]`, the list of classes for the label if the352            label is present as a string. Non-string labels will be cast to either353            'False' or 'True'.354          **kwargs: keyword arguments forwarded to super.355        """356        super(ZeroScrollsConfig, self).__init__(version=datasets.Version("1.0.0"), **kwargs)357        self.features = features358        self.data_url = data_url359        self.citation = citation360        self.url = url361 362 363class QualityConfig(ZeroScrollsConfig):364    def __init__(self, **kwargs):365        super().__init__(**kwargs)366        self.hard_only = False367 368 369class ZeroScrolls(datasets.GeneratorBasedBuilder):370    """The ZerpSCROLLS benchmark."""371 372    features = list(FEATURE_TO_TYPE.keys())373    features_with_multiple_inder_docs = features + ["inner_docs_start_indices"]374    DEFAULT_WRITER_BATCH_SIZE = 1000  # because Narrative QA is a rather large dataset375    BUILDER_CONFIGS = [376        ZeroScrollsConfig(377            name="summ_screen_fd",378            description=_SUMM_SCREEN_DESCRIPTION,379            features=features,380            data_url="https://huggingface.co/datasets/tau/zero_scrolls/resolve/main/summ_screen_fd.zip",381            citation=_SUMM_SCREEN_CITATION,382            url="https://github.com/mingdachen/SummScreen",383        ),384        ZeroScrollsConfig(385            name="qasper",386            description=_QASPER_DESCRIPTION,387            features=features,388            data_url="https://huggingface.co/datasets/tau/zero_scrolls/resolve/main/qasper.zip",389            citation=_QASPER_CITATION,390            url="https://allenai.org/project/qasper",391        ),392        ZeroScrollsConfig(393            name="qmsum",394            description=_QMSUM_DESCRIPTION,395            features=features,396            data_url="https://huggingface.co/datasets/tau/zero_scrolls/resolve/main/qmsum.zip",397            citation=_QMSUM_CITATION,398            url="https://github.com/Yale-LILY/QMSum",399        ),400        ZeroScrollsConfig(401            name="narrative_qa",402            description=_NARRATIVE_QA_DESCRIPTION,403            features=features,404            data_url="https://huggingface.co/datasets/tau/zero_scrolls/resolve/main/narrative_qa.zip",405            citation=_NARRATIVE_QA_CITATION,406            url="https://deepmind.com/research/publications/narrativeqa-reading-comprehension-challenge",407        ),408        ZeroScrollsConfig(409            name="gov_report",410            description=_GOV_REPORT_DESCRIPTION,411            features=features,412            data_url="https://huggingface.co/datasets/tau/zero_scrolls/resolve/main/gov_report.zip",413            citation=_GOV_REPORT_CITATION,414            url="https://gov-report-data.github.io/",415        ),416        QualityConfig(417            name="quality",418            description=_QUALITY_DESCRIPTION,419            features=features,420            data_url="https://huggingface.co/datasets/tau/zero_scrolls/resolve/main/quality.zip",421            citation=_QUALITY_CITATION,422            url="https://github.com/nyu-mll/quality",423        ),424        ZeroScrollsConfig(425            name="squality",426            description=_SQUALITY_DESCRIPTION,427            features=features,428            data_url="https://huggingface.co/datasets/tau/zero_scrolls/resolve/main/squality.zip",429            citation=_SQUALITY_CITATION,430            url="https://github.com/nyu-mll/SQuALITY",431        ),432        ZeroScrollsConfig(433            name="musique",434            description=_MUSIQUE_DESCRIPTION,435            features=features_with_multiple_inder_docs,436            data_url="https://huggingface.co/datasets/tau/zero_scrolls/resolve/main/musique.zip",437            citation=_MUSIQUE_CITATION,438            url="https://github.com/stonybrooknlp/musique",439        ),440        ZeroScrollsConfig(441            name="space_digest",442            description=_SPACE_DIGEST_DESCRIPTION,443            features=features_with_multiple_inder_docs,444            data_url="https://huggingface.co/datasets/tau/zero_scrolls/resolve/main/space_digest.zip",445            citation=_SPACE_CITATION,446            url="https://github.com/stangelid/qt",447        ),448        ZeroScrollsConfig(449            name="book_sum_sort",450            description=_BOOK_SUM_DESCRIPTION,451            features=features_with_multiple_inder_docs,452            data_url="https://huggingface.co/datasets/tau/zero_scrolls/resolve/main/book_sum_sort.zip",453            citation=_BOOK_SUM_CITATION,454            url="https://github.com/salesforce/booksum",455        ),456    ]457 458    def _info(self):459 460        features = {feature: datasets.Value(FEATURE_TO_TYPE[feature]) for feature in self.config.features if461                    feature != "inner_docs_start_indices"}462        if "inner_docs_start_indices" in self.config.features:463            features["inner_docs_start_indices"] = datasets.Sequence(datasets.Value("int32"))464 465        return datasets.DatasetInfo(466            description=_ZERO_SCROLLS_DESCRIPTION + self.config.description,467            features=datasets.Features(features),468            homepage=self.config.url,469            citation=self.config.citation + "\n" + _SCROLLS_CITATION + "\n" + _ZERO_SCROLLS_CITATION,470        )471 472    def _split_generators(self, dl_manager):473        dl_dir = dl_manager.download_and_extract(self.config.data_url)474        task_name = _get_task_name_from_data_url(self.config.data_url)475        dl_dir = os.path.join(dl_dir, task_name)476 477        data_files = {} if self.config.data_files is not None else None478        if data_files is not None:479            for split, paths in self.config.data_files.items():480                data_files[split] = paths[0]481 482        return [483            datasets.SplitGenerator(484                name=datasets.Split.VALIDATION,485                gen_kwargs={486                    "data_file": os.path.join(dl_dir, "validation.jsonl"),487                    "split": datasets.Split.VALIDATION,488                },489            ),490            datasets.SplitGenerator(491                name=datasets.Split.TEST,492                gen_kwargs={493                    "data_file": os.path.join(dl_dir, "test.jsonl") if data_files is None else data_files["test"],494                    "split": datasets.Split.TEST,495                },496            ),497        ]498 499    def _generate_examples(self, data_file, split):500        with open(data_file, encoding="utf-8") as f:501            for line in f:502                row = json.loads(line)503 504                if self.config.name == "quality":505                    is_hard = row.pop("is_hard", False)506                    if self.config.hard_only and not is_hard:507                        continue508 509                yield row["pid"], row510 511 512def _get_task_name_from_data_url(data_url):513    return data_url.split("/")[-1].split(".")[0]514