mainlp/aed_sst

1# coding=utf-82# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16"""17"""18import random19from typing import List, Tuple, Dict20from pathlib import Path21import json22 23import datasets24 25_CITATION = """\26@inproceedings{hemphill-etal-1990-atis,27    title = "The {ATIS} Spoken Language Systems Pilot Corpus",28    author = "Hemphill, Charles T.  and29      Godfrey, John J.  and30      Doddington, George R.",31    booktitle = "Speech and Natural Language: Proceedings of a Workshop Held at Hidden Valley, {P}ennsylvania, June 24-27,1990",32    year = "1990",33    url = "https://aclanthology.org/H90-1021",34}35 36@article{10.1162/coli_a_00464,37    author = {Klie, Jan-Christoph and Webber, Bonnie and Gurevych, Iryna},38    title = "{Annotation Error Detection: Analyzing the Past and Present for a More Coherent Future}",39    journal = {Computational Linguistics},40    pages = {1-42},41    year = {2022},42    month = {11},43    abstract = "{Annotated data is an essential ingredient in natural language processing for training and evaluating machine learning models. It is therefore very desirable for the annotations to be of high quality. Recent work, however, has shown that several popular datasets contain a surprising number of annotation errors or inconsistencies. To alleviate this issue, many methods for annotation error detection have been devised over the years. While researchers show that their approaches work well on their newly introduced datasets, they rarely compare their methods to previous work or on the same datasets. This raises strong concerns on methods’ general performance and makes it difficult to asses their strengths and weaknesses. We therefore reimplement 18 methods for detecting potential annotation errors and evaluate them on 9 English datasets for text classification as well as token and span labeling. In addition, we define a uniform evaluation setup including a new formalization of the annotation error detection task, evaluation protocol and general best practices. To facilitate future research and reproducibility, we release our datasets and implementations in an easy-to-use and open source software package.}",44    issn = {0891-2017},45    doi = {10.1162/coli_a_00464},46    url = {https://doi.org/10.1162/coli\_a\_00464},47    eprint = {https://direct.mit.edu/coli/article-pdf/doi/10.1162/coli\_a\_00464/2057485/coli\_a\_00464.pdf},48}49"""50 51_DATASETNAME = "aed_atis"52 53_DESCRIPTION = """\54This dataset is designed for Annotation Error Detection.55"""56 57_HOMEPAGE = ""58 59_LICENSE = ""60 61_URLS = {62    _DATASETNAME: "https://raw.githubusercontent.com/howl-anderson/ATIS_dataset/master/data/standard_format/rasa/train.json",63}64 65_SOURCE_VERSION = "1.0.0"66 67_SCHEMA = datasets.Features({68    "id": datasets.Value("string"),69    "text": datasets.Value("string"),70    "label": datasets.Value("string"),71    "true_label": datasets.Value("string"),72})73 74 75class AED_ATIS(datasets.GeneratorBasedBuilder):76    _VERSION = datasets.Version(_SOURCE_VERSION)77 78    BUILDER_CONFIGS = [79        datasets.BuilderConfig(80           name="aed_atis_5"81        ),82        datasets.BuilderConfig(83            name="small_aed_atis_5"84        ),85        datasets.BuilderConfig(86            name="aed_atis_10"87        ),88        datasets.BuilderConfig(89            name="small_aed_atis_10"90        ),91    ]92    DEFAULT_CONFIG_NAME = "aed_atis_5"93 94    def _info(self) -> datasets.DatasetInfo:95        return datasets.DatasetInfo(96            description=_DESCRIPTION,97            features=_SCHEMA,98            supervised_keys=None,99            homepage=_HOMEPAGE,100            citation=_CITATION,101            license=_LICENSE,102        )103 104    def _split_generators(self, dl_manager) -> List[datasets.SplitGenerator]:105        """Returns SplitGenerators."""106        urls = _URLS[_DATASETNAME]107        data_path = dl_manager.download_and_extract(urls)108 109        return [110            datasets.SplitGenerator(111                name=datasets.Split.TRAIN,112                # Whatever you put in gen_kwargs will be passed to _generate_examples113                gen_kwargs={114                    "data_path": Path(data_path),115                },116            ),117        ]118 119    # method parameters are unpacked from `gen_kwargs` as given in `_split_generators`120 121    def _generate_examples(self, data_path: Path) -> Tuple[int, Dict]:122        """Yields examples as (key, example) tuples."""123        with data_path.open() as f:124            data = json.load(f)125 126 127        examples = data["rasa_nlu_data"]["common_examples"]128 129        random.seed(42)130        if self.config.name.startswith("small_"):131            examples = random.sample(examples, 500)132 133        random.seed(42)134        noise_level = int(self.config.name.split("_")[-1]) / 100135        noise_indices = random.sample(list(range(len(examples))), int(len(examples) * noise_level))136        label_set = sorted(set(i["intent"] for i in examples))137        for i, example in enumerate(examples):138            if i in noise_indices:139                label = random.choice(label_set)140            else:141                label = example["intent"]142 143            yield (i, {144                "id": str(i),145                "text": example["text"],146                "label": label,147                "true_label": example["intent"]148            })149