baber/logiqa2

1# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""LogiQA dataset."""15 16import datasets17import json18import ast19 20_CITATION = """\21@ARTICLE{10174688,22  author={Liu, Hanmeng and Liu, Jian and Cui, Leyang and Teng, Zhiyang and Duan, Nan and Zhou, Ming and Zhang, Yue},23  journal={IEEE/ACM Transactions on Audio, Speech, and Language Processing},24  title={LogiQA 2.0 — An Improved Dataset for Logical Reasoning in Natural Language Understanding},25  year={2023},26  volume={},27  number={},28  pages={1-16},29  doi={10.1109/TASLP.2023.3293046}}30"""31 32_DESCRIPTION = """\33The dataset is an amendment and re-annotation of LogiQA in 2020, a large-scale logical reasoning reading comprehension dataset adapted from the Chinese Civil Service Examination. We increase the data size, refine the texts with manual translation by professionals, and improve the quality by removing items with distinctive cultural features like Chinese idioms. Furthermore, we conduct a fine-grained annotation on the dataset and turn it into a two-way natural language inference (NLI) task, resulting in 35k premise-hypothesis pairs with gold labels, making it the first large-scale NLI dataset for complex logical reasoning34"""35 36_HOMEPAGE = "https://github.com/csitfun/LogiQA2.0/tree/main"37 38_LICENSE = (39    "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License"40)41 42_URLS = {43    "logiqa2": {44        "train": "https://raw.githubusercontent.com/csitfun/LogiQA2.0/main/logiqa/DATA/LOGIQA/train.txt",45        "validation": "https://raw.githubusercontent.com/csitfun/LogiQA2.0/main/logiqa/DATA/LOGIQA/dev.txt",46        "test": "https://raw.githubusercontent.com/csitfun/LogiQA2.0/main/logiqa/DATA/LOGIQA/test.txt",47    },48    "logiqa2_zh": {49        "train": "https://raw.githubusercontent.com/csitfun/LogiQA2.0/main/logiqa/DATA/LOGIQA/train_zh.txt",50        "validation": "https://raw.githubusercontent.com/csitfun/LogiQA2.0/main/logiqa/DATA/LOGIQA/dev_zh.txt",51        "test": "https://raw.githubusercontent.com/csitfun/LogiQA2.0/main/logiqa/DATA/LOGIQA/test_zh.txt",52    },53    "logiqa2_nli": {54        "train": "https://raw.githubusercontent.com/csitfun/LogiQA2.0/main/logiqa2nli/DATA/QA2NLI/train.txt",55        "validation": "https://raw.githubusercontent.com/csitfun/LogiQA2.0/main/logiqa2nli/DATA/QA2NLI/dev.txt",56        "test": "https://raw.githubusercontent.com/csitfun/LogiQA2.0/main/logiqa2nli/DATA/QA2NLI/test.txt",57    },58    "logieval": {59        "train": "https://raw.githubusercontent.com/csitfun/LogiEval/main/Data/logiqa_ood.jsonl",60        "test": "https://raw.githubusercontent.com/csitfun/LogiEval/main/Data/logiqa.jsonl",61    },62}63 64 65class LogiQA2(datasets.GeneratorBasedBuilder):66    """TODO: Short description of my dataset."""67 68    VERSION = datasets.Version("2.0.0")69 70    # This is an example of a dataset with multiple configurations.71    # If you don't want/need to define several sub-sets in your dataset,72    # just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.73 74    # If you need to make complex sub-parts in the datasets with configurable options75    # You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig76    # BUILDER_CONFIG_CLASS = MyBuilderConfig77 78    # You will be able to load one or the other configurations in the following list with79    # data = datasets.load_dataset('my_dataset', 'first_domain')80    # data = datasets.load_dataset('my_dataset', 'second_domain')81    BUILDER_CONFIGS = [82        datasets.BuilderConfig(83            name="logiqa2",84            version=VERSION,85            description="The LogiQA multiple answer dataset translated in English from Chinese.",86        ),87        datasets.BuilderConfig(88            name="logiqa2_zh",89            version=VERSION,90            description="The original LogiQA multiple answer dataset in Chinese.",91        ),92        datasets.BuilderConfig(93            name="logiqa2_nli",94            version=VERSION,95            description="The NLI part of LogiQA2.0 dataset",96        ),97        datasets.BuilderConfig(98            name="logieval",99            version=VERSION,100            description="Instruction based MRC task",101        ),102    ]103    DEFAULT_CONFIG_NAME = "logiqa2"104 105    def _info(self):106 107        if self.config.name == "logiqa2_zh":108            features = datasets.Features(109                {110                    "answer": datasets.Value("int32"),111                    "text": datasets.Value("string"),112                    "question": datasets.Value("string"),113                    "options": datasets.features.Sequence(datasets.Value("string")),114                }115            )116        #  # major_premise (maybe minor) is sometimes str, sometimes list117        #  # can't get it to work.118        elif self.config.name == "logiqa2_nli":119            features = datasets.Features(120                {121                    "label": datasets.ClassLabel(122                        num_classes=2,123                        names=["not entailed", "entailed"],124                        names_file=None,125                        id=None,126                    ),127                    "major_premise": datasets.features.Sequence(128                        datasets.Value("string")129                    ),130                    "minor_premise": datasets.Value("string"),131                    "conclusion": datasets.Value("string"),132                }133            )134        elif self.config.name in ("logiqa2_nli", "logieval"):135            features = datasets.Features(136                {"content": datasets.Value("string"), "ideal": datasets.Value("string")}137            )138        else:139            features = datasets.Features(140                {141                    "id": datasets.Value("int32"),142                    "answer": datasets.Value("int32"),143                    "text": datasets.Value("string"),144                    # "type" is a dict with arbitrary keys and values145                    "type": datasets.Value("string"),146                    "question": datasets.Value("string"),147                    "options": datasets.features.Sequence(datasets.Value("string")),148                }149            )150        return datasets.DatasetInfo(151            description=_DESCRIPTION,152            features=features,153            homepage=_HOMEPAGE,154            license=_LICENSE,155            citation=_CITATION,156        )157 158    def _split_generators(self, dl_manager):159        _urls = _URLS[self.config.name]160        urls = {161            "train": _urls["train"],162            "test": _urls["test"],163        }164        if "validation" in _urls:165            urls["validation"] = _urls["validation"]166        data_dir = dl_manager.download_and_extract(urls)167        splits = [168            datasets.SplitGenerator(169                name=datasets.Split.TRAIN,170                # These kwargs will be passed to _generate_examples171                gen_kwargs={172                    "filepath": data_dir["train"],173                    "split": "train",174                },175            ),176            datasets.SplitGenerator(177                name=datasets.Split.TEST,178                # These kwargs will be passed to _generate_examples179                gen_kwargs={"filepath": data_dir["test"], "split": "test"},180            ),181        ]182        if "validation" in _urls:183            splits.append(184                datasets.SplitGenerator(185                    name=datasets.Split.VALIDATION,186                    # These kwargs will be passed to _generate_examples187                    gen_kwargs={188                        "filepath": data_dir["validation"],189                        "split": "validation",190                    },191                )192            )193        return splits194 195    def _generate_examples(self, filepath, split):196        with open(filepath, encoding="utf-8") as f:197            for key, row in enumerate(f):198                data = json.loads(row)199 200                if self.config.name == "logiqa2_zh":201                    yield key, {202                        "answer": data["answer"],203                        "text": data["text"],204                        "question": data["question"],205                        "options": data["options"],206                    }207                elif self.config.name == "logiqa2_nli":208                    if isinstance(data["major_premise"], str):209                        data["major_premise"] = [data["major_premise"]]210                    data["minor_premise"] = data["minor_premise"].strip()211                    yield key, {212                        "label": data["label"],213                        "major_premise": data["major_premise"],214                        "minor_premise": data["minor_premise"],215                        "conclusion": data["conclusion"],216                    }217                elif self.config.name == "logieval":218                    yield key, {219                        "content": data["input"][1]["content"],220                        "ideal": data["ideal"],221                    }222                else:223                    yield key, {224                        "id": data["id"],225                        "answer": data["answer"],226                        "text": data["text"].strip(),227                        "type": data["type"],228                        "question": data["question"].strip(),229                        "options": [x.strip() for x in data["options"]],230                    }231