google-research-datasets/sent_comp

1# coding=utf-82# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Google Sentence Compression dataset"""16 17 18import gzip19import json20 21import datasets22 23 24_CITATION = """\25@inproceedings{filippova-altun-2013-overcoming,26    title = "Overcoming the Lack of Parallel Data in Sentence Compression",27    author = "Filippova, Katja  and28      Altun, Yasemin",29    booktitle = "Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing",30    month = oct,31    year = "2013",32    address = "Seattle, Washington, USA",33    publisher = "Association for Computational Linguistics",34    url = "https://www.aclweb.org/anthology/D13-1155",35    pages = "1481--1491",36}37"""38 39_DESCRIPTION = """\40Large corpus of uncompressed and compressed sentences from news articles.41"""42 43_HOMEPAGE = "https://github.com/google-research-datasets/sentence-compression"44 45 46_URLs = {47    datasets.Split.VALIDATION: [48        "https://github.com/google-research-datasets/sentence-compression/raw/master/data/comp-data.eval.json.gz"49    ],50    datasets.Split.TRAIN: [51        f"https://github.com/google-research-datasets/sentence-compression/raw/master/data/sent-comp.train{str(i).zfill(2)}.json.gz"52        for i in range(1, 11)53    ],54}55 56 57class SentComp(datasets.GeneratorBasedBuilder):58    """Google Setence Compression dataset"""59 60    def _info(self):61        node_features = {62            "form": datasets.Value("string"),63            "type": datasets.Value("string"),64            "mid": datasets.Value("string"),65            "word": datasets.features.Sequence(66                {67                    "id": datasets.Value("int32"),68                    "form": datasets.Value("string"),69                    "stem": datasets.Value("string"),70                    "tag": datasets.Value("string"),71                }72            ),73            "gender": datasets.Value("int32"),74            "head_word_index": datasets.Value("int32"),75        }76        compression_edge_features = {77            "parent_id": datasets.Value("int32"),78            "child_id": datasets.Value("int32"),79        }80        edge_features = {**compression_edge_features, "label": datasets.Value("string")}81        entity_features = {82            "start": datasets.Value("int32"),83            "end": datasets.Value("int32"),84            "head": datasets.Value("int32"),85            "name": datasets.Value("string"),86            "type": datasets.Value("string"),87            "mid": datasets.Value("string"),88            "is_proper_name_entity": datasets.Value("bool"),89            "gender": datasets.Value("int32"),90        }91        tree_features = {92            "id": datasets.Value("string"),93            "sentence": datasets.Value("string"),94            "node": datasets.features.Sequence(node_features),95            "edge": datasets.features.Sequence(edge_features),96            "entity_mention": datasets.features.Sequence(entity_features),97        }98        compression_features = {99            "text": datasets.Value("string"),100            "edge": datasets.features.Sequence(compression_edge_features),101        }102 103        return datasets.DatasetInfo(104            description=_DESCRIPTION,105            features=datasets.Features(106                {107                    "graph": tree_features,108                    "compression": compression_features,109                    "headline": datasets.Value("string"),110                    "compression_ratio": datasets.Value("float"),111                    "doc_id": datasets.Value("string"),112                    "source_tree": tree_features,113                    "compression_untransformed": compression_features,114                }115            ),116            supervised_keys=None,117            homepage=_HOMEPAGE,118            citation=_CITATION,119        )120 121    def _split_generators(self, dl_manager):122        """Returns SplitGenerators."""123        return [124            datasets.SplitGenerator(125                name=split,126                # These kwargs will be passed to _generate_examples127                gen_kwargs={"filepaths": dl_manager.download(_URLs[split])},128            )129            for split in _URLs130        ]131 132    def _generate_examples(self, filepaths):133        """Yields examples."""134        id_ = -1135        for ix, filepath in enumerate(filepaths):136            with gzip.open(filepath, mode="rt", encoding="utf-8") as f:137                all_text = f.read()138 139            # in the data file, it's in the form of JSON objects, separated with '\n\n' characters140            # we'll format the file to be able to read with json package141            all_text = "[" + all_text + "]"142            all_text = all_text.replace("}\n\n{", "},\n{")143 144            samples = json.loads(all_text)145            for sample in samples:146                # add some default values147                for node in sample["graph"]["node"] + sample["source_tree"]["node"]:148                    if "type" not in node:149                        node["type"] = ""150                    if "mid" not in node:151                        node["mid"] = ""152 153                id_ += 1154                yield id_, sample155