asoroa/bsbasque

1# coding=utf-82# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""TODO: Add a description here."""16 17import datasets18import gzip19 20logger = datasets.logging.get_logger(__name__)21 22# TODO: Add BibTeX citation23# Find for instance the citation on arxiv or on the dataset repo/website24_CITATION = """\25@InProceedings{huggingface:dataset,26title = {A great new dataset},27author={huggingface, Inc.28},29year={2020}30}31"""32 33_VERSION= "1.1.0"34 35# TODO: Add description of the dataset here36# You can copy an official description37_DESCRIPTION = """\38BSBasque dataset. The text is extracted from the following domains:39 40 https://www.berria.eus41 https://eu.wikipedia.org42 https://goiena.eus43 https://www.argia.eus44 https://goierri.hitza.eus45 46"""47 48# TODO: Add a link to an official homepage for the dataset here49_HOMEPAGE = "to.be.announced.eus"50 51# TODO: Add the licence for the dataset here if you can find it52_LICENSE = "CC BY-SA 4.0"53 54_BASE_DATA_URL_STR = (55    "https://ixa2.si.ehu.es/~ccpsoeta/bsbasque/"56)57_BASE_CHECKSUM_FILE_NAME = "bsbasque_sha256.txt"58 59class BsBasqueConfig(datasets.BuilderConfig):60    "BsBasque corpus."61 62    def __init__(self, **kwargs):63        """BuilderConfig for BsBasque.64 65        Args:66            **kwargs: Keyword arguments forwarded to super.67        """68 69        # Initialize the base class.70        name = "bsbasque"71        description = "BsBasque dataset"72        super(BsBasqueConfig, self).__init__(name=name, description=description, **kwargs)73 74        # Additional attributes75        self.base_data_url = _BASE_DATA_URL_STR76 77 78# TODO: Name of the dataset usually match the script name with CamelCase instead of snake_case79class BSBasque(datasets.GeneratorBasedBuilder):80    """TODO: Short description of my dataset."""81 82    BUILDER_CONFIGS = [83        BsBasqueConfig(version=datasets.Version(_VERSION))84    ]85    BUILDER_CONFIG_CLASS = BsBasqueConfig86 87    def _info(self):88        # TODO: This method specifies the datasets.DatasetInfo object which contains informations and typings for the dataset89        return datasets.DatasetInfo(90            description=_DESCRIPTION,91            features=datasets.Features({"id": datasets.Value("int64"), "text": datasets.Value("string")}),92            supervised_keys=None,93            homepage=_HOMEPAGE,94            license=_LICENSE,95            citation=_CITATION,96        )97 98    def _split_generators(self, dl_manager):99        checksum_url = self.config.base_data_url + _BASE_CHECKSUM_FILE_NAME100        checksum_file = dl_manager.download(checksum_url)101        with open(checksum_file, encoding="utf-8") as f:102            data_filenames = [line.split("\t")[0] for line in f if line]103            data_urls = [self.config.base_data_url + data_filename for data_filename in data_filenames]104        downloaded_files = dl_manager.download(data_urls)105        return [106            datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepaths": downloaded_files}),107        ]108 109    def _generate_examples(self, filepaths):110        """This function returns the examples in the raw (text) form by iterating on all the files."""111        id_ = 0112        for filepath in filepaths:113            logger.info("generating examples from = %s", filepath)114            with gzip.open(filepath, "rt") as f:115                for line in f:116                    feature = id_, {"id": id_, "text": "".join(line).rstrip()}117                    yield feature118                    id_ += 1119