RollingMuffin/test_scripts

1# coding=utf-82# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Lyrics dataset parsed from Genius"""16 17 18import csv19import json20import os21import gzip22 23import datasets24 25 26_CITATION = """\27@InProceedings{huggingartists:dataset,28title = {Lyrics dataset},29author={Aleksey Korshuk30},31year={2021}32}33"""34 35 36_DESCRIPTION = """\37This dataset is designed to generate lyrics with HuggingArtists.38"""39 40# Add a link to an official homepage for the dataset here41_HOMEPAGE = "https://github.com/AlekseyKorshuk/huggingartists"42 43# Add the licence for the dataset here if you can find it44_LICENSE = "All rights belong to copyright holders"45 46_URL = "https://huggingface.co/datasets/RollingMuffin/test_scripts/raw/main/testScripts.json"47 48# Name of the dataset49class LyricsDataset(datasets.GeneratorBasedBuilder):50    """Lyrics dataset"""51 52    VERSION = datasets.Version("1.0.0")53 54    def _info(self):55        # This method specifies the datasets.DatasetInfo object which contains informations and typings for the dataset56        features = datasets.Features(57                {58                    "text": datasets.Value("string"),59                }60            )61        return datasets.DatasetInfo(62            # This is the description that will appear on the datasets page.63            description=_DESCRIPTION,64            # This defines the different columns of the dataset and their types65            features=features,  # Here we define them above because they are different between the two configurations66            # If there's a common (input, target) tuple from the features,67            # specify them here. They'll be used if as_supervised=True in68            # builder.as_dataset.69            supervised_keys=None,70            # Homepage of the dataset for documentation71            homepage=_HOMEPAGE,72            # License for the dataset if available73            license=_LICENSE,74            # Citation for the dataset75            citation=_CITATION,76        )77 78    def _split_generators(self, dl_manager):79        """Returns SplitGenerators."""80        # This method is tasked with downloading/extracting the data and defining the splits depending on the configuration81        # If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name82 83        # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLs84        # It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files.85        # By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive86 87        data_dir = dl_manager.download_and_extract(_URL)88        return [89            datasets.SplitGenerator(90                name=datasets.Split.TRAIN,91                # These kwargs will be passed to _generate_examples92                gen_kwargs={93                    "filepath": data_dir,94                    "split": "train",95                },96            ),97        ]98        99 100    def _generate_examples(self, filepath, split):101        """Yields examples as (key, example) tuples."""102        # This method handles input defined in _split_generators to yield (key, example) tuples from the dataset.103        104        with open(filepath, encoding="utf-8") as f:105            data = json.load(f)106            for id, pred in enumerate(data[split]):107                yield id, {"text": pred}