RealTimeData/bbc_alltime

1import datasets2import json3import os4import sys5 6dl = datasets.DownloadManager()7configs_file = dl.download('https://huggingface.co/datasets/RealTimeData/bbc_alltime/raw/main/configs.txt')8 9with open(configs_file, encoding="utf-8") as f:10    _TIMES = f.read().splitlines()11 12_TIMES += ['all']13 14_CITATION = """\15@misc{li2023estimating,16      title={Estimating Contamination via Perplexity: Quantifying Memorisation in Language Model Evaluation}, 17      author={Yucheng Li},18      year={2023},19      eprint={2309.10677},20      archivePrefix={arXiv},21      primaryClass={cs.CL}22}23"""24 25_DESCRIPTION = """\26This dataset contains BBC News articles from 2017 to 2022. The articles are arraged by month. Access the specific month by using the format "YYYY-MM" as config. Such as load_dataset("RealTimeData/bbc_alltime", "2021-1").27"""28 29_HOMEPAGE = "https://github.com/liyucheng09/Contamination_Detector"30 31class Bbc_alltimes(datasets.GeneratorBasedBuilder):32 33    BUILDER_CONFIGS = [34        datasets.BuilderConfig(35            name=time, version=datasets.Version("1.0.0"), description=f"BBC News articles published in the priod of {time}"36        )37        for time in _TIMES38    ]39 40    def _info(self):41        features = datasets.Features(42            {43                "title": datasets.Value("string"),44                "published_date": datasets.Value("string"),45                "authors": datasets.Value("string"),46                "description": datasets.Value("string"),47                "section": datasets.Value("string"),48                "content": datasets.Value("string"),49                "link": datasets.Value("string"),50            }51        )52        return datasets.DatasetInfo(53            description=_DESCRIPTION,54            features=features,55            homepage=_HOMEPAGE,56            citation=_CITATION,57        )58 59    def _split_generators(self, dl_manager):60        """Returns SplitGenerators."""61        if self.config.name == "all":62            times = _TIMES[:-1]63            files = dl_manager.download([f"articles/{time}.json" for time in _TIMES ])64            return [65                datasets.SplitGenerator(66                    name=datasets.Split.TRAIN,67                    gen_kwargs={"files": files},68                )69            ]70        else:71            time = self.config.name72            _URL = f"articles/{time}.json"73            file = dl_manager.download(_URL)74            return [75                datasets.SplitGenerator(76                    name=datasets.Split.TRAIN,77                    gen_kwargs={"files": file},78                )79            ]80 81    def _generate_examples(self, files):82        """Yields examples."""83        if self.config.name == "all":84            assert isinstance(files, list)85            for file in files:86                time = file.strip('.json')87                with open(file, encoding="utf-8") as f:88                    data = json.load(f)89                length = len(data['title'])90                for i in range(length):91                    yield f'{time}-{i}', {92                        "title": data['title'][i],93                        "published_date": data['published_date'][i],94                        "authors": data['authors'][i],95                        "description": data['description'][i],96                        "section": data['section'][i],97                        "content": data['content'][i],98                        "link": data['link'][i],99                    }100        else:101            assert isinstance(files, str)102            time = self.config.name103            with open(files, encoding="utf-8") as f:104                data = json.load(f)105            length = len(data['title'])106            for i in range(length):107                yield f'{time}-{i}', {108                    "title": data['title'][i],109                    "published_date": data['published_date'][i],110                    "authors": data['authors'][i],111                    "description": data['description'][i],112                    "section": data['section'][i],113                    "content": data['content'][i],114                    "link": data['link'][i],115                }