jerin/pib

1# coding=utf-82# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""CVIT PIB Multilingual Corpus"""16 17import datasets18 19 20_CITATION = """\21@inproceedings{siripragada-etal-2020-multilingual,22    title = "A Multilingual Parallel Corpora Collection Effort for {I}ndian Languages",23    author = "Siripragada, Shashank  and24      Philip, Jerin  and25      Namboodiri, Vinay P.  and26      Jawahar, C V",27    booktitle = "Proceedings of the 12th Language Resources and Evaluation Conference",28    month = may,29    year = "2020",30    address = "Marseille, France",31    publisher = "European Language Resources Association",32    url = "https://aclanthology.org/2020.lrec-1.462",33    pages = "3743--3751",34    language = "English",35    ISBN = "979-10-95546-34-4",36}37@article{2020,38   title={Revisiting Low Resource Status of Indian Languages in Machine Translation},39   url={http://dx.doi.org/10.1145/3430984.3431026},40   DOI={10.1145/3430984.3431026},41   journal={8th ACM IKDD CODS and 26th COMAD},42   publisher={ACM},43   author={Philip, Jerin and Siripragada, Shashank and Namboodiri, Vinay P. and Jawahar, C. V.},44   year={2020},45   month={Dec}46}47"""48 49_DESCRIPTION = """\50Sentence aligned parallel corpus between 11 Indian Languages, crawled and extracted from the press information bureau51website.52"""53 54_HOMEPAGE = "http://preon.iiit.ac.in/~jerin/bhasha/"55 56_LICENSE = "Creative Commons Attribution-ShareAlike 4.0 International"57 58_URL = {59    "0.0.0": "http://preon.iiit.ac.in/~jerin/resources/datasets/pib-v0.tar",60    "1.3.0": "http://preon.iiit.ac.in/~jerin/resources/datasets/pib_v1.3.tar.gz",61}62_ROOT_DIR = {63    "0.0.0": "pib",64    "1.3.0": "pib-v1.3",65}66 67_LanguagePairs = [68    "or-ur",69    "ml-or",70    "bn-ta",71    "gu-mr",72    "hi-or",73    "en-or",74    "mr-ur",75    "en-ta",76    "hi-ta",77    "bn-en",78    "bn-or",79    "ml-ta",80    "gu-ur",81    "bn-ml",82    "ml-pa",83    "en-pa",84    "bn-hi",85    "hi-pa",86    "gu-te",87    "pa-ta",88    "hi-ml",89    "or-te",90    "en-ml",91    "en-hi",92    "bn-pa",93    "mr-te",94    "mr-pa",95    "bn-te",96    "gu-hi",97    "ta-ur",98    "te-ur",99    "or-pa",100    "gu-ml",101    "gu-pa",102    "hi-te",103    "en-te",104    "ml-te",105    "pa-ur",106    "hi-ur",107    "mr-or",108    "en-ur",109    "ml-ur",110    "bn-mr",111    "gu-ta",112    "pa-te",113    "bn-gu",114    "bn-ur",115    "ml-mr",116    "or-ta",117    "ta-te",118    "gu-or",119    "en-gu",120    "hi-mr",121    "mr-ta",122    "en-mr",123]124 125 126class PibConfig(datasets.BuilderConfig):127    """BuilderConfig for PIB"""128 129    def __init__(self, language_pair, version=datasets.Version("1.3.0"), **kwargs):130        super().__init__(version=version, **kwargs)131        """132 133        Args:134            language_pair: language pair, you want to load135            **kwargs: keyword arguments forwarded to super.136        """137        self.src, self.tgt = language_pair.split("-")138 139 140class Pib(datasets.GeneratorBasedBuilder):141    """This new dataset is the large scale sentence aligned corpus in 11 Indian languages, viz.142    CVIT-PIB corpus that is the largest multilingual corpus available for Indian languages.143    """144 145    BUILDER_CONFIG_CLASS = PibConfig146    BUILDER_CONFIGS = [PibConfig(name=pair, description=_DESCRIPTION, language_pair=pair) for pair in _LanguagePairs]147 148    def _info(self):149        return datasets.DatasetInfo(150            description=_DESCRIPTION,151            features=datasets.Features(152                {"translation": datasets.features.Translation(languages=[self.config.src, self.config.tgt])}153            ),154            supervised_keys=(self.config.src, self.config.tgt),155            homepage=_HOMEPAGE,156            license=_LICENSE,157            citation=_CITATION,158        )159 160    def _split_generators(self, dl_manager):161        archive = dl_manager.download(_URL[str(self.config.version)])162        return [163            datasets.SplitGenerator(164                name=datasets.Split.TRAIN,165                gen_kwargs={166                    "archive": dl_manager.iter_archive(archive),167                },168            ),169        ]170 171    def _generate_examples(self, archive):172        root_dir = _ROOT_DIR[str(self.config.version)]173        data_dir = f"{root_dir}/{self.config.src}-{self.config.tgt}"174        src = tgt = None175        for path, file in archive:176            if data_dir in path:177                if f"{data_dir}/train.{self.config.src}" in path:178                    src = file.read().decode("utf-8").split("\n")[:-1]179                if f"{data_dir}/train.{self.config.tgt}" in path:180                    tgt = file.read().decode("utf-8").split("\n")[:-1]181            if src and tgt:182                break183        for idx, (s, t) in enumerate(zip(src, tgt)):184            yield idx, {"translation": {self.config.src: s, self.config.tgt: t}}185