idsedykh/codebleu2

1# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""TODO: Add a description here."""15 16import evaluate17import datasets18 19 20# TODO: Add BibTeX citation21_CITATION = """\22@InProceedings{huggingface:module,23title = {A great new module},24authors={huggingface, Inc.},25year={2020}26}27"""28 29# TODO: Add description of the module here30_DESCRIPTION = """\31This new module is designed to solve this great ML task and is crafted with a lot of care.32"""33 34 35# TODO: Add description of the arguments of the module here36_KWARGS_DESCRIPTION = """37Calculates how good are predictions given some references, using certain scores38Args:39    predictions: list of predictions to score. Each predictions40        should be a string with tokens separated by spaces.41    references: list of reference for each prediction. Each42        reference should be a string with tokens separated by spaces.43Returns:44    accuracy: description of the first score,45    another_score: description of the second score,46Examples:47    Examples should be written in doctest format, and should illustrate how48    to use the function.49 50    >>> my_new_module = evaluate.load("my_new_module")51    >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1])52    >>> print(results)53    {'accuracy': 1.0}54"""55 56# TODO: Define external resources urls if needed57BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"58 59 60@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)61class codebleu2(evaluate.Metric):62    """TODO: Short description of my evaluation module."""63 64    def _info(self):65        # TODO: Specifies the evaluate.EvaluationModuleInfo object66        return evaluate.MetricInfo(67            # This is the description that will appear on the modules page.68            module_type="metric",69            description=_DESCRIPTION,70            citation=_CITATION,71            inputs_description=_KWARGS_DESCRIPTION,72            # This defines the format of each prediction and reference73            features=datasets.Features({74                'predictions': datasets.Value('int64'),75                'references': datasets.Value('int64'),76            }),77            # Homepage of the module for documentation78            homepage="http://module.homepage",79            # Additional links to the codebase or references80            codebase_urls=["http://github.com/path/to/codebase/of/new_module"],81            reference_urls=["http://path.to.reference.url/new_module"]82        )83 84    def _download_and_prepare(self, dl_manager):85        """Optional: download external resources useful to compute the scores"""86        # TODO: Download external resources if needed87        pass88 89    def _compute(self, predictions, references):90        """Returns the scores"""91        # TODO: Compute the different scores of the module92        accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)93        return {94            "accuracy": accuracy,95        }