idsedykh/codebleu

1# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""CodeBLEU metric."""15 16import evaluate17import datasets18 19from .my_codebleu import calc_codebleu20 21 22# TODO: Add BibTeX citation23_CITATION = """\24@InProceedings{huggingface:module,25title = {CodeBLEU: A Metric for Evaluating Code Generation},26authors={Sedykh, Ivan},27year={2022}28}29"""30 31# TODO: Add description of the module here32_DESCRIPTION = """\33This new module is an adaptation of the original CodeBLEU metric from CodexGLUE benchmark 34for evaluating code generation.35"""36 37 38# TODO: Add description of the arguments of the module here39_KWARGS_DESCRIPTION = """40Calculates how good are predictions given some references, using certain scores41Args:42    predictions: list of predictions to score. Each predictions43        should be a string with tokens separated by spaces.44    references: list of lists of references. Each list 45        should contain len(predictions) items.46    lang: programming language in ['java','js','c_sharp','php','go','python','ruby']47    tokenizer: tokenizer function str -> List[str], Defaults to lambda s: s.split()48    params: str, weights for averaging(see CodeBLEU paper). 49        Defaults to equal weights "0.25,0.25,0.25,0.25".50Returns:51    CodeBLEU: resulting score,52    ngram_match_score: See paper CodeBLEU,53    weighted_ngram_match_score: See paper CodeBLEU,54    syntax_match_score: See paper CodeBLEU,55    dataflow_match_score: See paper CodeBLEU,56Examples:57 58    >>> codebleu = evaluate.load("my_new_module")59    >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1])60    >>> print(results)61    {'accuracy': 1.0}62"""63 64# TODO: Define external resources urls if needed65# BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"66 67 68@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)69class codebleu(evaluate.Metric):70    """CodeBLEU metric from CodexGLUE"""71 72    def _info(self):73        # TODO: Specifies the evaluate.EvaluationModuleInfo object74        return evaluate.MetricInfo(75            # This is the description that will appear on the modules page.76            module_type="metric",77            description=_DESCRIPTION,78            citation=_CITATION,79            inputs_description=_KWARGS_DESCRIPTION,80            # This defines the format of each prediction and reference81            features=datasets.Features(82                {83                    "predictions": datasets.Value("string"),84                    "references": datasets.Sequence(datasets.Value("string")),85                }86            ),87            # Homepage of the module for documentation88            homepage="",89            # Additional links to the codebase or references90            codebase_urls=[],91            reference_urls=[92                "https://github.com/microsoft/CodeXGLUE/tree/main/Code-Code/code-to-code-trans/evaluator",93                "https://arxiv.org/abs/2009.10297",94            ],95        )96 97    def _download_and_prepare(self, dl_manager):98        """Optional: download external resources useful to compute the scores"""99        # TODO: Download external resources if needed100        # source CodeBLEU/parser/build.sh101        pass102 103    def _compute(104        self,105        predictions,106        references,107        lang,108        tokenizer=None,109        params="0.25,0.25,0.25,0.25",110    ):111        """Returns the scores"""112        res = calc_codebleu(113            predictions=predictions,114            references=references,115            lang=lang,116            tokenizer=tokenizer,117            params=params,118        )119        return res120