yzha/ctc_eval

1# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""TODO: Add a description here."""15 16from typing import final17import evaluate18import datasets19 20 21# TODO: Add BibTeX citation22_CITATION = """\23@inproceedings{deng2021compression,24  title={Compression, Transduction, and Creation: A Unified Framework for Evaluating Natural Language Generation},25  author={Deng, Mingkai and Tan, Bowen and Liu, Zhengzhong and Xing, Eric and Hu, Zhiting},26  booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},27  pages={7580--7605},28  year={2021}29}30"""31 32# TODO: Add description of the module here33_DESCRIPTION = """\34This repo contains code of an automatic evaluation metric described in the paper35Compression, Transduction, and Creation: A Unified Framework for Evaluating Natural Language Generation36"""37 38 39# TODO: Add description of the arguments of the module here40_KWARGS_DESCRIPTION = """41Calculates how good are predictions given some references, using certain scores42Args:43    predictions: List of texts (Hypothesis) to score. The list now only supports one piece of text44    references: List of texts (Premise) to score. The list now only supports one piece of text45Returns:46    ctc_score: The CTC score47Examples:48    >>> ctc_score = evaluate.load("yzha/ctc_eval")49    >>> results = ctc_score.compute(references=['hello world'], predictions=['hi world'])50    >>> print(results)51    {'ctc_score': 0.5211202502250671}52"""53 54# TODO: Define external resources urls if needed55BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"56 57 58@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)59class CTC_Eval(evaluate.EvaluationModule):60    """TODO: Short description of my evaluation module."""61 62    def _info(self):63        # TODO: Specifies the evaluate.EvaluationModuleInfo object64        return evaluate.EvaluationModuleInfo(65            # This is the description that will appear on the modules page.66            module_type="metric",67            description=_DESCRIPTION,68            citation=_CITATION,69            inputs_description=_KWARGS_DESCRIPTION,70            # This defines the format of each prediction and reference71            features=datasets.Features({72                'predictions': datasets.Value('large_string'),73                'references': datasets.Value('large_string'),74            }),75            # Homepage of the module for documentation76            homepage="https://github.com/tanyuqian/ctc-gen-eval",77            # Additional links to the codebase or references78            codebase_urls=["https://github.com/tanyuqian/ctc-gen-eval"],79            reference_urls=["https://github.com/tanyuqian/ctc-gen-eval"]80        )81 82    def _download_and_prepare(self, dl_manager):83        """Optional: download external resources useful to compute the scores"""84        # TODO: Download external resources if needed85        import nltk86        nltk.download('stopwords')87        import subprocess88        import sys89 90        def install(package):91            subprocess.check_call([sys.executable, "-m", "pip", "install", package])92        93        94        try:95            from ctc_score import StyleTransferScorer, SummarizationScorer, DialogScorer96        except:97            print('ctc package is not installed. installing...')98            install('ctc-score')99 100        if self.config_name == 'default':101            self.config_name = 'D-cnndm,consistency'102 103        model_name, self.aspect = self.config_name.split(',')104        if self.aspect in ['consistency', 'relevance']:105            self.scorer = SummarizationScorer(align=model_name, device='cpu')106        elif self.aspect in ['preservation']:107            self.scorer = StyleTransferScorer(align=model_name)108        elif self.aspect in ['engagingness', 'groundedness']:109            self.scorer = DialogScorer(align=model_name)110 111        print(self.compute(references=['hello world'], predictions=['hi world']))112        113 114    def _compute(self, predictions, references):115        """Returns the scores"""116        # TODO: Compute the different scores of the module117        assert len(predictions) == len(references)118        print('computing...')119        print(predictions)120        print(references)121        ctc_score = self.scorer.score(doc=references[0], refs=[], hypo=predictions[0], aspect=self.aspect)122        return {123            "ctc_score": ctc_score124        }