cffl/Exploring_Intelligent_Writing_Assistance

1# ###########################################################################2#3#  CLOUDERA APPLIED MACHINE LEARNING PROTOTYPE (AMP)4#  (C) Cloudera, Inc. 20225#  All rights reserved.6#7#  Applicable Open Source License: Apache 2.08#9#  NOTE: Cloudera open source products are modular software products10#  made up of hundreds of individual components, each of which was11#  individually copyrighted.  Each Cloudera open source product is a12#  collective work under U.S. Copyright Law. Your license to use the13#  collective work is as provided in your written agreement with14#  Cloudera.  Used apart from the collective work, this file is15#  licensed for your use pursuant to the open source license16#  identified above.17#18#  This code is provided to you pursuant a written agreement with19#  (i) Cloudera, Inc. or (ii) a third-party authorized to distribute20#  this code. If you do not have a written agreement with Cloudera nor21#  with an authorized and properly licensed third party, you do not22#  have any rights to access nor to use this code.23#24#  Absent a written agreement with Cloudera, Inc. (“Cloudera”) to the25#  contrary, A) CLOUDERA PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY26#  KIND; (B) CLOUDERA DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED27#  WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT LIMITED TO28#  IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND29#  FITNESS FOR A PARTICULAR PURPOSE; (C) CLOUDERA IS NOT LIABLE TO YOU,30#  AND WILL NOT DEFEND, INDEMNIFY, NOR HOLD YOU HARMLESS FOR ANY CLAIMS31#  ARISING FROM OR RELATED TO THE CODE; AND (D)WITH RESPECT TO YOUR EXERCISE32#  OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, CLOUDERA IS NOT LIABLE FOR ANY33#  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR34#  CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO, DAMAGES35#  RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF36#  BUSINESS ADVANTAGE OR UNAVAILABILITY, OR LOSS OR CORRUPTION OF37#  DATA.38#39# ###########################################################################40 41from typing import List, Union42 43import torch44import numpy as np45from pyemd import emd46from transformers import pipeline47 48 49class StyleIntensityClassifier:50    """51    Utility for classifying style and calculating Style Transfer Intensity between52    two pieces of text (i.e. input and output of TST model).53 54    This custom evaluation metric aims to quantify the magnitude of transferred55    style between two texts. To accomplish this, we pass input and output texts56    through a trained style classifier to produce two distributions. We then57    utilize Earth Movers Distance (EMD) to calculate the minimum "cost"/"work"58    required to turn the input distribution into the output distribution. This59    metric allows us to capture a more nuanced, per-example measure of style60    transfer when compared to simply aggregating binary classifications over61    records in a dataset.62 63    Attributes:64        model_identifier (str)65 66    """67 68    def __init__(self, model_identifier: str):69        self.model_identifier = model_identifier70        self.device = torch.cuda.current_device() if torch.cuda.is_available() else -171        self._build_pipeline()72 73    def _build_pipeline(self):74 75        self.pipeline = pipeline(76            task="text-classification",77            model=self.model_identifier,78            device=self.device,79            return_all_scores=True,80        )81 82    def score(self, input_text: Union[str, List[str]]):83        """84        Classify a given input text using the model initialized by the class.85 86        Args:87            input_text (`str` or `List[str]`) - Input text for classification88 89        Returns:90            classification (dict) - a dictionary containing the label, score, and91                distribution between classes92 93        """94        if isinstance(input_text, str):95            tmp = list()96            tmp.append(input_text)97            input_text = tmp98 99        result = self.pipeline(input_text)100        distributions = np.array(101            [[label["score"] for label in item] for item in result]102        )103        return [104            {105                "label": self.pipeline.model.config.id2label[scores.argmax()],106                "score": round(scores.max(), 4),107                "distribution": scores.tolist(),108            }109            for scores in distributions110        ]111 112    def calculate_transfer_intensity(113        self, input_text: List[str], output_text: List[str], target_class_idx: int = 1114    ) -> List[float]:115        """116        Calcualates the style transfer intensity (STI) between two pieces of text.117 118        Args:119            input_text (list) - list of input texts with indicies corresponding120                to counterpart in output_text121            ouptput_text (list) - list of output texts with indicies corresponding122                to counterpart in input_text123            target_class_idx (int) - index of the target style class used for directional124                score correction125 126        Returns:127            A list of floats with corresponding style transfer intensity scores.128 129        """130 131        if len(input_text) != len(output_text):132            raise ValueError(133                "input_text and output_text must be of same length with corresponding items"134            )135 136        input_dist = [item["distribution"] for item in self.score(input_text)]137        output_dist = [item["distribution"] for item in self.score(output_text)]138 139        return [140            self.calculate_emd(input_dist[i], output_dist[i], target_class_idx)141            for i in range(len(input_dist))142        ]143 144    def calculate_transfer_intensity_fraction(145        self, input_text: List[str], output_text: List[str], target_class_idx: int = 1146    ) -> List[float]:147        """148        Calcualates the style transfer intensity (STI) _fraction_ between two pieces of text.149        See `calcualte_sti_fraction()` for details.150 151        Args:152            input_text (list) - list of input texts with indicies corresponding153                to counterpart in output_text154            ouptput_text (list) - list of output texts with indicies corresponding155                to counterpart in input_text156            target_class_idx (int) - index of the target style class used for directional157                score correction158 159        Returns:160            A list of floats with corresponding style transfer intensity scores.161 162        """163 164        if len(input_text) != len(output_text):165            raise ValueError(166                "input_text and output_text must be of same length with corresponding items"167            )168 169        input_dist = [item["distribution"] for item in self.score(input_text)]170        output_dist = [item["distribution"] for item in self.score(output_text)]171 172        return [173            self.calculate_sti_fraction(174                input_dist[i],175                output_dist[i],176                ideal_dist=[0.0, 1.0],177                target_class_idx=target_class_idx,178            )179            for i in range(len(input_dist))180        ]181 182    def calculate_sti_fraction(183        self, input_dist, output_dist, ideal_dist=[0.0, 1.0], target_class_idx=1184    ):185        """186        Calculate the direction-corrected style transfer intensity fraction between187        two style distributions of equal length.188 189        If output_dist moves closer towards target style class, the metric represents the percentage of190        the possible _target_ style distribution that was captured during the transfer. If output_dist191        moves further from the target style class, the metric represents the percentage of the possible192        _source_ style distribution that was captured.193 194        Args:195            input_dist (list) - probabilities assigned to the style classes196                from the input text to style transfer model197            output_dist (list) - probabilities assigned to the style classes198                from the outut text of the style transfer model199            ideal_dist (list, optional): The maximum possibly distribution. Defaults to [0.0, 1.0].200            target_class_idx (int, optional)201 202        Returns:203            sti_fraction (float)204        """205 206        sti = self.calculate_emd(input_dist, output_dist, target_class_idx)207 208        if sti > 0:209            potential = self.calculate_emd(input_dist, ideal_dist, target_class_idx)210        else:211            potential = self.calculate_emd(212                input_dist, ideal_dist[::-1], target_class_idx213            )214 215        return sti / potential216 217    @staticmethod218    def calculate_emd(input_dist, output_dist, target_class_idx):219        """220        Calculate the direction-corrected Earth Mover's Distance (aka Wasserstein distance)221        between two distributions of equal length. Here we penalize the EMD score if222        the output text style moved further away from the target style.223 224        Reference: https://github.com/passeul/style-transfer-model-evaluation/blob/master/code/style_transfer_intensity.py225 226        Args:227            input_dist (list) - probabilities assigned to the style classes228                from the input text to style transfer model229            output_dist (list) - probabilities assigned to the style classes230                from the outut text of the style transfer model231 232        Returns:233            emd (float) - Earth Movers Distance between the two distributions234 235        """236 237        N = len(input_dist)238        distance_matrix = np.ones((N, N))239        dist = emd(np.array(input_dist), np.array(output_dist), distance_matrix)240 241        transfer_direction_correction = (242            1 if output_dist[target_class_idx] >= input_dist[target_class_idx] else -1243        )244 245        return round(dist * transfer_direction_correction, 4)246