BridgeAI-Lab/Sem-nCG
1
1# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""15Sem-NCG metric16Author: Naman Bansal17"""18 19import statistics20from dataclasses import dataclass21from typing import List, Tuple, Union22 23import datasets24import evaluate25import nltk26import numpy as np27from sklearn.metrics.pairwise import cosine_similarity28from tqdm import tqdm29 30from .encoder_models import get_sbert_encoder, get_encoder31from .type_aliases import DEVICE_TYPE, NDArray, DOCUMENT_TYPE32from .utils import get_gpu, flatten_list, slice_embeddings, is_nested_list_of_type, \33 tokenize_and_prep_document34 35_CITATION = """\36@inproceedings{akter-etal-2022-revisiting,37 title = "Revisiting Automatic Evaluation of Extractive Summarization Task: Can We Do Better than {ROUGE}?",38 author = "Akter, Mousumi and39 Bansal, Naman and40 Karmaker, Shubhra Kanti",41 editor = "Muresan, Smaranda and42 Nakov, Preslav and43 Villavicencio, Aline",44 booktitle = "Findings of the Association for Computational Linguistics: ACL 2022",45 month = may,46 year = "2022",47 address = "Dublin, Ireland",48 publisher = "Association for Computational Linguistics",49 url = "https://aclanthology.org/2022.findings-acl.122",50 doi = "10.18653/v1/2022.findings-acl.122",51 pages = "1547--1560",52 abstract = "It has been the norm for a long time to evaluate automated summarization tasks using the popular ROUGE metric. Although several studies in the past have highlighted the limitations of ROUGE, researchers have struggled to reach a consensus on a better alternative until today. One major limitation of the traditional ROUGE metric is the lack of semantic understanding (relies on direct overlap of n-grams). In this paper, we exclusively focus on the extractive summarization task and propose a semantic-aware nCG (normalized cumulative gain)-based evaluation metric (called Sem-nCG) for evaluating this task. One fundamental contribution of the paper is that it demonstrates how we can generate more reliable semantic-aware ground truths for evaluating extractive summarization tasks without any additional human intervention. To the best of our knowledge, this work is the first of its kind. We have conducted extensive experiments with this new metric using the widely used CNN/DailyMail dataset. Experimental results show that the new Sem-nCG metric is indeed semantic-aware, shows higher correlation with human judgement (more reliable) and yields a large number of disagreements with the original ROUGE metric (suggesting that ROUGE often leads to inaccurate conclusions also verified by humans).",53}54"""55 56_DESCRIPTION = """\57Sem-nCG (Semantic Normalized Cumulative Gain) Metric evaluates the quality of predicted sentences 58(abstractive/extractive) in relation to reference sentences and documents using Semantic Normalized Cumulative Gain 59(NCG). It computes gain values and NCG scores based on cosine similarity between sentence embeddings, leveraging a 60Sentence-BERT encoder. This metric is designed to assess the relevance and ranking of predicted sentences, making it 61useful for tasks such as summarization and information retrieval.62"""63 64_KWARGS_DESCRIPTION = """65Sem-nCG (Semantic Normalized Cumulative Gain) compares the system-generated summaries (predictions) with ground truth 66reference summaries (references) and input documents (documents) using Semantic Normalized Cumulative Gain (NCG). 67It computes gain values and NCG scores based on sentence embeddings.68 69Args:70 predictions (DOCUMENT_TYPE): The predicted sentences. 71 `tokenize_sentences`=True -> predictions: List[str]72 `tokenize_sentences`=False -> predictions: List[List[str]]73 references (DOCUMENT_TYPE): The reference sentences.74 `tokenize_sentences`=True -> references: List[str]75 `tokenize_sentences`=False -> references: List[List[str]]76 documents (DOCUMENT_TYPE): Input documents.77 `tokenize_sentences`=True -> documents: List[str]78 `tokenize_sentences`=False -> documents: List[List[str]]79 k (int): The rank threshold used for evaluating gains (typically top-k sentences). Default is 3.80 gpu (Union[bool, str, int, List[Union[str, int]]]): Whether to use GPU or CPU for computation.81 bool - 82 False - CPU (Default)83 True - GPU (device 0) if gpu is available else CPU84 int - 85 n - GPU, device index n86 str - 87 'cuda', 'gpu', 'cpu' 88 List[Union[str, int]] - Multiple GPUs/cpus i.e. use multiple processes when computing embeddings89 batch_size (int): Batch size for encoding. Default is 32.90 verbose (bool): Flag to indicate verbose output. Default is False.91 tokenize_sentences (bool): Flag to indicate whether to tokenize the sentences in the input documents. Default: True.92 pre_compute_embeddings (bool): Flag to indicate whether to pre-compute embeddings for all sentences. This speeds up 93 computation but requires more memory. Default is False.94 debug (bool): Flag to return detailed debug information including ranked gains. Default is False.95 96Returns:97 Union[Tuple[float, List[float]], Tuple[float, List[RankedGains]]]:98 If `debug` is False, returns a tuple containing the mean SemnCG score and a list of SemnCG scores for each document.99 If `debug` is True, returns a tuple containing the mean SemnCG score and a list of `RankedGains` objects with 100 detailed gain information for each document.101 102Examples of input formats:103 104Case 1: tokenize_sentences = True105 predictions: List[str] - List of predictions where each prediction is a document.106 references: List[str] - List of references where each reference is a document.107 documents: List[str] - List of input documents where each document is a document.108 Example:109 predictions = ["This is a prediction sentence 1. This is a prediction sentence 2."]110 references = ["This is a reference sentence 1. This is a reference sentence 2."]111 documents = ["This is a document sentence 1. This is a document sentence 2."]112 113Case 2: tokenize_sentences = False114 predictions: List[List[str]] - List of predictions where each prediction is a list of sentences.115 references: List[List[str]] - List of references where each reference is a list of sentences.116 documents: List[List[str]] - List of input documents where each document is a list of sentences.117 Example:118 predictions = [["This is a prediction sentence 1.", "This is a prediction sentence 2."]]119 references = [["This is a reference sentence 1.", "This is a reference sentence 2."]]120 documents = [["This is a document sentence 1.", "This is a document sentence 2."]]121 122Examples:123 124 >>> import evaluate125 >>> predictions = ["This is a prediction sentence 1. This is a prediction sentence 2."]126 >>> references = ["This is a reference sentence 1. This is a reference sentence 2."]127 >>> documents = ["This is a document sentence 1. This is a document sentence 2."]128 >>> metric = evaluate.load("nbansal/semncg", model_name="all-MiniLM-L6-v2") 129 >>> mean_score, scores = metric.compute(predictions=predictions, references=references, documents=documents)130 >>> print(f"Mean SemnCG: {mean_score}")131"""132 133 134@dataclass135class RankedGains:136 """137 Dataclass to store ranked gains and associated metadata.138 139 Attributes:140 gt_gains (List[Tuple[str, float]]): List of tuples representing ground truth (ideal) gains,141 where each tuple contains a document sentence and its corresponding gain value.142 pred_gains (List[Tuple[str, float]]): List of tuples representing predicted gains by the model,143 where each tuple contains a document identifier and its corresponding gain value.144 k (int): The rank threshold used for evaluating gains (typically top-k documents).145 ncg (float): Normalized Cumulative Gain (NCG) score calculated based on the predicted gains146 compared to the ground truth gains.147 148 Notes:149 - `gt_gains` and `pred_gains` are typically sorted in descending order150 - `k` specifies the top-k threshold used for evaluating the gains.151 - `ncg` provides a normalized measure of the model's performance.152 """153 gt_gains: List[Tuple[str, float]]154 pred_gains: List[Tuple[str, float]]155 k: int156 ncg: float157 158 159def compute_cosine_similarity(doc_embeds: NDArray, ref_embeds: NDArray) -> List[float]:160 """161 Compute cosine similarity scores between each document embedding and reference embeddings.162 163 Args:164 doc_embeds (NDArray): 2D array of shape (#Docs, Embedding_dim) containing document embeddings.165 ref_embeds (NDArray): 2D array of shape (#Refs, Embedding_dim) containing reference embeddings.166 167 Returns:168 List[float]: A list of mean cosine similarity scores between each document and reference embeddings.169 The length of the list is equal to the number of documents (#Docs).170 171 Notes:172 - Uses cosine_similarity function from sklearn.metrics.pairwise to compute pairwise cosine similarities.173 - Returns the mean cosine similarity scores across reference embeddings for each document embedding.174 """175 # Compute cosine similarity between predicted and reference embeddings176 cosine_scores = cosine_similarity(doc_embeds, ref_embeds) # [#Docs, #Refs]177 return np.mean(cosine_scores, axis=1).tolist()178 179 180def compute_gain(sim_scores: List[float]) -> List[Tuple[int, float]]:181 """182 Compute gain values for ranked similarity scores.183 184 Args:185 sim_scores (List[float]): List of similarity scores for documents (`compute_cosine_similarity(doc_embeds, ref_embeds)`)186 187 Returns:188 List[Tuple[int, float]]: A list of tuples where each tuple contains a document index and its corresponding gain 189 value. The list is sorted by descending order of gain values.190 191 Notes:192 - Computes gain values based on the rank order of similarity scores, where higher scores indicate higher gains.193 - Uses the formula: gain = rank_position / sum of ranks, where rank_position starts from 1 for the highest score194 - Returns a list sorted by descending gain values.195 """196 count = len(sim_scores)197 sim_scores = np.array(sim_scores).argsort()[::-1] # Reverse Sorted Order of doc sentence indices198 denominator = count * (count + 1) / 2 # (n * (n+1))/2199 return [(s_idx, val / denominator) for s_idx, val in zip(sim_scores, range(count, 0, -1))]200 201 202def score_ncg(model_relevance: List[float], gt_relevance: List[float]) -> float:203 """204 Calculate the Normalized Cumulative Gain (NCG) score based on model relevance and ground truth relevance.205 206 Args:207 model_relevance (List[float]): List of gain values representing the relevance scores predicted by the model.208 gt_relevance (List[float]): List of gain values representing the ground truth (ideal) relevance scores.209 210 Returns:211 float: Normalized Cumulative Gain (NCG) score, which measures the effectiveness of the model's relevance212 predictions compared to the ideal relevance scores. The score ranges from 0 to 1, where higher values213 indicate better performance.214 215 Notes:216 - Calculates Cumulative Gain (CG) for both model and ground truth relevance lists.217 - Normalizes CG scores by dividing model CG by ground truth CG to get the NCG score.218 - Returns 0 if the ground truth CG (icg) is 0 to avoid division by zero.219 """220 221 # CG score222 cg = sum(model_relevance)223 224 # ICG score225 icg = sum(gt_relevance)226 227 # Normalized CG score228 return cg / icg if icg != 0 else 0229 230 231def compute_ncg(pred_gains: List[Tuple[int, float]], gt_gains: List[Tuple[int, float]], k: int) -> float:232 """233 Compute the Normalized Cumulative Gain (NCG) score based on predicted and ground truth gains up to rank k.234 235 Args:236 pred_gains (List[Tuple[int, float]]): List of tuples representing predicted gains by the model,237 where each tuple contains a document position (or index) and its corresponding gain value. 238 (Sorted in Descending Order)239 gt_gains (List[Tuple[int, float]]): List of tuples representing ground truth gains (ideal gains),240 where each tuple contains a document position (or index) and its corresponding gain value. 241 (Sorted in Descending Order)242 k (int): The rank threshold used for evaluating gains (typically top-k documents).243 244 Returns:245 float: Normalized Cumulative Gain (NCG) score based on the predicted gains compared to the ground truth gains.246 247 Notes:248 - Both `pred_gains` and `gt_gains` should be sorted lists (in descending order) where higher gain values indicate249 higher relevance.250 - The function calculates NCG up to rank `k`, considering only the top-k documents.251 - Uses the `score_ncg` function to compute the NCG score based on the model's predicted gains and the ground252 truth.253 """254 gt_dict = dict(gt_gains)255 gt_rel = [v for _, v in gt_gains[:k]]256 model_rel = [gt_dict[position] for position, _ in pred_gains[:k]]257 return score_ncg(model_rel, gt_rel)258 259 260def _validate_input_format(261 tokenize_sentences: bool,262 predictions: DOCUMENT_TYPE,263 references: DOCUMENT_TYPE,264 documents: DOCUMENT_TYPE265):266 """267 Validate the format of predictions, references, and documents based on specified criteria.268 269 Args:270 tokenize_sentences (bool): Flag indicating whether sentences should be tokenized.271 predictions (DOCUMENT_TYPE): Predictions to validate.272 references (DOCUMENT_TYPE): References to validate.273 documents (DOCUMENT_TYPE): Documents to validate.274 275 Raises:276 ValueError: If the format of predictions, references, or documents does not meet the specified criteria.277 278 Validation Criteria:279 The function validates predictions, references, and documents based on the following conditions:280 1. If `tokenize_sentences` is True:281 - Predictions, references, and documents must all be lists of strings (`is_list_of_strings_at_depth(obj, 1)`).282 283 2. If `tokenize_sentences` is False:284 - Predictions, references, and documents must all be lists of lists of strings285 (`is_list_of_strings_at_depth(obj, 2)`).286 287 The function checks these conditions and raises a ValueError if any condition is not met,288 indicating that predictions, references, or documents are not in the valid input format.289 290 Notes:291 - `DOCUMENT_TYPE`: Union[List[str], List[List[str]]]292 - Uses helper function `is_list_of_strings_at_depth` to validate the format of lists of strings.293 294 Example:295 >>> tokenize_sentences = True296 >>> predictions = ["This is prediction 1.", "This is prediction 2."]297 >>> references = ["Reference for prediction 1.", "Reference for prediction 2."]298 >>> documents = ["Document 1 content.", "Document 2 content."]299 >>> _validate_input_format(tokenize_sentences, predictions, references, documents)300 301 Example:302 >>> tokenize_sentences = False303 >>> predictions = [["Sentence 1 in prediction 1.", "Sentence 2 in prediction 1."],304 >>> ["Sentence 1 in prediction 2.", "Sentence 2 in prediction 2."]]305 >>> references = [["Sentences in reference 1."], ["Sentences in reference 2."]]306 >>> documents = [["Sentence 1 in document 1.", "Sentence 2 in document 1."],307 >>> ["Sentence 1 in document 2.", "Sentence 2 in document 2."]]308 >>> _validate_input_format(tokenize_sentences, predictions, references, documents)309 """310 if not (len(predictions) == len(references) == len(documents)):311 raise ValueError(312 f"Predictions, References and Documents must have the same length. "313 f"Got {len(predictions)} predictions, {len(references)} references and {len(documents)} documents."314 )315 316 if len(predictions) == 0:317 raise ValueError("Can't have empty inputs")318 319 def check_format(lst_obj, expected_depth: int, name: str):320 is_valid, error_message = is_nested_list_of_type(lst_obj, element_type=str, depth=expected_depth)321 if not is_valid:322 raise ValueError(f"{name} are not in the expected format.\n"323 f"Error: {error_message}.")324 325 try:326 if tokenize_sentences:327 check_format(predictions, expected_depth=1, name="predictions")328 check_format(references, expected_depth=1, name="references")329 check_format(documents, expected_depth=1, name="documents")330 else:331 check_format(predictions, expected_depth=2, name="predictions")332 check_format(references, expected_depth=2, name="references")333 check_format(documents, expected_depth=2, name="documents")334 except ValueError as ve:335 raise ValueError(f"Input validation error: {ve}")336 337 338@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)339class SemNCG(evaluate.Metric):340 """341 SemnCG (Semantic Normalized Cumulative Gain) Metric.342 343 This metric evaluates the quality of predicted sentences in relation to reference sentences and documents344 using Semantic Normalized Cumulative Gain (NCG). It computes the gain values and NCG scores based on345 cosine similarity between sentence embeddings, leveraging a Sentence-BERT encoder.346 """347 348 def __init__(self, model_name: str = "all-MiniLM-L6-v2", **kwargs):349 self.sbert_encoder = get_sbert_encoder(model_name)350 super().__init__(**kwargs)351 352 def _info(self):353 # TODO: Specifies the evaluate.EvaluationModuleInfo object354 return evaluate.MetricInfo(355 # This is the description that will appear on the modules page.356 module_type="metric",357 description=_DESCRIPTION,358 citation=_CITATION,359 inputs_description=_KWARGS_DESCRIPTION,360 # This defines the format of each prediction and reference361 features=[362 # Tokenize_Sentences = True363 datasets.Features(364 {365 "predictions": datasets.Value("string"),366 "references": datasets.Value("string"),367 "documents": datasets.Value("string"),368 }369 ),370 # Tokenize_Sentences = False371 datasets.Features(372 {373 "predictions": datasets.Sequence(datasets.Value("string", id="sequence"), id="predictions"),374 "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"),375 "documents": datasets.Sequence(datasets.Value("string", id="sequence"), id="documents"),376 }377 ),378 ],379 # # Homepage of the module for documentation380 # homepage="http://module.homepage",381 # # Additional links to the codebase or references382 # codebase_urls=["http://github.com/path/to/codebase/of/new_module"],383 reference_urls=["https://aclanthology.org/2022.findings-acl.122/"]384 )385 386 def _download_and_prepare(self, dl_manager):387 """Optional: download external resources useful to compute the scores"""388 nltk.download("punkt", quiet=True)389 390 def _compute(391 self,392 predictions: DOCUMENT_TYPE,393 references: DOCUMENT_TYPE,394 documents: DOCUMENT_TYPE,395 k: int = 3,396 gpu: DEVICE_TYPE = False,397 verbose: bool = False,398 batch_size: int = 32,399 tokenize_sentences: bool = True,400 pre_compute_embeddings: bool = False,401 debug: bool = False,402 ) -> Union[Tuple[float, List[float]], Tuple[float, List[RankedGains]]]:403 """404 Compute the Semantic Normalized Cumulative Gain (SemnCG) score.405 406 Args:407 predictions (DOCUMENT_TYPE): The predicted sentences. 408 `tokenize_sentences`=True -> predictions: List[str]409 `tokenize_sentences`=False -> predictions: List[List[str]]410 references (DOCUMENT_TYPE): The reference sentences.411 `tokenize_sentences`=True -> references: List[str]412 `tokenize_sentences`=False -> references: List[List[str]]413 documents (DOCUMENT_TYPE): Input documents.414 `tokenize_sentences`=True -> references: List[str]415 `tokenize_sentences`=False -> references: List[List[str]]416 k (int, optional): The rank threshold used for evaluating gains (typically top-k sentences). Default is 3.417 gpu (DEVICE_TYPE, optional): Whether to use GPU for computation. Default is False.418 verbose (bool, optional): Whether to print verbose logs and use a progress bar. Default is False.419 batch_size (int, optional): The batch size for encoding sentences. Default is 32.420 tokenize_sentences (bool, optional): Whether to tokenize sentences. If True, sentences are tokenized before421 processing. Default is True.422 pre_compute_embeddings (bool, optional): Whether to pre-compute embeddings for all sentences. This speeds up423 computation but requires more memory. Default is False.424 debug (bool, optional): Whether to return detailed debug information including ranked gains. Default=False.425 426 Returns:427 Union[Tuple[float, List[float]], Tuple[float, List[RankedGains]]]:428 If `debug` is False, returns a tuple containing the mean SemnCG score and a list of SemnCG scores for each document.429 If `debug` is True, returns a tuple containing the mean SemnCG score and a list of `RankedGains` objects with detailed gain information for each document.430 431 Raises:432 ValueError: If the format of predictions, references, or documents does not meet the specified criteria.433 434 Notes:435 - Validates the format of predictions, references, and documents based on `tokenize_sentences`.436 - Computes embeddings using a Sentence-BERT encoder.437 - Computes cosine similarity between document, reference, and prediction embeddings.438 - Calculates gain values and Normalized Cumulative Gain (NCG) scores.439 - Optionally returns detailed debug information for each document if `debug` is True.440 """441 442 # Validate inputs corresponding to flags443 _validate_input_format(tokenize_sentences, predictions, references, documents)444 445 try:446 N = len(predictions)447 except Exception as e:448 N = None449 450 # Get GPU451 device = get_gpu(gpu)452 if verbose:453 print(f"Using devices: {device}")454 455 # Get model456 encoder = get_encoder(self.sbert_encoder, device=device, batch_size=batch_size, verbose=verbose)457 458 if pre_compute_embeddings: # fast but takes more memory459 predictions = [tokenize_and_prep_document(pred, tokenize_sentences) for pred in predictions]460 references = [tokenize_and_prep_document(ref, tokenize_sentences) for ref in references]461 documents = [tokenize_and_prep_document(doc, tokenize_sentences) for doc in documents]462 463 # This is only done for debug case464 sent_tokenized_documents = documents465 466 # Compute All Embeddings467 all_sentences = flatten_list(documents) + flatten_list(references) + flatten_list(predictions)468 embeddings = encoder.encode(all_sentences)469 470 prediction_sentences_count = [len(pred) for pred in predictions]471 reference_sentences_count = [len(ref) for ref in references]472 document_sentences_count = [len(doc) for doc in documents]473 474 # Get embeddings corresponding to documents, references and predictions (IN ORDER)475 doc_embeddings = slice_embeddings(embeddings, document_sentences_count)476 ref_embeddings = slice_embeddings(embeddings[sum(document_sentences_count):], reference_sentences_count)477 pred_embeddings = slice_embeddings(478 embeddings[sum(document_sentences_count + reference_sentences_count):], prediction_sentences_count479 )480 481 iterable_obj = zip(pred_embeddings, ref_embeddings, doc_embeddings)482 483 else:484 iterable_obj = zip(predictions, references, documents)485 486 out = []487 for idx, (pred, ref, doc) in tqdm(488 enumerate(iterable_obj), 489 total=N,490 disable=not verbose):491 492 if not pre_compute_embeddings: # Compute embeddings493 ref_sentences = tokenize_and_prep_document(ref, tokenize_sentences)494 pred_sentences = tokenize_and_prep_document(pred, tokenize_sentences)495 doc_sentences = tokenize_and_prep_document(doc, tokenize_sentences)496 497 # Compute Embeddings498 doc_sentence_count = len(doc_sentences)499 ref_sentence_count = len(ref_sentences)500 all_sentences = doc_sentences + ref_sentences + pred_sentences501 embeddings = encoder.encode(all_sentences)502 doc_embeddings = embeddings[:doc_sentence_count]503 ref_embeddings = embeddings[doc_sentence_count:doc_sentence_count + ref_sentence_count]504 pred_embeddings = embeddings[doc_sentence_count + ref_sentence_count:]505 else: # we already have embeddings506 doc_embeddings = doc507 ref_embeddings = ref508 pred_embeddings = pred509 510 doc_sentences = sent_tokenized_documents[idx]511 512 # Compute Pair-Wise Cosine Similarity513 ref_sim_scores = compute_cosine_similarity(doc_embeddings, ref_embeddings)514 pred_sim_scores = compute_cosine_similarity(doc_embeddings, pred_embeddings)515 516 # Compute Gains517 ground_truth_gain = compute_gain(ref_sim_scores)518 519 # this is used to compute top-predicted sentence indices520 pred_gain = compute_gain(pred_sim_scores)521 real_k = min(len(pred_gain), k)522 523 # Compute NCG Scores524 ncg_score = compute_ncg(pred_gain, ground_truth_gain, real_k)525 526 if debug:527 ground_truth_gain = [(doc_sentences[sent_idx], gain_val) for sent_idx, gain_val in ground_truth_gain]528 pred_gain = [(doc_sentences[sent_idx], gain_val) for sent_idx, gain_val in pred_gain]529 out.append(RankedGains(ground_truth_gain, pred_gain, k=real_k, ncg=ncg_score))530 else:531 out.append(ncg_score)532 533 if debug:534 return statistics.mean([ele.ncg for ele in out]), out535 536 return statistics.mean(out), out537