symanto/absa_evaluator
0
1from typing import Dict, List, Set2 3import evaluate4from datasets import Features, Sequence, Value5from sklearn.metrics import accuracy_score6from itertools import chain7from random import choice8from typing import Any, Dict, List, Optional, Tuple9 10 11_CITATION = """12"""13 14_DESCRIPTION = """15This module provides evaluation metrics for Aspect-Based Sentiment Analysis (ABSA). 16The metrics include precision, recall, and F1 score for both aspect terms and category detection.17Additionally it calculates the accuracy for polarities from aspect terms and category detection.18ABSA evaluates the capability of a model to identify and correctly classify the sentiment of specific aspects within a text.19"""20 21_KWARGS_DESCRIPTION = """22Computes precision, recall, and F1 score for aspect terms and category detection in Aspect-Based Sentiment Analysis (ABSA). Also calculates de accuracy for polarities on each task.23 24Args:25 predictions: List of ABSA predictions with the following structure:26 - 'aspects': Sequence of aspect annotations, each with the following keys:27 - 'term': Aspect term28 - 'polarity': Polarity of the aspect term29 - 'category': Sequence of category annotations, each with the following keys:30 - 'category': Category31 - 'polarity': polarity of the category32 references: List of ABSA references with the same structure as predictions.33 34Examples for predictions:35 [36 {37 "aspects": [38 {"term": "battery life", "polarity": "positive"},39 {"term": "camera", "polarity": "negative"}40 ],41 "category": [42 {"category": "Battery", "polarity": "positive"},43 {"category": "Camera", "polarity": "negative"}44 ]45 }46 ]47 48Returns:49 term_extraction_results: f1 score, precision and recall for aspect terms50 term_polarity_results_accuracy: accuracy for polarities on aspect terms51 category_detection_results: f1 score, precision and recall for category detection52 category_polarity_results_accuracy: accuracy for polarities on categories53"""54 55 56class AbsaEvaluator(evaluate.Metric):57 def _info(self):58 return evaluate.MetricInfo(59 description=_DESCRIPTION,60 citation=_CITATION,61 inputs_description=_KWARGS_DESCRIPTION,62 features=Features(63 {64 "predictions": Features(65 {66 "aspects": Features(67 {68 "term": Sequence(Value("string")),69 "polarity": Sequence(Value("string")),70 }71 ),72 "category": Features(73 {74 "category": Sequence(Value("string")),75 "polarity": Sequence(Value("string")),76 }77 ),78 }79 ),80 "references": Features(81 {82 "aspects": Features(83 {84 "term": Sequence(Value("string")),85 "polarity": Sequence(Value("string")),86 }87 ),88 "category": Features(89 {90 "category": Sequence(Value("string")),91 "polarity": Sequence(Value("string")),92 }93 ),94 }95 ),96 }97 ),98 )99 100 def _compute(self, predictions, references):101 # preprocess aspect term102 (103 truth_aspect_terms,104 pred_aspect_terms,105 truth_term_polarities,106 pred_term_polarities,107 ) = absa_term_preprocess(108 references=references,109 predictions=predictions,110 subtask_key="aspects",111 subtask_value="term",112 )113 # evaluate114 term_results = self.semeval_metric(115 truth_aspect_terms, pred_aspect_terms116 )117 term_polarity_acc = accuracy_score(118 truth_term_polarities, pred_term_polarities119 )120 121 # preprocess category detection122 (123 truth_categories,124 pred_categories,125 truth_cat_polarities,126 pred_cat_polarities,127 ) = absa_term_preprocess(128 references=references,129 predictions=predictions,130 subtask_key="category",131 subtask_value="category",132 )133 134 # evaluate135 category_results = self.semeval_metric(136 truth_categories, pred_categories137 )138 cat_polarity_acc = accuracy_score(139 truth_cat_polarities, pred_cat_polarities140 )141 142 return {143 "term_extraction_results": term_results,144 "term_polarity_results_accuracy": term_polarity_acc,145 "category_detection_results": category_results,146 "category_polarity_results_accuracy": cat_polarity_acc,147 }148 149 def semeval_metric(150 self, truths: List[List[str]], preds: List[List[str]]151 ) -> Dict[str, float]:152 """153 Implements evaluation for extraction tasks using precision, recall, and F1 score.154 155 Parameters:156 - truths: List of lists, where each list contains the ground truth labels for a sample.157 - preds: List of lists, where each list contains the predicted labels for a sample.158 159 Returns:160 - A dictionary containing the precision, recall, F1 score, and counts of common, retrieved, and relevant.161 162 link for this code: https://github.com/davidsbatista/Aspect-Based-Sentiment-Analysis/blob/1d9c8ec1131993d924e96676fa212db6b53cb870/libraries/baselines.py#L387163 """164 b = 1165 common, relevant, retrieved = 0.0, 0.0, 0.0166 for truth, pred in zip(truths, preds):167 common += len([a for a in pred if a in truth])168 retrieved += len(pred)169 relevant += len(truth)170 precision = common / retrieved if retrieved > 0 else 0.0171 recall = common / relevant if relevant > 0 else 0.0172 f1 = (173 (1 + (b**2))174 * precision175 * recall176 / ((precision * b**2) + recall)177 if precision > 0 and recall > 0178 else 0.0179 )180 return {181 "precision": precision,182 "recall": recall,183 "f1_score": f1,184 "common": common,185 "retrieved": retrieved,186 "relevant": relevant,187 }188 189def adjust_predictions(190 refs: List[List[Any]], preds: List[List[Any]], choices: Set[Any]191) -> List[List[Any]]:192 """Adjust predictions to match the length of references with either a special token or random choice."""193 choices_list = list(choices)194 adjusted_preds = []195 for ref, pred in zip(refs, preds):196 if len(pred) < len(ref):197 missing_count = len(ref) - len(pred)198 pred.extend([choice(choices_list) for _ in range(missing_count)])199 elif len(pred) > len(ref):200 pred = pred[:len(ref)]201 adjusted_preds.append(pred)202 return adjusted_preds203 204 205def extract_aspects(206 data: List[Dict[str, Dict[str, Any]]], specific_key: str, specific_val: str207) -> List[List[Any]]:208 """Extracts and returns a list of specified aspect details from the nested 'aspects' data."""209 return [item[specific_key][specific_val] for item in data]210 211 212def absa_term_preprocess(213 references: List[Dict[str, Any]],214 predictions: List[Dict[str, Any]],215 subtask_key: str,216 subtask_value: str,217) -> Tuple[List[str], List[str], List[str], List[str]]:218 """219 Preprocess the terms and polarities for aspect-based sentiment analysis.220 221 Args:222 references (List[Dict]): A list of dictionaries containing the actual terms and polarities under 'aspects'.223 predictions (List[Dict]): A list of dictionaries containing predicted aspect categories to terms and their sentiments.224 subtask_key (str): The key under which aspects are stored.225 subtask_value (str): The specific aspect value to extract.226 227 Returns:228 Tuple[List[str], List[str], List[str], List[str]]: A tuple containing lists of true aspect terms,229 adjusted predicted aspect terms, true polarities, and adjusted predicted polarities.230 """231 232 # Extract aspect terms and polarities233 truth_aspect_terms = extract_aspects(references, subtask_key, subtask_value)234 pred_aspect_terms = extract_aspects(predictions, subtask_key, subtask_value)235 truth_polarities = extract_aspects(references, subtask_key, "polarity")236 pred_polarities = extract_aspects(predictions, subtask_key, "polarity")237 238 # Define adjustment parameters239 special_token = "NONE" # For missing aspect terms240 sentiment_choices = set(flatten_list(truth_polarities)) 241 242 # Adjust the predictions to match the length of references243 adjusted_pred_terms = adjust_predictions(244 truth_aspect_terms, pred_aspect_terms, [special_token]245 )246 adjusted_pred_polarities = adjust_predictions(247 truth_polarities, pred_polarities, sentiment_choices248 )249 250 return (251 flatten_list(truth_aspect_terms),252 flatten_list(adjusted_pred_terms),253 flatten_list(truth_polarities),254 flatten_list(adjusted_pred_polarities),255 )256 257 258def flatten_list(nested_list):259 """Flatten a nested list into a single-level list."""260 return list(chain.from_iterable(nested_list))