hyperml/balanced_accuracy
0
1# Copyright 2023 HyperML Authors and the current HyperML contributor.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""Balanced Accuracy metric."""15 16import evaluate17import datasets18from sklearn.metrics import balanced_accuracy_score19 20 21_DESCRIPTION = """22Balanced Accuracy is the average of recall obtained on each class. It can be computed with:23Balanced Accuracy = (TPR + TNR) / N24Where:25TPR: True positive rate26TNR: True negative rate27N: Number of classes28"""29 30_KWARGS_DESCRIPTION = """31Args:32 predictions (`list` of `int`): Predicted labels.33 references (`list` of `int`): Ground truth labels.34 sample_weight (`list` of `float`): Sample weights Defaults to None.35 adjusted (`boolean`): When true, the result is adjusted for chance, so that random performance would score 0, while keeping perfect performance at a score of 1. Defaults to False.36 37Returns:38 balanced_accuracy (`float`): Balanced Accuracy score. Minimum possible value is 0. Maximum possible value is 1.0. A higher score means higher balanced accuracy.39 40Examples:41 42 Example 1-A simple example43 >>> balanced_accuracy_metric = evaluate.load("balanced_accuracy")44 >>> results = balanced_accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0])45 >>> print(results)46 {'balanced_accuracy': 0.5}47 48 Example 2-The same as Example 1, except with `sample_weight` set.49 >>> balanced_accuracy_metric = evaluate.load("balanced_accuracy")50 >>> results = balanced_accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4])51 >>> print(results)52 {'balanced_accuracy': 0.8778625954198473} # TODO: check if this is correct53 54 Example 3-The same as Example 1, except with `adjusted` set to `True`.55 >>> balanced_accuracy_metric = evaluate.load("balanced_accuracy")56 >>> results = balanced_accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], adjusted=True)57 >>> print(results)58 {'balanced_accuracy': 0.8} # TODO: check if this is correct59"""60 61_CITATION = """62@article{scikit-learn,63 title={Scikit-learn: Machine Learning in {P}ython},64 author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.65 and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.66 and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and67 Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},68 journal={Journal of Machine Learning Research},69 volume={12},70 pages={2825--2830},71 year={2011}72}73"""74 75class BalancedAccuracy(evaluate.Metric):76 def _info(self):77 return evaluate.MetricInfo(78 description=_DESCRIPTION,79 citation=_CITATION,80 inputs_description=_KWARGS_DESCRIPTION,81 features=datasets.Features(82 {83 "predictions": datasets.Sequence(datasets.Value("int32")),84 "references": datasets.Sequence(datasets.Value("int32")),85 }86 if self.config_name == "multilabel"87 else {88 "predictions": datasets.Value("int32"),89 "references": datasets.Value("int32"),90 }91 ),92 reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.balanced_accuracy_score.html"],93 )94 95 def _compute(self, predictions, references, sample_weight=None, adjusted=False):96 return {97 "balanced_accuracy": float(98 balanced_accuracy_score(references, predictions, sample_weight=sample_weight, adjusted=adjusted)99 )100 }