HuggingFaceM4/IDEFICS_Data_Measurement_Tool
2
1# Copyright 2021 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15# TODO: Change print statements to logging?16# from evaluate import logging as logs17import warnings18 19import datasets20import evaluate21import numpy as np22import pandas as pd23from sklearn.preprocessing import MultiLabelBinarizer24 25_CITATION = """\26Osman Aka, Ken Burke, Alex Bauerle, Christina Greer, and Margaret Mitchell. \272021. Measuring Model Biases in the Absence of Ground Truth. \28In Proceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society \29(AIES '21). Association for Computing Machinery, New York, NY, USA, 327–335. \30https://doi.org/10.1145/3461702.346255731"""32 33_DESCRIPTION = """\34Normalized Pointwise Information (nPMI) is an entropy-based measurement35of association, used here to measure the association between words.36"""37 38_KWARGS_DESCRIPTION = """\39Args:40 references (list of lists): List of tokenized sentences.41 vocab_counts (dict or dataframe): Vocab terms and their counts42Returns:43 npmi_df: A dataframe with (1) nPMI association scores for each term; \44 (2) the difference between them.45"""46 47# TODO: Is this necessary?48warnings.filterwarnings(action="ignore", category=UserWarning)49# When we divide by 0 in log50np.seterr(divide="ignore")51 52# treating inf values as NaN as well53pd.set_option("use_inf_as_na", True)54 55# This can be changed to whatever a person likes;56# it is the number of batches to use when iterating through the vocabulary.57_NUM_BATCHES = 50058PROP = "proportion"59CNT = "count"60 61class nPMI(evaluate.Measurement):62 def _info(self):63 return evaluate.MeasurementInfo(64 module_type="measurement",65 description=_DESCRIPTION,66 citation=_CITATION,67 inputs_description=_KWARGS_DESCRIPTION,68 features=datasets.Features(69 {70 "references": datasets.Sequence(71 datasets.Value("string", id="sequence"),72 id="references"),73 }74 )75 # TODO: Create docs for this.76 # reference_urls=["https://huggingface.co/docs/..."],77 )78 79 def _compute(self, references, vocab_counts, subgroup):80 if isinstance(vocab_counts, dict):81 vocab_counts_df = pd.DataFrame.from_dict(vocab_counts,82 orient='index',83 columns=[CNT])84 elif isinstance(vocab_counts, pd.DataFrame):85 vocab_counts_df = vocab_counts86 else:87 print("Can't support the data structure for the vocab counts. =(")88 return89 # These are used throughout the rest of the functions90 self.references = references91 self.vocab_counts_df = vocab_counts_df92 self.vocab_counts_df[PROP] = vocab_counts_df[CNT] / sum(93 vocab_counts_df[CNT])94 # self.mlb_list holds num batches x num_sentences95 self.mlb_list = []96 # Index of the subgroup word in the sparse vector97 subgroup_idx = vocab_counts_df.index.get_loc(subgroup)98 print("Calculating co-occurrences...")99 df_coo = self.calc_cooccurrences(subgroup, subgroup_idx)100 vocab_cooc_df = self.set_idx_cols(df_coo, subgroup)101 print("Calculating PMI...")102 pmi_df = self.calc_PMI(vocab_cooc_df, subgroup)103 print("Calculating nPMI...")104 npmi_df = self.calc_nPMI(pmi_df, vocab_cooc_df, subgroup)105 npmi_bias = npmi_df.max(axis=0) + abs(npmi_df.min(axis=0))106 return {"bias": npmi_bias, "co-occurrences": vocab_cooc_df,107 "pmi": pmi_df, "npmi": npmi_df}108 109 def _binarize_words_in_sentence(self):110 print("Creating co-occurrence matrix for PMI calculations.")111 batches = np.linspace(0, len(self.references), _NUM_BATCHES).astype(int)112 i = 0113 # Creates list of size (# batches x # sentences)114 while i < len(batches) - 1:115 # Makes a sparse matrix (shape: # sentences x # words),116 # with the occurrence of each word per sentence.117 mlb = MultiLabelBinarizer(classes=self.vocab_counts_df.index)118 print(119 "%s of %s sentence binarize batches." % (120 str(i), str(len(batches)))121 )122 # Returns series: batch size x num_words123 mlb_series = mlb.fit_transform(124 self.references[batches[i]:batches[i + 1]]125 )126 i += 1127 self.mlb_list.append(mlb_series)128 129 def calc_cooccurrences(self, subgroup, subgroup_idx):130 initialize = True131 coo_df = None132 # Big computation here! Should only happen once.133 print(134 "Approaching big computation! Here, we binarize all words in the sentences, making a sparse matrix of sentences."135 )136 if not self.mlb_list:137 self._binarize_words_in_sentence()138 for batch_id in range(len(self.mlb_list)):139 print(140 "%s of %s co-occurrence count batches"141 % (str(batch_id), str(len(self.mlb_list)))142 )143 # List of all the sentences (list of vocab) in that batch144 batch_sentence_row = self.mlb_list[batch_id]145 # Dataframe of # sentences in batch x vocabulary size146 sent_batch_df = pd.DataFrame(batch_sentence_row)147 # Subgroup counts per-sentence for the given batch148 subgroup_df = sent_batch_df[subgroup_idx]149 subgroup_df.columns = [subgroup]150 # Remove the sentences where the count of the subgroup is 0.151 # This way we have less computation & resources needs.152 subgroup_df = subgroup_df[subgroup_df > 0]153 mlb_subgroup_only = sent_batch_df[sent_batch_df[subgroup_idx] > 0]154 # Create cooccurrence matrix for the given subgroup and all words.155 batch_coo_df = pd.DataFrame(mlb_subgroup_only.T.dot(subgroup_df))156 157 # Creates a batch-sized dataframe of co-occurrence counts.158 # Note these could just be summed rather than be batch size.159 if initialize:160 coo_df = batch_coo_df161 else:162 coo_df = coo_df.add(batch_coo_df, fill_value=0)163 initialize = False164 print("Returning co-occurrence matrix")165 return pd.DataFrame(coo_df)166 167 def set_idx_cols(self, df_coo, subgroup):168 """169 :param df_coo: Co-occurrence counts for subgroup, length is num_words170 :return:171 """172 count_df = df_coo.set_index(self.vocab_counts_df.index)173 count_df.columns = [subgroup + "-count"]174 count_df[subgroup + "-count"] = count_df[subgroup + "-count"].astype(175 int)176 return count_df177 178 def calc_PMI(self, vocab_cooc_df, subgroup):179 """180 # PMI(x;y) = h(y) - h(y|x)181 # = h(subgroup) - h(subgroup|word)182 # = log (p(subgroup|word) / p(subgroup))183 # nPMI additionally divides by -log(p(x,y)) = -log(p(x|y)p(y))184 """185 # Calculation of p(subgroup)186 # TODO: Is this better?187 # subgroup_prob = vocab_counts_df.loc[subgroup][PROP]188 subgroup_prob = self.vocab_counts_df.loc[subgroup][CNT] / sum(189 self.vocab_counts_df[CNT])190 # Calculation of p(subgroup|word) = count(subgroup,word) / count(word)191 # Because the indices match (the vocab words),192 # this division doesn't need to specify the index (I think?!)193 p_subgroup_g_word = (194 vocab_cooc_df[subgroup + "-count"] / self.vocab_counts_df[195 CNT]196 )197 pmi_df = pd.DataFrame()198 pmi_df[subgroup + "-pmi"] = np.log(p_subgroup_g_word / subgroup_prob)199 # Note: A potentially faster solution for adding count, npmi,200 # can be based on this zip idea:201 # df_test['size_kb'], df_test['size_mb'], df_test['size_gb'] =202 # zip(*df_test['size'].apply(sizes))203 return pmi_df.dropna()204 205 def calc_nPMI(self, pmi_df, vocab_cooc_df, subgroup):206 """207 # nPMI additionally divides by -log(p(x,y)) = -log(p(x|y)p(y))208 # = -log(p(word|subgroup)p(word))209 """210 p_word_g_subgroup = vocab_cooc_df[subgroup + "-count"] / sum(211 vocab_cooc_df[subgroup + "-count"]212 )213 p_word = pmi_df.apply(214 lambda x: self.vocab_counts_df.loc[x.name][PROP], axis=1215 )216 normalize_pmi = -np.log(p_word_g_subgroup * p_word)217 npmi_df = pd.DataFrame()218 npmi_df[subgroup + "-npmi"] = pmi_df[subgroup + "-pmi"] / normalize_pmi219 return npmi_df.dropna()220