CoreyMorris/MMLU-by-task-Leaderboard
16
1import pandas as pd2import os3import fnmatch4import json5import re6import numpy as np7import logging8 9logging.basicConfig(filename='error_log.log', level=logging.ERROR)10 11class ResultDataProcessor:12 13 14 def __init__(self, directory='results', pattern='results*.json'):15 16 self.directory = directory17 self.pattern = pattern18 self.data = self.process_data()19 self.ranked_data = self.rank_data()20 21 def _find_files(self, directory='results', pattern='results*.json'):22 matching_files = {}23 for root, dirs, files in os.walk(directory):24 for basename in files:25 if fnmatch.fnmatch(basename, pattern):26 filename = os.path.join(root, basename)27 matching_files[root] = filename28 # TODO decide on removing this since I am catching the error when processing the file29 matching_files = {key: value for key, value in matching_files.items() if 'gpt-j-6b' not in key}30 matching_files = list(matching_files.values())31 return matching_files32 33 def _read_and_transform_data(self, filename):34 with open(filename) as f:35 data = json.load(f)36 df = pd.DataFrame(data['results']).T37 return df38 39 def _cleanup_dataframe(self, df, model_name):40 df = df.rename(columns={'acc': model_name})41 df.index = (df.index.str.replace('hendrycksTest-', 'MMLU_', regex=True)42 .str.replace('harness\|', '', regex=True)43 .str.replace('\|5', '', regex=True))44 return df[[model_name]]45 46 def _extract_mc1(self, df, model_name):47 df = df.rename(columns={'mc1': model_name})48 # rename row harness|truthfulqa:mc|0 to truthfulqa:mc149 df.index = (df.index.str.replace('mc\|0', 'mc1', regex=True))50 # just return the harness|truthfulqa:mc1 row51 df = df.loc[['harness|truthfulqa:mc1']]52 return df[[model_name]]53 54 def _extract_mc2(self, df, model_name):55 # rename row harness|truthfulqa:mc|0 to truthfulqa:mc256 df = df.rename(columns={'mc2': model_name})57 df.index = (df.index.str.replace('mc\|0', 'mc2', regex=True))58 df = df.loc[['harness|truthfulqa:mc2']]59 return df[[model_name]]60 61 # remove extreme outliers from column harness|truthfulqa:mc162 def _remove_mc1_outliers(self, df):63 mc1 = df['harness|truthfulqa:mc1']64 # Identify the outliers65 # outliers_condition = mc1 > mc1.quantile(.95)66 outliers_condition = mc1 == 1.067 # Replace the outliers with NaN68 df.loc[outliers_condition, 'harness|truthfulqa:mc1'] = np.nan69 return df70 71 72 73 @staticmethod74 def _extract_parameters(model_name):75 """76 Function to extract parameters from model name.77 It handles names with 'b/B' for billions and 'm/M' for millions. 78 """79 # pattern to match a number followed by 'b' (representing billions) or 'm' (representing millions)80 pattern = re.compile(r'(\d+\.?\d*)([bBmM])')81 82 match = pattern.search(model_name)83 84 if match:85 num, magnitude = match.groups()86 num = float(num)87 88 # convert millions to billions89 if magnitude.lower() == 'm':90 num /= 100091 92 return num93 94 # return NaN if no match95 return np.nan96 97 98 def process_data(self):99 full_model_name_count = 0100 full_model_names = []101 dataframes = []102 organization_names = []103 for filename in self._find_files(self.directory, self.pattern):104 # try:105 raw_data = self._read_and_transform_data(filename)106 split_path = filename.split('/')107 model_name = split_path[2]108 organization_name = split_path[1]109 full_model_name = f'{organization_name}/{model_name}'110 full_model_name_count += 1111 # print count every 100 models112 if full_model_name_count % 100 == 0:113 print(full_model_name_count)114 115 cleaned_data = self._cleanup_dataframe(raw_data, model_name)116 # mc1 = self._extract_mc1(raw_data, full_model_name)117 # mc2 = self._extract_mc2(raw_data, full_model_name)118 # cleaned_data = pd.concat([cleaned_data, mc1])119 # cleaned_data = pd.concat([cleaned_data, mc2])120 organization_names.append(organization_name)121 full_model_names.append(full_model_name)122 dataframes.append(cleaned_data)123 # except Exception as e:124 # # logging.error(f'Error processing {filename}')125 # # logging.error(f'The error is: {e}')126 # print(f'Error processing {filename}')127 # print(f'The error is: {e}')128 # continue129 130 131 data = pd.concat(dataframes, axis=1).transpose()132 133 # Add organization column134 # data['organization'] = organization_names135 print("full_model_names")136 print(len(full_model_names))137 print("organization_names")138 print(len(organization_name))139 data['full_model_name'] = full_model_names140 141 # Add Model Name and rearrange columns142 data['Model Name'] = data.index143 cols = data.columns.tolist()144 cols = cols[-1:] + cols[:-1]145 data = data[cols]146 147 # Remove the 'Model Name' column148 data = data.drop(columns=['Model Name'])149 150 # Add average column151 data['MMLU_average'] = data.filter(regex='MMLU').mean(axis=1)152 153 # Reorder columns to move 'MMLU_average' to the third position154 cols = data.columns.tolist()155 cols = cols[:2] + cols[-1:] + cols[2:-1]156 data = data[cols]157 158 159 160 161 162 163 # Add parameter count column using extract_parameters function164 data['Parameters'] = data.index.to_series().apply(self._extract_parameters)165 166 # move the parameters column to the front of the dataframe167 cols = data.columns.tolist()168 cols = cols[-1:] + cols[:-1]169 print(cols)170 data = data[cols]171 172 173 new_columns = ['full_model_name'] + [col for col in data.columns if col != 'full_model_name']174 data = data.reindex(columns=new_columns)175 176 # # Reorder columns to move 'organization' to the second position177 # cols = data.columns.tolist()178 # cols = cols[-1:] + cols[:-1]179 # data = data[cols]180 181 # remove extreme outliers from column harness|truthfulqa:mc1182 # data = self._remove_mc1_outliers(data)183 184 data = self.manual_removal_of_models(data)185 186 187 # drop rows if MMLU_abstract_algebra is NaN188 data = data.dropna(subset=['MMLU_abstract_algebra'])189 190 # add a URL column that takes https://huggingface.co/ + full_model_name191 data['URL'] = 'https://huggingface.co/' + data['full_model_name']192 193 new_columns = ['URL'] + [col for col in data.columns if col != 'URL']194 data = data.reindex(columns=new_columns)195 196 # drop columns drop|3 gsm8k and winogrande197 data = data.drop(columns=['drop|3', 'gsm8k', 'winogrande'])198 # # Drop specific columns199 data = data.drop(columns=['all', 'truthfulqa:mc|0'])200 201 # save to csv with the current date as part of the filename202 data.to_csv(f'processed_data_{pd.Timestamp.now().strftime("%Y-%m-%d")}.csv')203 204 return data205 206 def manual_removal_of_models(self, df):207 # remove models verified to be trained on evaluation data208 # load the list of models209 with open('contaminated_models.txt') as f:210 contaminated_models = f.read().splitlines()211 # remove the models from the dataframe212 df = df[~df.index.isin(contaminated_models)]213 return df214 215 216 def rank_data(self):217 # add rank for each column to the dataframe218 # copy the data dataframe to avoid modifying the original dataframe219 rank_data = self.data.copy()220 for col in list(rank_data.columns):221 rank_data[col + "_rank"] = rank_data[col].rank(ascending=False, method='min')222 223 return rank_data224 225 def get_data(self, selected_models):226 return self.data[self.data.index.isin(selected_models)]227