HuggingFaceH4/Elo

1import numpy as np2import pandas as pd3import streamlit as st4 5 6def create_synthetic_data(n_tasks=100, n_models=4, n_ratings=3):7    """Create a synthetic dataframe with human ratings of model performance on a set of tasks.8 9    Parameters10    ----------11    n_tasks : int12        The number of tasks.13    n_models : int14        The number of models.15    n_ratings : int16        The number of human ratings of model performance on a set of tasks.17 18    Returns19    -------20    pandas.DataFrame21        DataFrame containing human ratings of model performance on a set of tasks.22    """23    # create a synthetic dataframe with 3 human ratings of 4 models performance on a set of 100 tasks24    df = pd.DataFrame({'task': np.repeat(range(n_tasks), n_models * n_ratings),25                       'model': np.tile(np.repeat(range(n_models), n_ratings), n_tasks),26                       'rating': np.tile(np.random.randint(0, 5, n_models * n_ratings), n_tasks)})27    # calculate score for each model28    df['score'] = df.groupby(['task', 'model'])['rating'].transform('mean')29    # calculate baseline score for each task30    df['baseline'] = df.groupby('task')['score'].transform('min')31    # calculate score for each model relative to baseline score32    df['score'] = df['score'] - df['baseline']33    # drop unnecessary columns34    df = df.drop(['rating', 'baseline'], axis=1)35    # drop duplicates36    df = df.drop_duplicates()37    return df38 39 40def calculate_elo_rating(df, k=32, initial_rating=0):41    """Calculate ELORating for each model based on human ratings of model performance on a set of tasks.42 43    Parameters44    ----------45    df : pandas.DataFrame46        DataFrame containing human ratings of model performance on a set of tasks.47    k : int48        The k-factor.49    initial_rating : int50        The initial rating.51 52    Returns53    -------54    pandas.DataFrame55        DataFrame containing ELORating for each model based on human ratings of model performance on a set of tasks.56    """57    # calculate ELORating for each model based on human ratings of model performance on a set of tasks58    # create a dat59    df = df.copy()60    # create a dataframe with all possible combinations of tasks and models61    df_all = pd.DataFrame({'task': np.repeat(range(df['task'].max() + 1), df['model'].max() + 1),62                            'model': np.tile(range(df['model'].max() + 1), df['task'].max() + 1)})63    # merge with original dataframe64    df = df_all.merge(df, on=['task', 'model'], how='left')65    # fill missing values with 066    df['score'] = df['score'].fillna(0)67    # calculate expected score for each model68    df['expected_score'] = df.groupby('model')['score'].transform(lambda x: 1 / (1 + 10 ** (-x / 400)))69    # calculate actual score for each model70    df['actual_score'] = df.groupby('model')['score'].transform(lambda x: x > 0).astype(int)71    # calculate rating for each model72    df['rating'] = df.groupby('model')['expected_score'].transform(lambda x: x * k + initial_rating)73    # calculate rating change for each model74    df['rating_change'] = df.groupby('model')['actual_score'].transform(lambda x: x * k)75    # calculate new rating for each model76    df['new_rating'] = df['rating'] + df['rating_change']77    # drop unnecessary columns78    df = df.drop(['score', 'expected_score', 'actual_score', 'rating', 'rating_change'], axis=1)79    return df80 81def display_leaderboard(elo, n_models=4):82    """Display Elo rating for each model as a leaderboard based on their ranking.83 84    Parameters85    ----------86    elo : pandas.DataFrame87        DataFrame containing ELORating for each model based on human ratings of model performance on a set of tasks.88    n_models : int89        The number of models.90    """91    # calculate average Elo rating for each model92    elo = elo.groupby('model')['new_rating'].mean().reset_index()93    # sort models by Elo rating94    elo = elo.sort_values('new_rating', ascending=False)95    # add rank column96    elo['rank'] = range(1, n_models + 1)97    # display Elo rating for each model as a leaderboard based on their ranking98    st.write(elo)