lschlessinger/usatt-rating-analyzer

1import logging2from pathlib import Path3from typing import Optional, Tuple4 5import matplotlib.pyplot as plt6import pandas as pd7import plotly.graph_objects as go8import requests9import seaborn as sns10from bs4 import BeautifulSoup11from wordcloud import WordCloud12 13from util import get_max_abs_int, snake_case_to_human_readable, int_csv_to_list14 15 16def _rename_columns(df: pd.DataFrame, is_tournament: bool) -> pd.DataFrame:17    columns = {18        "Rating": "rating",19        "Result": "result",20        "Scores": "scores",21        "Opponent": "opponent",22        "OpponentRating": "opponent_rating",23    }24 25    if is_tournament:26        columns.update({27            "TournamentStartDate": "tournament_start_date",28            "TournamentEndDate": "tournament_end_date",29            " Touranament": "tournament",30        })31    else:32        columns.update({33            "EventDate": "event_date",34            "LeagueName": "league_name"35        })36 37    return df.rename(columns=columns)38 39 40def _fix_dtypes(df: pd.DataFrame, is_tournament: bool) -> pd.DataFrame:41    if is_tournament:42        df["tournament_start_date"] = pd.to_datetime(df["tournament_start_date"])43        df["tournament_end_date"] = pd.to_datetime(df["tournament_end_date"])44        df["tournament"] = df["tournament"].astype('category')45    else:46        df["event_date"] = pd.to_datetime(df["event_date"])47        df["league_name"] = df["league_name"].astype('string')48 49    df["rating"] = df["rating"].astype('int')50    df["result"] = df["result"].astype('category')51    df["scores"] = df["scores"].astype('string')52    df["opponent"] = df["opponent"].astype('category')53    df["opponent_rating"] = df["opponent_rating"].astype('int')54 55    return df56 57 58def make_df_columns_readable(df: Optional[pd.DataFrame], is_tournament: bool) -> Optional[pd.DataFrame]:59    """Make a data frame's columns human-readable."""60    if df is None:61        return None62 63    nat_to_none = lambda x: None if x == "NaT" else x64    if is_tournament:65        if "tournament_start_date" in df.columns and "tournament_end_date" in df.columns:66            df['tournament_start_date'] = pd.to_datetime(df['tournament_start_date'])67            df['tournament_end_date'] = pd.to_datetime(df['tournament_end_date'])68            df['tournament_start_date'] = df['tournament_start_date'].dt.date.astype(str).apply(nat_to_none)69            df['tournament_end_date'] = df['tournament_end_date'].dt.date.astype(str).apply(nat_to_none)70 71            def create_date(tournament_start_date, tournament_end_date):72                missing_start_date = tournament_start_date is None73                missing_end_date = tournament_end_date is None74                if not missing_start_date and not missing_end_date:75                    if tournament_start_date is not tournament_end_date:76                        return ' - '.join((tournament_start_date, tournament_end_date))77                    else:78                        return tournament_start_date79                else:80                    return tournament_start_date if missing_end_date else tournament_end_date81 82            df["date"] = df.apply(lambda row: create_date(row['tournament_start_date'], row['tournament_end_date']),83                                  axis=1)84            df = df.drop(columns=["tournament_start_date", "tournament_end_date"])85 86            # Move date to the front.87            columns = list(df.columns)88            columns.insert(0, columns.pop(columns.index("date")))89            df = df.loc[:, columns]90    else:91        if "event_date" in df.columns:92            df['event_date'] = pd.to_datetime(df['event_date'])93            df['event_date'] = df['event_date'].dt.date.astype(str).apply(nat_to_none)94        df = df.rename(columns={"league_name": "league"})95 96    df = df.rename(columns=lambda c: snake_case_to_human_readable(c))97    return df98 99 100def _check_match_type(match_type: str) -> str:101    allowed_match_types = {"tournament", "league"}102    if match_type not in allowed_match_types:103        raise ValueError(104            f"The only supported match types are {allowed_match_types}. Found match type of '{match_type}'.")105    return match_type106 107 108def fetch_player_name(profile_id: int) -> str:109    """Fetch a player name from theUSATT website.110 111    note: the profile ID is NOT the USATT number.112    """113    url = f"https://usatt.simplycompete.com/userAccount/up/{profile_id}"114    logging.info(f"Fetching player name from {url}")115    page = requests.get(url)116    soup = BeautifulSoup(page.content, "html.parser")117    profile_elt = soup.find("div", class_="profile-header")118    return profile_elt.find(class_="title").text.strip()119 120 121def get_player_name(file_stem: str) -> str:122    profile_id = int(file_stem.split(" ")[0].replace("_", "").split("matches")[-1])123    return fetch_player_name(profile_id)124 125 126def get_num_competitions_played(df: pd.DataFrame, is_tournament: bool) -> int:127    key_name = "tournament_end_date" if is_tournament else "event_date"128    return df[key_name].nunique()129 130 131def get_first_competition_year(df: pd.DataFrame, is_tournament: bool) -> int:132    key_name = "tournament_end_date" if is_tournament else "event_date"133    return df[key_name].min().year134 135 136def get_num_active_years(df: pd.DataFrame, is_tournament: bool) -> int:137    key_name = "tournament_end_date" if is_tournament else "event_date"138    return df[key_name].dt.year.nunique()139 140 141def get_current_rating(df: pd.DataFrame) -> int:142    return df.rating.iloc[0]143 144 145def get_max_rating(df: pd.DataFrame) -> int:146    return df.rating.max()147 148 149def get_matches_per_competition_fig(df: pd.DataFrame, is_tournament: bool):150    fig = plt.figure()151    plt.title('Matches per competition')152    sns.histplot(df.groupby('tournament' if is_tournament else "event_date", observed=False).size())153    plt.xlabel('Number of matches in competition')154    return fig155 156 157def get_competition_name_word_cloud_fig(df: pd.DataFrame, is_tournament: bool):158    fig = plt.figure()159    key_name = "tournament" if is_tournament else "league_name"160    wordcloud = WordCloud().generate(" ".join(df[key_name].values.tolist()))161    plt.imshow(wordcloud, interpolation='bilinear')162    plt.axis("off")163    return fig164 165 166def get_opponent_name_word_cloud_fig(df: pd.DataFrame):167    fig = plt.figure()168    wordcloud = WordCloud().generate(" ".join(df.opponent.values.tolist()))169    plt.imshow(wordcloud, interpolation='bilinear')170    plt.axis("off")171    return fig172 173 174def get_rating_over_time_fig(df: pd.DataFrame, is_tournament: bool, span: int = 60):175    df['ema'] = df['rating'].ewm(span=span, adjust=False).mean()176 177    fig = go.Figure()178 179    # Raw rating over time trace180    x_key_name = "tournament_end_date" if is_tournament else "event_date"181    fig.add_trace(go.Scatter(x=df[x_key_name],182                             y=df["rating"],183                             name='Rating',184                             mode='lines+markers',185                             line=dict(width=0.9),186                             marker=dict(size=4))),187 188    # EMA trace189    fig.add_trace(go.Scatter(x=df[x_key_name],190                             y=df["ema"],191                             mode='lines',192                             name='Rating EMA',193                             visible='legendonly',194                             line=dict(width=1.5, dash='dot')))195 196    fig.update_layout(197        title='Rating over time',198        xaxis_title='Competition date',199        yaxis_title='Rating',200        showlegend=True,201        template="plotly_white",202    )203 204    return fig205 206 207def get_match_with_longest_game(df: pd.DataFrame, is_tournament: bool) -> Optional[pd.DataFrame]:208    if not is_tournament:209        return None210    df_non_null = df.loc[~df.scores.isna()]211    return df_non_null.iloc[[df_non_null.scores.apply(get_max_abs_int).argmax()]]212 213 214def get_win_loss_record_str(group_df) -> str:215    if len(group_df) > 0:216        win_loss_counts = group_df.value_counts()217        n_wins = win_loss_counts.Won if hasattr(win_loss_counts, "Won") else 0218        n_losses = win_loss_counts.Lost if hasattr(win_loss_counts, "Lost") else 0219    else:220        n_wins = 0221        n_losses = 0222 223    return f"{n_wins}, {n_losses}"224 225 226def get_most_frequent_opponents(df: pd.DataFrame, top_n: int = 5) -> pd.DataFrame:227    df_with_opponents = df.loc[df.opponent != "-, -"]228 229    most_common_opponents_df = df_with_opponents.groupby('opponent', observed=False).agg(230        {"result": [get_win_loss_record_str, "size"]})231    most_common_opponents_df.columns = most_common_opponents_df.columns.get_level_values(1)232    most_common_opponents_df.rename({"get_win_loss_record_str": "Win/loss record", "size": "Number of matches"}, axis=1,233                                    inplace=True)234    most_common_opponents_df["Opponent"] = most_common_opponents_df.index235    return most_common_opponents_df.sort_values("Number of matches", ascending=False)[236        ["Opponent", "Number of matches", "Win/loss record"]].head(top_n)237 238 239def get_best_wins(df: pd.DataFrame, top_n: int = 5) -> pd.DataFrame:240    """Get the top-n wins sorted by opponent rating."""241    return df.loc[df.result == 'Won'].sort_values("opponent_rating", ascending=False).head(top_n)242 243 244def get_biggest_upsets(df: pd.DataFrame, top_n: int = 5) -> pd.DataFrame:245    """Get the top-n wins sorted by rating difference."""246    df['rating_difference'] = df['opponent_rating'] - df['rating']247    return df.loc[df.result == 'Won'].sort_values("rating_difference", ascending=False).head(top_n)248 249 250def get_worst_recent_losses(df: pd.DataFrame,251                            is_tournament: bool,252                            top_k_losses: int = 5,253                            top_n_comps: int = 5) -> pd.DataFrame:254    """Get the top-k most recent worst losses from the top-n most recent competitions."""255    x_key_name = "tournament_end_date" if is_tournament else "event_date"256    most_recent_competition_dates = df.groupby(x_key_name).first().reset_index().nlargest(top_n_comps,257                                                                                          columns=x_key_name)[258        x_key_name]259    df_recent = df.loc[df[x_key_name].isin(most_recent_competition_dates)]260    return df_recent.loc[df_recent.result == 'Lost'].sort_values("opponent_rating", ascending=True).head(top_k_losses)261 262 263def get_best_competitions(df: pd.DataFrame, is_tournament: bool, top_n: int = 5) -> pd.DataFrame:264    # First add pre-competition ratings265    x_key_name = "tournament_end_date" if is_tournament else "event_date"266    grouped = df.groupby(x_key_name)267 268    # We incorrectly fill the first pre-competition rating to the first rating so that269    # the top-k rating differences make sense.270    fill_value = df.iloc[-1].rating271    pre_comp_ratings_by_group = grouped['rating'].first().shift(periods=1, fill_value=fill_value)272 273    def assign_pre_comp_rating(group_df):274        """Assign a pre-competition rating to a given group."""275        comp_end_date = group_df[x_key_name].unique()[0]276        group_df['pre-competition_rating'] = pre_comp_ratings_by_group.loc[comp_end_date]277        return group_df278 279    df = grouped.apply(lambda x: assign_pre_comp_rating(x))280 281    df['rating_increase'] = df['rating'] - df['pre-competition_rating']282    df.reset_index(drop=True, inplace=True)283    best_competition_dates = df.groupby(x_key_name)["rating_increase"].first().nlargest(top_n).index284 285    tournament_df = df.loc[df[x_key_name].isin(best_competition_dates)].groupby(286        [x_key_name]).first().sort_values(by='rating_increase', ascending=False).reset_index()287 288    cols = []289    if is_tournament:290        cols += ['tournament_start_date', 'tournament_end_date', 'tournament']291    else:292        cols += ["event_date", "league_name"]293    cols += ['rating_increase', 'pre-competition_rating', 'rating']294 295    tournament_df = tournament_df[cols]296    tournament_df = tournament_df.rename(columns={"rating": "post-competition_rating"})297 298    return tournament_df299 300 301def get_highest_rated_opponent(df: pd.DataFrame) -> pd.DataFrame:302    return df.iloc[df.opponent_rating.idxmax()].to_frame().transpose()303 304 305def get_opponent_rating_distr_fig(df: pd.DataFrame):306    fig = plt.figure()307    plt.title('Opponent rating distribution')308    sns.histplot(data=df, x="opponent_rating", hue='result')309    plt.xlabel('Opponent rating')310    return fig311 312 313def get_opponent_rating_dist_over_time_fig(df: pd.DataFrame, is_tournament: bool):314    fig, ax = plt.subplots(figsize=(12, 8))315    plt.title(f'Opponent rating distribution over time')316    x_key_name = "tournament_end_date" if is_tournament else "event_date"317    sns.violinplot(data=df,318                   x=df[x_key_name].dt.year,319                   y="opponent_rating",320                   hue="result",321                   split=True,322                   inner='points',323                   cut=1,324                   ax=ax)325    plt.xticks(rotation=30)326    plt.xlabel('Competition year')327    plt.ylabel('Opponent rating')328    return fig329 330 331def get_total_match_points(score_str: str) -> int:332    single_game_scores = int_csv_to_list(score_str)333    total_points = 0334    for single_game_score in single_game_scores:335        abs_gscore = abs(single_game_score)336        if abs_gscore < 10:337            total_points += abs_gscore + 11338        else:339            total_points += 2 * abs_gscore + 2340    return total_points341 342 343def get_longest_match(df: pd.DataFrame, is_tournament: bool) -> Optional[pd.DataFrame]:344    """Get the longest match, where longest is defined as the most number of points played."""345    if not is_tournament:346        return None347    df_non_null = df.loc[~df.scores.isna()]348    df_non_null["total_points"] = df_non_null.scores.apply(get_total_match_points)349    return df_non_null.iloc[[df_non_null["total_points"].argmax()]]350 351 352def load_match_df(file_path: Path) -> Tuple[pd.DataFrame, bool]:353    match_type = _check_match_type(file_path.name.split('_')[0])354    is_tournament = match_type == "tournament"355 356    df = pd.read_csv(file_path)357    df = _rename_columns(df, is_tournament)358    df = _fix_dtypes(df, is_tournament)359 360    logging.info(f"Loaded match CSV {file_path}.")361 362    return df, is_tournament363