OpenHands/openhands-index

1"""2Additional visualizations for the OpenHands Index leaderboard.3 4These functions use the generic create_scatter_chart() from leaderboard_transformer5as the single source of truth for scatter plot styling and behavior.6"""7import pandas as pd8import plotly.graph_objects as go9import aliases10 11# Import the generic scatter chart function - single source of truth12from leaderboard_transformer import create_scatter_chart, STANDARD_LAYOUT, STANDARD_FONT13 14 15def _find_column(df: pd.DataFrame, candidates: list, default: str = None) -> str:16    """Find the first matching column name from candidates."""17    for col in candidates:18        if col in df.columns:19            return col20    return default21 22 23def create_evolution_over_time_chart(df: pd.DataFrame, mark_by: str = None) -> go.Figure:24    """25    Create a chart showing model performance evolution over release dates.26    27    Args:28        df: DataFrame with release_date and score columns29        mark_by: One of "Company", "Openness", or "Country" for marker icons30    31    Returns:32        Plotly figure showing score evolution over time33    """34    # Find the release date column35    release_date_col = _find_column(df, ['release_date', 'Release_Date', 'Release Date'])36    37    if df.empty or release_date_col is None:38        fig = go.Figure()39        fig.add_annotation(40            text="No release date data available",41            xref="paper", yref="paper",42            x=0.5, y=0.5, showarrow=False,43            font=STANDARD_FONT44        )45        fig.update_layout(**STANDARD_LAYOUT, title="Model Performance Evolution Over Time")46        return fig47    48    # Find score column49    score_col = _find_column(df, ['Average Score', 'average score', 'Average score'])50    if score_col is None:51        # Try to find any column with 'score' and 'average'52        for col in df.columns:53            if 'score' in col.lower() and 'average' in col.lower():54                score_col = col55                break56    57    if score_col is None:58        fig = go.Figure()59        fig.add_annotation(60            text="No score data available",61            xref="paper", yref="paper",62            x=0.5, y=0.5, showarrow=False,63            font=STANDARD_FONT64        )65        fig.update_layout(**STANDARD_LAYOUT, title="Model Performance Evolution Over Time")66        return fig67    68    # Use the generic scatter chart69    return create_scatter_chart(70        df=df,71        x_col=release_date_col,72        y_col=score_col,73        title="Model Performance Evolution Over Time",74        x_label="Model Release Date",75        y_label="Average Score",76        mark_by=mark_by,77        x_type="date",78        pareto_lower_is_better=False,  # Later dates with higher scores are better79    )80 81 82def create_accuracy_by_size_chart(df: pd.DataFrame, mark_by: str = None) -> go.Figure:83    """84    Create a scatter plot showing accuracy vs parameter count for open-weights models.85    86    Args:87        df: DataFrame with parameter_count and score columns88        mark_by: One of "Company", "Openness", or "Country" for marker icons89    90    Returns:91        Plotly figure showing accuracy vs model size92    """93    # Find parameter count column94    param_col = _find_column(df, ['parameter_count_b', 'Parameter_Count_B', 'Parameter Count B'])95    96    if df.empty or param_col is None:97        fig = go.Figure()98        fig.add_annotation(99            text="No parameter count data available",100            xref="paper", yref="paper",101            x=0.5, y=0.5, showarrow=False,102            font=STANDARD_FONT103        )104        fig.update_layout(**STANDARD_LAYOUT, title="Open Model Accuracy by Size")105        return fig106    107    # Filter to only open-weights models108    open_aliases = [aliases.CANONICAL_OPENNESS_OPEN] + list(109        aliases.OPENNESS_ALIASES.get(aliases.CANONICAL_OPENNESS_OPEN, [])110    )111    openness_col = _find_column(df, ['Openness', 'openness'])112    if openness_col is None:113        fig = go.Figure()114        fig.add_annotation(115            text="No openness data available",116            xref="paper", yref="paper",117            x=0.5, y=0.5, showarrow=False,118            font=STANDARD_FONT119        )120        fig.update_layout(**STANDARD_LAYOUT, title="Open Model Accuracy by Size")121        return fig122    123    plot_df = df[124        (df[param_col].notna()) & 125        (df[openness_col].isin(open_aliases))126    ].copy()127    128    if plot_df.empty:129        fig = go.Figure()130        fig.add_annotation(131            text="No open-weights models with parameter data available",132            xref="paper", yref="paper",133            x=0.5, y=0.5, showarrow=False,134            font=STANDARD_FONT135        )136        fig.update_layout(**STANDARD_LAYOUT, title="Open Model Accuracy by Size")137        return fig138    139    # Find score column140    score_col = _find_column(plot_df, ['Average Score', 'average score', 'Average score'])141    if score_col is None:142        for col in plot_df.columns:143            if 'score' in col.lower() and 'average' in col.lower():144                score_col = col145                break146    147    if score_col is None:148        fig = go.Figure()149        fig.add_annotation(150            text="No score data available",151            xref="paper", yref="paper",152            x=0.5, y=0.5, showarrow=False,153            font=STANDARD_FONT154        )155        fig.update_layout(**STANDARD_LAYOUT, title="Open Model Accuracy by Size")156        return fig157    158    # Use the generic scatter chart159    return create_scatter_chart(160        df=plot_df,161        x_col=param_col,162        y_col=score_col,163        title="Open Model Accuracy by Size",164        x_label="Parameters (Billions)",165        y_label="Average Score",166        mark_by=mark_by,167        x_type="log",168        pareto_lower_is_better=True,  # Smaller models with higher scores are better169    )170