venkatasg/fizzbuzz-bench
1
1import gradio as gr2import pandas as pd3import matplotlib4matplotlib.use("Agg") 5import seaborn as sns6import matplotlib.pyplot as plt7import numpy as np8 9sns.set(style='whitegrid', context='notebook', font_scale=1.75)10 11DESCRIPTION = """\12# FizzBuzz LLM Benchmark13 14A (silly) benchmark for testing how well LLMs can play the children's game [Fizzbuzz](https://en.wikipedia.org/wiki/Fizz_buzz). By modifying the game's standard rules, this benchmark tests generalization, long multi-turn conversation, arithmetic and counting abilities of LLMs.15See [the GitHub repository](https://github.com/venkatasg/fizzbuzz-bench) for all the details on how I tested the models. I'll try to keep this updated with the latest models.16"""17 18COLUMNS = ["Model", "Standard FizzBuzz", "Buzz=7"]19 20INITIAL_DATA = [21 ["gpt-5.2-pro", 200, 200],22 ["claude-opus-4-6", 200, 186],23 ["claude-opus-4-5", 200, 200],24 ["claude-sonnet-4", 200, 200],25 ["gemini-3-pro-preview", 200, 103],26 ["GLM-4.7", 13, 52],27 ["gemini-2.0-flash", 200, 39],28 ["claude-sonnet-4-5", 200, 200],29 ["Llama-4-Maverick", 5, 41],30 ["gemini-3-flash-preview", 115, 83],31 ["gpt-5.1", 39, 27],32 ["Kimi-K2-Thinking", 21, 9],33 ["claude-haiku-4-5", 5, 13],34 ["gpt-5.2", 5, 11],35 ["Qwen3-235B-A22B", 25, 11],36 ["DeepSeek-V3-0324", 43, 9],37 ["gemini-2.5-pro", 200, 200],38 ["gpt-4.1", 23, 5],39 ["gpt-4.1-mini", 33, 5],40 ["gpt-3.5-turbo", 11, 1],41 ["claude-3-7-sonnet", 200, 5],42 ["gemini-2.5-flash", 3, 5],43 ["gemma-3-27b-it", 5, 5],44 ["DeepSeek-V3.1", 95, 5],45 ["Qwen3-Next-80B-A3B", 17, 5],46 ["gpt-4.1-nano", 3, 3],47 ["Llama-3.3-70B", 3, 3],48 ["Kimi-K2-Instruct-0905", 200, 19],49 ["Minimax-M2.5", 9, 9],50 ["GLM-5", 53, 45],51]52 53 54def make_sorted_df(raw=None):55 """Build a DataFrame sorted by Buzz=7 descending."""56 if raw is None:57 df = pd.DataFrame(INITIAL_DATA, columns=COLUMNS)58 elif isinstance(raw, pd.DataFrame):59 df = raw.copy()60 df.columns = COLUMNS61 else:62 df = pd.DataFrame(raw, columns=COLUMNS)63 df["Standard FizzBuzz"] = pd.to_numeric(df["Standard FizzBuzz"], errors="coerce").fillna(0).astype(int)64 df["Buzz=7"] = pd.to_numeric(df["Buzz=7"], errors="coerce").fillna(0).astype(int)65 df = df.sort_values(["Buzz=7", "Standard FizzBuzz"], ascending=False).reset_index(drop=True)66 return df67 68 69def create_chart(df):70 """Create a grouped horizontal bar chart of the top 15 models using seaborn."""71 top10 = df.head(15).copy() 72 73 # Reshape to long format for seaborn74 plot_df = (75 top1076 .melt(id_vars="Model", 77 var_name="Task", 78 value_name="Score")79 )80 81 fig, ax = plt.subplots(figsize=(15, 15))82 83 sns.barplot(84 data=plot_df,85 y="Model",86 x="Score",87 hue="Task",88 hue_order=['Buzz=7', 'Standard FizzBuzz'],89 orient="h",90 ax=ax,91 palette='colorblind'92 )93 94 ax.set_xlim(0, 200)95 ax.set_xlabel("Successful turns")96 ax.set_ylabel("")97 ax.set_title("Top 15 models ranked by modified FizzBuzz (Buzz=7) score")98 ax.legend(loc="lower right")99 ax.tick_params(axis="y")100 101 fig.tight_layout()102 return fig103 104 105def on_data_edit(table_data):106 """Regenerate the chart when the user edits the table."""107 df = make_sorted_df(table_data)108 return create_chart(df)109 110 111# --- Build the initial state ---112initial_df = make_sorted_df()113 114# --- UI ---115with gr.Blocks(title="FizzBuzz LLM Benchmark") as demo:116 gr.Markdown(DESCRIPTION)117 plot = gr.Plot(value=create_chart(initial_df))118 gr.Markdown("### All Model Scores")119 table = gr.Dataframe(120 value=initial_df,121 interactive=True,122 column_count=(3, "fixed"),123 )124 table.input(on_data_edit, inputs=[table], outputs=[plot])125 126# Define the custom CSS127 css = """128 .gradio-container {129 max-width: 800px !important;130 margin-left: auto !important;131 margin-right: auto !important;132 }133 """134 135demo.launch(css=css)136