livecodebench/code_generation_samples

1import os2import random3import glob4import json5 6import numpy as np7from flask import Flask, render_template, request8 9app = Flask(__name__)10 11 12with open("problems.json") as f:13    problems = json.load(f)14    problem_choices = [q["question_title"] for q in problems]15 16random_idxs = list(range(len(problems)))17# random.seed(42)18# random.shuffle(random_idxs)19problems = [problems[idx] for idx in random_idxs]20 21with open("all_outputs.json") as f:22    all_outputs = json.load(f)23    all_models = list(all_outputs.keys())24 25 26num_questions_filtered = len(problems)27 28all_correctness_by_problem = {29    idx: {model: np.mean(all_outputs[model][idx]["pass1_list"]) for model in all_models}30    for idx in random_idxs31}32 33 34def calculate_color(performance):35    # Convert performance to a value between 0 and 136    # Calculate the red and green components of the color37    if performance > 0.75:38        return f"rgba(0, 150, 0, 0.5)"39    elif performance > 0.5:40        return f"rgba(50, 150, 0, {performance})"41    elif performance > 0.25:42        return f"rgba(150, 50, 0, {1-performance})"43    else:44        return f"rgba(150, 0, 0,  0.5)"45 46 47all_evaluations_by_problem_colored = [48    (49        trueidx,50        {51            model: {52                "correctness": f"{all_correctness_by_problem[idx][model]*100:.1f}",53                "correctness_color": calculate_color(54                    all_correctness_by_problem[idx][model]55                ),56            }57            for model in all_models58        },59        problems[idx]["difficulty"],60        problems[idx]["question_id"],61    )62    for trueidx, idx in enumerate(random_idxs)63]64 65all_data_for_view_formatted = {66    model: [67        [68            {"code": a, "pass1": b, "metadata": c}69            for a, b, c in zip(70                row["code_list"], row["pass1_list"], row["metadata_list"]71            )72        ]73        # print(row)74        for idx in random_idxs75        for row in [resp[idx]]76    ]77    for model, resp in all_outputs.items()78}79 80 81@app.route("/")82def home():83    # Fetch your data here84    print(all_models)85    return render_template(86        "index.html", models=all_models, problems=all_evaluations_by_problem_colored87    )88 89 90@app.route("/problem/<int:problem_idx>")91def problem(problem_idx):92    # Fetch your data here93 94    data = {95        model: all_data_for_view_formatted[model][problem_idx] for model in all_models96    }97    evaluation = all_evaluations_by_problem_colored[problem_idx][1]98    question = problems[problem_idx]99 100    # print(data)101 102    return render_template(103        "problem.html",104        problem_idx=problem_idx,105        question_id=all_evaluations_by_problem_colored[problem_idx][3],106        evaluation=evaluation,107        models=all_models,108        question=question,109        data=data,110    )111 112 113mini_models = [114    # "DeepSeek-V2",115    "DeepSeek-V3",116    "DeepSeek-R1-Preview",117    # "DSCoder-33b-Ins",118    # "GPT-4-Turbo-2024-04-09",119    "GPT-4O-2024-05-13",120    "Claude-3.5-Sonnet-20240620",121    "Gemini-Flash-2.0-Thinking",122    # "Gemini-Exp-1206",123    # "Claude-3-Sonnet",124    "O1-2024-12-17 (N=1) (High)",125    "QwQ-32B-Preview (N=1)",126]127 128 129@app.route("/mini")130def mini():131    # Fetch your data here132    return render_template(133        "index_mini.html",134        models=mini_models,135        problems=all_evaluations_by_problem_colored,136    )137 138 139@app.route("/problem_mini/<int:problem_idx>")140def problem_mini(problem_idx):141    # Fetch your data here142 143    data = {144        model: all_data_for_view_formatted[model][problem_idx] for model in mini_models145    }146    evaluation = all_evaluations_by_problem_colored[problem_idx][1]147    question = problems[problem_idx]148 149    # print(data)150 151    return render_template(152        "problem_mini.html",153        problem_idx=problem_idx,154        question_id=all_evaluations_by_problem_colored[problem_idx][3],155        evaluation=evaluation,156        models=mini_models,157        question=question,158        data=data,159    )160 161 162if __name__ == "__main__":163    app.run(host="0.0.0.0", port=7860)164