NUPA-Anonymous/Performance

1import os2import json3import plotly.graph_objects as go4import dash5from dash import dcc, html, Input, Output6 7# 创建 Dash 应用8app = dash.Dash(__name__)9 10# 文件名称列表11file_names = [12    'GPT-4o_statistics.txt',13    'GPT-4o-mini_statistics.txt',14    'Llama-3.1-8B-ft_statistics.txt',15    'Llama-3.1-8B_statistics.txt',16    'Llama-3.1-70B_statistics.txt',17    'Mixtral-8x7B_statistics.txt',18    'Qwen2-72B_statistics.txt',19    'Qwen2-7B_statistics.txt',20    'Llama-2-7b-hf_statistics.txt',21]22with open("./test_results_report/GPT-4o_statistics.txt", "r") as f:23    results = json.load(f)24keys = list(results["exact_match"].keys())25 26def load_data(file_name, main_metric="exact_match", r=(0, len(keys))):27    tasks = []28    well_learned_digit = []29    has_performance_digit = []30    in_domain = []31    out_domain = []32    short_range = []33    medium_range = []34    long_range = []35    very_long_range = []36 37    with open(f"./test_results_report/{file_name}", "r") as f:38        stats = json.load(f)39 40    stats_exm = stats[main_metric]41    for key in keys[r[0]:r[1]]:42        words = key.split("_")43        domain_3 = words.pop()44        domain_2 = words.pop()45        domain_1 = words.pop()46        task = " ".join(list(map(str.capitalize, words)))47        tasks.append(f"{task}<br />{domain_1}")48 49        for metric in ["well_learned_digit", "has_performance_digit", "in_domain", "out_domain", "short_range", "medium_range", "long_range", "very_long_range"]:50            eval(f"{metric}.append(stats_exm['{key}']['{metric}'])")51    return tasks, well_learned_digit, has_performance_digit, in_domain, out_domain, short_range, medium_range, long_range, very_long_range 52 53# 加载任务列表54intTasks = ["Add", "Sub", "Max", "Max Hard", "Multiply Hard", "Multiply Easy", "Digit Max", "Digit Add", "Get Digit", "Length", "Truediv", "Floordiv", "Mod", "Mod Easy", "Count", "Sig", "To Scient"]55floatTasks = ["Add", "Sub", "Max", "Max Hard", "Multiply Hard", "Multiply Easy", "Digit Max", "Digit Add", "Get Digit", "Length", "To Scient"]56fractionTasks = ["Add", "Add Easy", "Sub", "Max", "Multiply Hard", "Multiply Easy", "Truediv", "To Float"]57sciTasks = ["Add", "Sub", "Max", "Max Hard", "Multiply Hard", "Multiply Easy", "To Float"]58 59tasks, well_learned_digit, has_performance_digit, in_domain, out_domain, short_range, medium_range, long_range, very_long_range = load_data("GPT-4o_statistics.txt")60 61# 去重并排序62unique_tasks = sorted(list(set(tasks)))63 64def plot(main_metric, selected_files, selected_metrics, selected_tasks, r):65    colors = ["#2C6344", "#5F9C61", "#A4C97C", "#61496D", "#B092B6", "#CAC1D4", "#308192", "#E38D26", "#F1CC74", "#C74D26", "#5EA7B8", "#AED2E2"]66    colors.reverse()67 68    fig = go.Figure()69 70    for idx, file_name in enumerate(selected_files):71        tasks, well_learned_digit, has_performance_digit, in_domain, out_domain, short_range, medium_range, long_range, very_long_range = load_data(file_name, main_metric=main_metric, r=r)72        tasks_new = []73        performance = []74        tasks_old = []75        for i, task in enumerate(tasks):76            if task in selected_tasks:77                tasks_new += [task] * len(selected_metrics)78                tasks_old += [task]79                for selected_metric in selected_metrics:80                    performance += [eval(selected_metric)[i]]81 82        fig.add_trace(go.Bar(83            x=[tasks_new, ["S", "M", "L", "XL"] * len(tasks_old)],84            y=performance,85            name=file_name[:-15],86            marker_color=colors[idx % len(colors)]87        ))88 89    fig.update_layout(90        barmode='group',91        xaxis_tickangle=-45,92        template="ggplot2",93        autosize=False,94        width=1500,95        height=400,96        xaxis=dict(showgrid=False),97        title=" ".join(list(map(str.capitalize, main_metric.split("_")))),98        margin=dict(l=20, r=10, t=80, b=20),99    )100 101    return fig102 103# 定义应用程序布局104app.layout = html.Div([105    html.H1("NUPA Performance", style={"textAlign": "center", "marginBottom": "20px"}),106 107    # Metric 选择单选框108    html.Div([109        html.Label("Select Metric:", style={"fontWeight": "bold", "marginRight": "10px"}),110        dcc.RadioItems(111            id='metric-selector',112            options=[113                {'label': 'Exact Match', 'value': 'exact_match'},114                {'label': 'Digit Match', 'value': 'digit_match'},115                {'label': 'Dlength', 'value': 'dlength'}116            ],117            value='exact_match',  # 默认值118            inline=True,119            style={"marginBottom": "20px"}120        ),121    ], style={"padding": "10px", "border": "1px solid #ccc", "borderRadius": "5px", "marginBottom": "20px"}),122 123    # 文件选择复选框124    html.Div([125        html.Label("Select Models:", style={"fontWeight": "bold", "marginRight": "10px"}),126        dcc.Checklist(127            id='file-selector',128            options=[{'label': file_name[:-15], 'value': file_name} for file_name in file_names],129            value=['GPT-4o_statistics.txt', 'Llama-3.1-8B-ft_statistics.txt', 'Mixtral-8x7B_statistics.txt', 'Qwen2-72B_statistics.txt'], 130            inline=True,131            style={"marginBottom": "20px"}132        ),133    ], style={"padding": "10px", "border": "1px solid #ccc", "borderRadius": "5px", "marginBottom": "20px"}),134 135    # 任务选择复选框（按组分组）136    html.Div([137        html.H4("Integer Tasks", style={"fontWeight": "bold", "marginTop": "20px"}),138        dcc.Checklist(139            id='int-task-selector',140            options=[{'label': task, 'value': task + '<br />' + 'Integer'} for task in intTasks],141            value=['Add<br />Integer'],142            inline=True,143            style={"marginBottom": "10px"}144        ),145        html.H4("Float Tasks", style={"fontWeight": "bold", "marginTop": "20px"}),146        dcc.Checklist(147            id='float-task-selector',148            options=[{'label': task, 'value': task + '<br />' + 'Float'} for task in floatTasks],149            value=['Add<br />Float'],150            inline=True,151            style={"marginBottom": "10px"}152        ),153        html.H4("Fraction Tasks", style={"fontWeight": "bold", "marginTop": "20px"}),154        dcc.Checklist(155            id='fraction-task-selector',156            options=[{'label': task, 'value': task + '<br />' + 'Fraction'} for task in fractionTasks],157            value=['Add<br />Fraction'],158            inline=True,159            style={"marginBottom": "10px"}160        ),161        html.H4("Scientific Tasks", style={"fontWeight": "bold", "marginTop": "20px"}),162        dcc.Checklist(163            id='sci-task-selector',164            options=[{'label': task, 'value': task + '<br />' + 'ScientificNotation'} for task in sciTasks],165            value=['Add<br />ScientificNotation'],166            inline=True,167            style={"marginBottom": "10px"}168        ),169    ], style={"padding": "10px", "border": "1px solid #ccc", "borderRadius": "5px", "marginBottom": "20px"}),170 171    # 显示图表172    dcc.Graph(id='performance-plot'),173], style={"maxWidth": "1200px", "margin": "0 auto"})174 175# 定义回调函数以更新图表176@app.callback(177    Output('performance-plot', 'figure'),178    Input('metric-selector', 'value'),179    Input('file-selector', 'value'),180    Input('int-task-selector', 'value'),181    Input('float-task-selector', 'value'),182    Input('fraction-task-selector', 'value'),183    Input('sci-task-selector', 'value')184)185def update_figure(main_metric, selected_files, selected_int_tasks, selected_float_tasks, selected_fraction_tasks, selected_sci_tasks):186    selected_metrics = ["short_range", "medium_range", "long_range", "very_long_range"]187    selected_tasks = selected_int_tasks + selected_float_tasks + selected_fraction_tasks + selected_sci_tasks188    r = (0, 42)  # 使用示例范围189    return plot(main_metric, selected_files, selected_metrics, selected_tasks, r)190 191# 运行应用程序192if __name__ == '__main__':193    app.run(debug=True, host='0.0.0.0', port=7860)194