logits/DD-Ranking

1import gradio as gr2import numpy as np3import pandas as pd4 5from constants import *6 7 8def get_data(verified, dataset, ipc, label_type, metric_weights=None):9    if metric_weights is None:10        metric_weights = [1.0 / len(METRICS) for _ in METRICS]11    if not isinstance(label_type, list):12        label_type = [label_type]13 14    data = pd.read_csv("data.csv")15    # filter data with no hlr or ior (no nan)16    data = data.dropna(subset=["hlr", "ior"])17    data["verified"] = data["verified"].apply(lambda x: bool(x))18    data["dataset"] = data["dataset"].apply(lambda x: DATASET_LIST[x])19    data["ipc"] = data["ipc"].apply(lambda x: IPC_LIST[x])20    data["label_type"] = data["label_type"].apply(lambda x: LABEL_TYPE_LIST[x])21    if verified:22        data = data[data["verified"] == verified]23    data = data[data["dataset"] == dataset]24    data = data[data["ipc"] == ipc]25    data = data[data["label_type"].apply(lambda x: x in label_type)]26 27    if len(data) == 0:28        return pd.DataFrame(columns=COLUMN_NAMES)29 30    # create a new column for the score31    data["score"] = data[METRICS[0].lower()] * 0.032    for i, metric in enumerate(METRICS):33        data["score"] += data[metric.lower()] * metric_weights[i] * METRICS_SIGN[i]34    data["score"] = 100 * (np.exp(-0.01 * data["score"]) - np.exp(-1.0)) / (np.exp(1.0) - np.exp(-1.0))35    data = data.sort_values(by="score", ascending=False)36    data["ranking"] = range(1, len(data) + 1)37 38    for metric in METRICS:39        data[metric.lower()] = data[metric.lower()].apply(lambda x: f"{x:.1f}")40    data["score"] = data["score"].apply(lambda x: f"{x:.1f}")41 42    # formatting43    data["method"] = "[" + data["method"] + "](" + data["method_reference"] + ")"44    data["verified"] = data["verified"].apply(lambda x: "✅" if x else "")45    data = data.drop(columns=["method_reference", "dataset", "ipc"])46    data = data[['ranking', 'method', 'verified', 'date', 'label_type', 'hlr', 'ior', 'score']]47    if label_type == "Hard Label":48        data = data.rename(columns={"ranking": "Ranking", "method": "Method", "date": "Date", "label_type": "Label Type", "hlr": "HLR%↓", "ior": "IOR%↑", "score": "LRS%↑", "verified": "Verified"})49    else:50        data = data.rename(columns={"ranking": "Ranking", "method": "Method", "date": "Date", "label_type": "Label Type", "hlr": "HLR%↓", "ior": "IOR%↑", "score": "LRS%↑", "verified": "Verified"})51    return data52 53 54with gr.Blocks() as leaderboard:55    gr.HTML(LEADERBOARD_HEADER)56    gr.Markdown(LEADERBOARD_INTRODUCTION)57 58    verified = gr.Checkbox(59        label="Verified by DD-Ranking Team (Uncheck to view all submissions)",60        value=True,61        interactive=True62    )63 64    dataset = gr.Radio(65        label="Dataset",66        choices=DATASET_LIST,67        value=DATASET_LIST[0],68        interactive=True,69    )70    ipc = gr.Radio(71        label="IPC",72        choices=DATASET_IPC_LIST[dataset.value],73        value=DATASET_IPC_LIST[dataset.value][0],74        interactive=True,75        info=IPC_INFO76    )77    label = gr.CheckboxGroup(78        label="Label Type",79        choices=LABEL_TYPE_LIST,80        value=LABEL_TYPE_LIST,81        info=LABEL_TYPE_INFO,82        interactive=True,83    )84 85    with gr.Accordion("Adjust Score Weights", open=False):86        gr.Markdown(WEIGHT_ADJUSTMENT_INTRODUCTION, latex_delimiters=[87              {'left': '$$', 'right': '$$', 'display': True},88              {'left': '$', 'right': '$', 'display': False},89              {'left': '\\(', 'right': '\\)', 'display': False},90              {'left': '\\[', 'right': '\\]', 'display': True}91          ])92        metric_sliders = []93        # for metric in METRICS:94        #     metric_sliders.append(gr.Slider(label=f"Weight for {metric}", minimum=0.0, maximum=1.0, value=0.5, interactive=True))95        metric_sliders.append(96            gr.Slider(label=f"Weight for HLR", minimum=0.0, maximum=1.0, value=0.5, interactive=True))97        adjust_btn = gr.Button("Adjust Weights")98 99    with gr.Accordion("Metric Definitions", open=False):100        gr.Markdown(METRIC_DEFINITION_INTRODUCTION, latex_delimiters=[101              {'left': '$$', 'right': '$$', 'display': True},102              {'left': '$', 'right': '$', 'display': False},103              {'left': '\\(', 'right': '\\)', 'display': False},104              {'left': '\\[', 'right': '\\]', 'display': True}105          ])106 107    # metric_weights = [s.value for s in metric_sliders]108    metric_weights = [metric_sliders[0].value, 1.0 - metric_sliders[0].value]109    board = gr.components.Dataframe(110        value=get_data(verified.value, dataset.value, ipc.value, label.value, metric_weights),111        headers=COLUMN_NAMES,112        type="pandas",113        datatype=DATA_TITLE_TYPE,114        interactive=False,115        visible=True,116        max_height=500,117    )118 119    for component in [verified, dataset, ipc, label]:120        component.change(lambda v, d, i, l, *m: gr.components.Dataframe(121            value=get_data(v, d, i, l, [m[0], 1.0 - m[0]]),122            headers=COLUMN_NAMES,123            type="pandas",124            datatype=DATA_TITLE_TYPE,125            interactive=False,126            visible=True,127            max_height=500,128        ), inputs=[verified, dataset, ipc, label] + metric_sliders, outputs=board)129 130    dataset.change(lambda d, i: gr.Radio(131        label="IPC",132        choices=DATASET_IPC_LIST[d],133        value=i if i in DATASET_IPC_LIST[d] else DATASET_IPC_LIST[d][0],134        interactive=True,135        info=IPC_INFO136    ), inputs=[dataset, ipc], outputs=ipc)137 138    adjust_btn.click(fn=lambda v, d, i, l, *m: gr.components.Dataframe(139            value=get_data(v, d, i, l, [m[0], 1.0 - m[0]]),140            headers=COLUMN_NAMES,141            type="pandas",142            datatype=DATA_TITLE_TYPE,143            interactive=False,144            visible=True,145            max_height=500,146        ), inputs=[verified, dataset, ipc, label] + metric_sliders, outputs=board)147 148    citation_button = gr.Textbox(149        value=CITATION_BUTTON_TEXT,150        label=CITATION_BUTTON_LABEL,151        elem_id="citation-button",152        lines=6,153        show_copy_button=True,154    )155 156leaderboard.launch()157