jordyvl/ece

1import evaluate2import json3import sys4from pathlib import Path5import gradio as gr6 7import numpy as np8import pandas as pd9import ast10 11# from ece import ECE  # loads local instead12 13 14import matplotlib.pyplot as plt15import matplotlib.patches as mpatches16 17"""18import seaborn as sns19sns.set_style('white')20sns.set_context("paper", font_scale=1)21"""22# plt.rcParams['figure.figsize'] = [10, 7]23plt.rcParams["figure.dpi"] = 30024plt.switch_backend(25    "agg"26)  # ; https://stackoverflow.com/questions/14694408/runtimeerror-main-thread-is-not-in-main-loop27 28sliders = [29    gr.Slider(0, 100, value=10, label="n_bins"),30    gr.Slider(31        0, 100, value=None, label="bin_range", visible=False32    ),  # DEV: need to have a double slider33    gr.Dropdown(choices=["equal-range", "equal-mass"], value="equal-range", label="scheme"),34    gr.Dropdown(choices=["upper-edge", "center"], value="upper-edge", label="proxy"),35    gr.Dropdown(choices=[1, 2, np.inf], value=1, label="p"),36]37 38slider_defaults = [slider.value for slider in sliders]39 40# example data41df = dict()42df["predictions"] = [[0.6, 0.2, 0.2], [0, 0.95, 0.05], [0.7, 0.1, 0.2]]43df["references"] = [0, 1, 2]44 45component = gr.Dataframe(46    headers=["predictions", "references"], col_count=2, datatype="number", type="pandas"47)48 49component.value = [50    [[0.6, 0.2, 0.2], 0],51    [[0.7, 0.1, 0.2], 2],52    [[0, 0.95, 0.05], 1],53]54sample_data = [[component] + slider_defaults]  ##json.dumps(df)55 56 57local_path = Path(sys.path[0])58metric = evaluate.load("jordyvl/ece")59# ECE()60# module = evaluate.load("jordyvl/ece")61# launch_gradio_widget(module)62 63"""l64Switch inputs and compute_fn65"""66 67 68def default_plot():69    fig = plt.figure()70    ax1 = plt.subplot2grid((3, 1), (0, 0), rowspan=2)71    ax2 = plt.subplot2grid((3, 1), (2, 0))72    ranged = np.linspace(0, 1, 10)73    ax1.plot(74        ranged,75        ranged,76        color="darkgreen",77        ls="dotted",78        label="Perfect",79    )80 81    # Bin differences82    ax1.set_ylabel("Conditional Expectation")83    ax1.set_ylim([0, 1.05])  # respective to bin range84    ax1.set_title("Reliability Diagram")85    ax1.set_xlim([-0.05, 1.05])  # respective to bin range86 87    # Bin frequencies88    ax2.set_xlabel("Confidence")89    ax2.set_ylabel("Count")90    ax2.legend(loc="upper left")  # , ncol=291    ax2.set_xlim([-0.05, 1.05])  # respective to bin range92 93    return fig, ax1, ax294 95 96def reliability_plot(results):97    # DEV: might still need to write tests in case of equal mass binning98    # DEV: nicer would be to plot like a polygon99    # see: https://github.com/markus93/fit-on-the-test/blob/main/Experiments_Synthetic/binnings.py100 101    def over_under_confidence(bins, patches):102        colors = []103        for j, bin in enumerate(bins):104            perfect = bin105            if j == len(patches):106              j = len(patches) -1107            empirical = patches[j].get_height()108 109            bin_color = (110                "limegreen"111                if np.allclose(perfect, empirical)112                else "dodgerblue"113                if empirical < perfect114                else "orangered"115            )116            colors.append(bin_color)117        return colors118 119    fig, ax1, ax2 = default_plot()120 121    # Bin differences122    bins_with_left_edge = np.insert(results["y_bar"], 0, 0, axis=0)123    B, bins, patches = ax1.hist(124        results["y_bar"],125        weights=np.nan_to_num(results["p_bar"][:-1], copy=True, nan=0),126        bins=bins_with_left_edge,127    )128    colors = over_under_confidence(bins, patches)129    for b in range(len(B)):130        patches[b].set_facecolor(colors[b])  # color based on over/underconfidence131    132    ax1handles = [133        mpatches.Patch(color="orangered", label="Overconfident"),134        mpatches.Patch(color="limegreen", label="Perfect", linestyle="dotted"),135        mpatches.Patch(color="dodgerblue", label="Underconfident"),136    ]137 138    # Bin frequencies139    anindices = np.where(~np.isnan(results["p_bar"][:-1]))[0]140    n_bins = len(results["y_bar"])141    bin_freqs = np.zeros(n_bins)142    bin_freqs[anindices] = results["bin_freq"]143    B, newbins, patches = ax2.hist(144        results["y_bar"], weights=bin_freqs, color="midnightblue", bins=bins_with_left_edge145    )146 147    acc_plt = ax2.axvline(x=results["accuracy"], ls="solid", lw=3, c="black", label="Accuracy")148    conf_plt = ax2.axvline(149        x=results["p_bar_cont"], ls="dotted", lw=3, c="#444", label="Avg. confidence"150    )151 152    ax1.legend(loc="lower right", handles=ax1handles)153    ax2.legend(handles=[acc_plt, conf_plt])154    ax1.set_xticks(bins_with_left_edge)155    ax2.set_xticks(bins_with_left_edge)156    plt.tight_layout()157    return fig158 159 160def compute_and_plot(data, n_bins, bin_range, scheme, proxy, p):161    # DEV: check on invalid datatypes with better warnings162 163    if isinstance(data, pd.DataFrame):164        data.dropna(inplace=True)165 166    predictions = [167        ast.literal_eval(prediction) if not isinstance(prediction, list) else prediction168        for prediction in data["predictions"]169    ]170    references = [reference for reference in data["references"]]171 172    results = metric._compute(173        predictions,174        references,175        n_bins=n_bins,176        scheme=scheme,177        proxy=proxy,178        p=p,179        detail=True,180    )181    print(results)182    plot = reliability_plot(results)183    return results["ECE"], plot184 185 186outputs = [gr.outputs.Textbox(label="ECE"), gr.Plot(label="Reliability diagram")]187# outputs[1].value = default_plot().__dict__ #Does not work; yet needs to be JSON encoded188 189iface = gr.Interface(190    fn=compute_and_plot,191    inputs=[component] + sliders,192    outputs=outputs,193    description=metric.info.description,194    article=evaluate.utils.parse_readme(local_path / "README.md"),195    title=f"Metric: {metric.name}",196    # examples=sample_data; # ValueError: Examples argument must either be a directory or a nested list, where each sublist represents a set of inputs.197).launch()