OPTML-Group/UnlearnCanvas-Benchmark

1 2from datetime import datetime, timedelta3import numpy as np4import pandas as pd5import plotly.express as px6from plotly.graph_objs import Figure7 8# Dummy data creation9 10 11def dummy_data_for_plot(metrics, num_days=30):12    dates = [datetime.now() - timedelta(days=i) for i in range(num_days)]13    data = []14 15    for metric in metrics:16        for date in dates:17            model = f"Model_{metric}"18            score = np.random.uniform(50, 55)19            data.append([date, metric, score, model])20 21    df = pd.DataFrame(data, columns=["date", "task", "score", "model"])22    return df23 24 25def create_metric_plot_obj_1(26    df: pd.DataFrame, metrics: list[str], title: str27) -> Figure:28    """29    Create a Plotly figure object with lines representing different metrics30    and horizontal dotted lines representing human baselines.31 32    :param df: The DataFrame containing the metric values, names, and dates.33    :param metrics: A list of strings representing the names of the metrics34                    to be included in the plot.35    :param title: A string representing the title of the plot.36    :return: A Plotly figure object with lines representing metrics and37             horizontal dotted lines representing human baselines.38    """39 40    # Filter the DataFrame based on the specified metrics41    df = df[df["task"].isin(metrics)]42 43    # Filter the human baselines based on the specified metrics44    # filtered_human_baselines = {k: v for k, v in HUMAN_BASELINE.items() if k in metrics}45 46    # Create a line figure using plotly express with specified markers and custom data47    fig = px.line(48        df,49        x="date",50        y="score",51        color="task",52        markers=True,53        custom_data=["task", "score", "model"],54        title=title,55    )56 57    # Update hovertemplate for better hover interaction experience58    fig.update_traces(59        hovertemplate="<br>".join(60            [61                "Model Name: %{customdata[2]}",62                "Metric Name: %{customdata[0]}",63                "Date: %{x}",64                "Metric Value: %{y}",65            ]66        )67    )68 69    # Update the range of the y-axis70    fig.update_layout(yaxis_range=[0, 100])71 72    # Create a dictionary to hold the color mapping for each metric73    metric_color_mapping = {}74 75    # Map each metric name to its color in the figure76    for trace in fig.data:77        metric_color_mapping[trace.name] = trace.line.color78 79    # Iterate over filtered human baselines and add horizontal lines to the figure80    # for metric, value in filtered_human_baselines.items():81    #     color = metric_color_mapping.get(metric, "blue")  # Retrieve color from mapping; default to blue if not found82    #     location = "top left" if metric == "HellaSwag" else "bottom left"  # Set annotation position83    #     # Add horizontal line with matched color and positioned annotation84    #     fig.add_hline(85    #         y=value,86    #         line_dash="dot",87    #         annotation_text=f"{metric} human baseline",88    #         annotation_position=location,89    #         annotation_font_size=10,90    #         annotation_font_color=color,91    #         line_color=color,92    #     )93 94    return fig95 96 97def dummydf():98    # data = [{"Model": "gpt-35-turbo-1106",99    #          "Agent": "prompt agent",100    #         "Opponent Model": "gpt-4",101    #          "Opponent Agent": "prompt agent",102    #          'Breakthrough': 0,103    #          'Connect Four': 0,104    #          'Blind Auction': 0,105    #          'Kuhn Poker': 0,106    #          "Liar's Dice": 0,107    #          'Negotiation': 0,108    #          'Nim': 0,109    #          'Pig': 0,110    #          'Iterated Prisoners Dilemma': 0,111    #          'Tic-Tac-Toe': 0112    #          },113    #         {"Model": "Llama-2-70b-chat-hf",114    #         "Agent": "prompt agent",115    #          "Opponent Model": "gpt-4",116    #          "Opponent Agent": "prompt agent",117    #          'Breakthrough': 1,118    #          'Connect Four': 0,119    #          'Blind Auction': 0,120    #          'Kuhn Poker': 0,121    #          "Liar's Dice": 0,122    #          'Negotiation': 0,123    #          'Nim': 0,124    #          'Pig': 0,125    #          'Iterated Prisoners Dilemma': 0,126    #          'Tic-Tac-Toe': 0127    #          },128    #         {"Model": "gpt-35-turbo-1106",129    #          "Agent": "ToT agent",130    #         "Opponent Model": "gpt-4",131    #          "Opponent Agent": "prompt agent",132    #          'Breakthrough': 0,133    #          'Connect Four': 0,134    #          'Blind Auction': 0,135    #          'Kuhn Poker': 0,136    #          "Liar's Dice": 0,137    #          'Negotiation': 0,138    #          'Nim': 0,139    #          'Pig': 0,140    #          'Iterated Prisoners Dilemma': 0,141    #          'Tic-Tac-Toe': 0142    #          },143    #         {"Model": "Llama-2-70b-chat-hf",144    #         "Agent": "CoT agent",145    #          "Opponent Model": "gpt-4",146    #          "Opponent Agent": "prompt agent",147    #          'Breakthrough': 0,148    #          'Connect Four': 0,149    #          'Blind Auction': 0,150    #          'Kuhn Poker': 0,151    #          "Liar's Dice": 0,152    #          'Negotiation': 0,153    #          'Nim': 0,154    #          'Pig': 0,155    #          'Iterated Prisoners Dilemma': 0,156    #          'Tic-Tac-Toe': 0157    #          }]158    df = pd.read_csv('./assets/uc_result.csv')159    return df160