MCILAB/LLM_Alignment_Evaluation
3
1import gradio as gr2from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns3import pandas as pd4from apscheduler.schedulers.background import BackgroundScheduler5from huggingface_hub import snapshot_download6 7from src.about import (8 CITATION_BUTTON_LABEL,9 CITATION_BUTTON_TEXT,10 EVALUATION_QUEUE_TEXT,11 INTRODUCTION_TEXT,12 LLM_BENCHMARKS_TEXT,13 TITLE,14)15 16from src.data_request import(17 DATASET_REQUEST_TEXT 18)19 20from src.display.css_html_js import custom_css21from src.display.utils import (22 BENCHMARK_COLS,23 COLS,24 EVAL_COLS,25 EVAL_TYPES,26 AutoEvalColumn,27 ModelType,28 fields,29 WeightType,30 Precision31)32from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN33from src.populate import get_evaluation_queue_df, get_leaderboard_df34from src.submission.submit import add_new_eval35 36 37def restart_space():38 API.restart_space(repo_id=REPO_ID)39 40### Space initialisation41# try:42# print(EVAL_REQUESTS_PATH)43# snapshot_download(44# repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN45# )46# except Exception:47# restart_space()48try:49 print(EVAL_RESULTS_PATH)50 snapshot_download(51 repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN52 )53except Exception:54 restart_space()55 56 57LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)58 59# (60# finished_eval_queue_df,61# running_eval_queue_df,62# pending_eval_queue_df,63# ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)64 65def init_leaderboard(dataframe):66 if dataframe is None or dataframe.empty:67 raise ValueError("Leaderboard DataFrame is empty or None.")68 return Leaderboard(69 value=dataframe,70 datatype=[c.type for c in fields(AutoEvalColumn)],71 select_columns=SelectColumns(72 default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],73 cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],74 label="Select Columns to Display:",75 ),76 search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],77 hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],78 filter_columns=[79 ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),80 ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),81 ColumnFilter(82 AutoEvalColumn.params.name,83 type="slider",84 min=0.01,85 max=150,86 label="Select the number of parameters (B)",87 ),88 ],89 bool_checkboxgroup_label="Hide models",90 interactive=False,91 )92 93 94demo = gr.Blocks(css=custom_css)95with demo:96 gr.HTML(TITLE)97 gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")98 99 with gr.Tabs(elem_classes="tab-buttons") as tabs:100 with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):101 leaderboard = init_leaderboard(LEADERBOARD_DF)102 103 with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=1):104 gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")105 106 with gr.TabItem("📂 Dataset request", elem_id="dataset-request-tab", id=2): # New tab107 gr.Markdown(DATASET_REQUEST_TEXT, elem_classes="markdown-text")108 109 with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):110 with gr.Column():111 with gr.Row():112 gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")113 114 # with gr.Column():115 # with gr.Accordion(116 # f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",117 # open=False,118 # ):119 # with gr.Row():120 # finished_eval_table = gr.components.Dataframe(121 # value=finished_eval_queue_df,122 # headers=EVAL_COLS,123 # datatype=EVAL_TYPES,124 # row_count=5,125 # )126 # with gr.Accordion(127 # f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",128 # open=False,129 # ):130 # with gr.Row():131 # running_eval_table = gr.components.Dataframe(132 # value=running_eval_queue_df,133 # headers=EVAL_COLS,134 # datatype=EVAL_TYPES,135 # row_count=5,136 # )137 138 # with gr.Accordion(139 # f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",140 # open=False,141 # ):142 # with gr.Row():143 # pending_eval_table = gr.components.Dataframe(144 # value=pending_eval_queue_df,145 # headers=EVAL_COLS,146 # datatype=EVAL_TYPES,147 # row_count=5,148 # )149 with gr.Row():150 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")151 152 with gr.Row():153 with gr.Column():154 model_name_textbox = gr.Textbox(label="Model name")155 revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")156 model_type = gr.Dropdown(157 choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],158 label="Model type",159 multiselect=False,160 value=None,161 interactive=True,162 )163 164 with gr.Column():165 precision = gr.Dropdown(166 choices=[i.value.name for i in Precision if i != Precision.Unknown],167 label="Precision",168 multiselect=False,169 value="float16",170 interactive=True,171 )172 weight_type = gr.Dropdown(173 choices=[i.value.name for i in WeightType],174 label="Weights type",175 multiselect=False,176 value="Original",177 interactive=True,178 )179 base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")180 181 submit_button = gr.Button("Submit Eval")182 submission_result = gr.Markdown()183 submit_button.click(184 add_new_eval,185 [186 model_name_textbox,187 base_model_name_textbox,188 revision_name_textbox,189 precision,190 weight_type,191 model_type,192 ],193 submission_result,194 )195 196 with gr.Row():197 with gr.Accordion("📙 Citation", open=False):198 citation_button = gr.Textbox(199 value=CITATION_BUTTON_TEXT,200 label=CITATION_BUTTON_LABEL,201 lines=20,202 elem_id="citation-button",203 show_copy_button=True,204 )205 206scheduler = BackgroundScheduler()207scheduler.add_job(restart_space, "interval", seconds=24*3600)208scheduler.start()209demo.queue(default_concurrency_limit=1).launch()