CoderDocSagun/ResearchPaperQuestionandAnswers
0
1import gradio as gr2from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns3import pandas as pd4from apscheduler.schedulers.background import BackgroundScheduler5from huggingface_hub import snapshot_download6 7from src.about import (8 CITATION_BUTTON_LABEL,9 CITATION_BUTTON_TEXT,10 EVALUATION_QUEUE_TEXT,11 INTRODUCTION_TEXT,12 LLM_BENCHMARKS_TEXT,13 TITLE,14)15from src.display.css_html_js import custom_css16from src.display.utils import (17 BENCHMARK_COLS,18 COLS,19 EVAL_COLS,20 EVAL_TYPES,21 AutoEvalColumn,22 ModelType,23 fields,24 WeightType,25 Precision26)27from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN28from src.populate import get_evaluation_queue_df, get_leaderboard_df29from src.submission.submit import add_new_eval30 31 32def restart_space():33 API.restart_space(repo_id=REPO_ID)34 35### Space initialisation36try:37 print(EVAL_REQUESTS_PATH)38 snapshot_download(39 repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN40 )41except Exception:42 restart_space()43try:44 print(EVAL_RESULTS_PATH)45 snapshot_download(46 repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN47 )48except Exception:49 restart_space()50 51 52LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)53 54(55 finished_eval_queue_df,56 running_eval_queue_df,57 pending_eval_queue_df,58) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)59 60def init_leaderboard(dataframe):61 if dataframe is None or dataframe.empty:62 raise ValueError("Leaderboard DataFrame is empty or None.")63 return Leaderboard(64 value=dataframe,65 datatype=[c.type for c in fields(AutoEvalColumn)],66 select_columns=SelectColumns(67 default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],68 cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],69 label="Select Columns to Display:",70 ),71 search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],72 hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],73 filter_columns=[74 ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),75 ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),76 ColumnFilter(77 AutoEvalColumn.params.name,78 type="slider",79 min=0.01,80 max=150,81 label="Select the number of parameters (B)",82 ),83 ColumnFilter(84 AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True85 ),86 ],87 bool_checkboxgroup_label="Hide models",88 interactive=False,89 )90 91 92demo = gr.Blocks(css=custom_css)93with demo:94 gr.HTML(TITLE)95 gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")96 97 with gr.Tabs(elem_classes="tab-buttons") as tabs:98 with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):99 leaderboard = init_leaderboard(LEADERBOARD_DF)100 101 with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):102 gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")103 104 with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):105 with gr.Column():106 with gr.Row():107 gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")108 109 with gr.Column():110 with gr.Accordion(111 f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",112 open=False,113 ):114 with gr.Row():115 finished_eval_table = gr.components.Dataframe(116 value=finished_eval_queue_df,117 headers=EVAL_COLS,118 datatype=EVAL_TYPES,119 row_count=5,120 )121 with gr.Accordion(122 f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",123 open=False,124 ):125 with gr.Row():126 running_eval_table = gr.components.Dataframe(127 value=running_eval_queue_df,128 headers=EVAL_COLS,129 datatype=EVAL_TYPES,130 row_count=5,131 )132 133 with gr.Accordion(134 f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",135 open=False,136 ):137 with gr.Row():138 pending_eval_table = gr.components.Dataframe(139 value=pending_eval_queue_df,140 headers=EVAL_COLS,141 datatype=EVAL_TYPES,142 row_count=5,143 )144 with gr.Row():145 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")146 147 with gr.Row():148 with gr.Column():149 model_name_textbox = gr.Textbox(label="Model name")150 revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")151 model_type = gr.Dropdown(152 choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],153 label="Model type",154 multiselect=False,155 value=None,156 interactive=True,157 )158 159 with gr.Column():160 precision = gr.Dropdown(161 choices=[i.value.name for i in Precision if i != Precision.Unknown],162 label="Precision",163 multiselect=False,164 value="float16",165 interactive=True,166 )167 weight_type = gr.Dropdown(168 choices=[i.value.name for i in WeightType],169 label="Weights type",170 multiselect=False,171 value="Original",172 interactive=True,173 )174 base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")175 176 submit_button = gr.Button("Submit Eval")177 submission_result = gr.Markdown()178 submit_button.click(179 add_new_eval,180 [181 model_name_textbox,182 base_model_name_textbox,183 revision_name_textbox,184 precision,185 weight_type,186 model_type,187 ],188 submission_result,189 )190 191 with gr.Row():192 with gr.Accordion("📙 Citation", open=False):193 citation_button = gr.Textbox(194 value=CITATION_BUTTON_TEXT,195 label=CITATION_BUTTON_LABEL,196 lines=20,197 elem_id="citation-button",198 show_copy_button=True,199 )200 201scheduler = BackgroundScheduler()202scheduler.add_job(restart_space, "interval", seconds=1800)203scheduler.start()204demo.queue(default_concurrency_limit=40).launch()