Effyis/LLm-benchmark-2
1
1import gradio as gr2import numpy as np3import pandas as pd4import argparse5 6def make_default_md():7 leaderboard_md = f"""8 # 🏆 LLms Benchmark9 10 The main goal of this project is to utilize Large Language Models (LLMs) to extract specific information from PDF documents and organize it into a structured JSON format.11 12 To achieve this objective, we are assessing various LLMs on two benchmarks:13 14 1. [Benchmark1](https://huggingface.co/spaces/Nechba/LLms-Benchmark/blob/main/dataset.jsonl): 15 This benchmark consists of a dataset of 59 pages as context and corresponding JSON extracts from "Interchange and Service Fees Manual: Europe Region".16 17 2. [Benchmark2](https://huggingface.co/datasets/Effyis/Table-Extraction): 18 This benchmark comprises a dataset of 16573 tables as context and corresponding JSON extracts.19 """20 return leaderboard_md21 22 23def make_arena_leaderboard_md(total_models):24 leaderboard_md = f"""25Total #models: **{total_models}**. Last updated: Juin 01, 2024.26 27"""28 return leaderboard_md29 30def model_hyperlink(model_name, link):31 return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'32 33def load_leaderboard_table_csv(filename, add_hyperlink=True):34 rows = []35 with open(filename, 'r') as file:36 lines = file.readlines()37 heads = [v.strip() for v in lines[0].split(",")]38 for line in lines[1:]:39 row = [v.strip() for v in line.split(",")]40 item = {}41 for h, v in zip(heads, row):42 item[h] = v43 if add_hyperlink:44 item["Model"] = model_hyperlink(item["Model"], item["Link"])45 item["Notebook link"] = model_hyperlink("Notebook", item["Notebook link"])46 rows.append(item)47 return rows48 49def get_arena_table(model_table_df):50 # change type Percentage of values column of df51 model_table_df["Percentage of values"] = model_table_df["Percentage of values"].astype(float)52 model_table_df["Percentage of keys"] = model_table_df["Percentage of keys"].astype(float)53 model_table_df["Average time (s)"] = model_table_df["Average time (s)"].astype(float)54 arena_df = model_table_df.sort_values(by=["Percentage of values"], ascending=False)55 values = []56 if not arena_df.empty: # Check if arena_df is not empty57 for i in range(len(arena_df)):58 row = []59 model_name = arena_df["Model"].values[i] # Access model name directly without index 060 row.append(model_name)61 row.append(arena_df.iloc[i]["Percentage of values"])62 row.append(arena_df.iloc[i]["Percentage of keys"])63 row.append(arena_df.iloc[i]["Average time (s)"])64 row.append(arena_df.iloc[i]["Notebook link"])65 row.append(arena_df.iloc[i]["License"])66 # row.append(arena_df.iloc[i]["Link"])67 values.append(row)68 return values69 70def build_leaderboard_tab(leaderboard_table_file1,leaderboard_table_file2, show_plot=False):71 default_md = make_default_md()72 md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown")73 if leaderboard_table_file1:74 data1 = load_leaderboard_table_csv(leaderboard_table_file1)75 model_table_df1 = pd.DataFrame(data1)76 data2 = load_leaderboard_table_csv(leaderboard_table_file2)77 model_table_df2 = pd.DataFrame(data2)78 with gr.Tabs() as tabs:79 with gr.Tab(" 🏅 Benchmark 1", id=0):80 arena_table_vals = get_arena_table(model_table_df1)81 md = make_arena_leaderboard_md(len(arena_table_vals))82 gr.Markdown(md, elem_id="leaderboard_markdown")83 gr.Dataframe(84 headers=[85 "Model",86 "Percentage of values (%)",87 "Percentage of keys (%)",88 "Average time (s)",89 "Code",90 "License",91 ],92 datatype=[93 "markdown",94 "number",95 "number",96 "number",97 "markdown",98 "str"99 ],100 value=arena_table_vals,101 elem_id="arena_leaderboard_dataframe",102 height=700,103 column_widths=[200, 150, 150, 130, 100, 140],104 wrap=True,105 )106 # Displaying a note about the leaderboard analysis107 gr.Markdown(108 f"""Note: Upon reviewing the leaderboard, it's evident that two models, Gemini and OpenHermes, outperform the others. Our next step involves a detailed analysis and comparison of the results obtained by these two models.""",109 elem_id="leaderboard_markdown"110 )111 112 # Displaying additional statistics for Gemini and OpenHermes113 gr.Markdown(114 f"""## More Statistics for Gemini and OpenHermes\n115 Now we will focus on Gemini and OpenHermes, diving deeper into their performance for a comprehensive comparison.""",116 elem_id=0117 )118 119 # Displaying the confusion matrices for Gemini and OpenHermes120 with gr.Row():121 with gr.Column():122 gr.Markdown(123 "#### Figure 1: Gemini Confusion Matrix"124 )125 plot_1 = gr.Image("./Benchmark1/gemini_cm.png", show_label=False)126 # Detailed analysis of Gemini's performance127 gr.Markdown(128 """### True Positives:129 Our model correctly identified all 18 pages lacking the desired information (Payment product, FeeTier, and Rate).130 131 ### True Negatives:132 The model successfully predicted desired information on 39 out of 41 pages with an accuracy ranging from 12% to 100%. (For more details about accuracy, check the Notebook [here](https://huggingface.co/spaces/Effyis/LLms-Benchmark/blob/main/Benchmark1/gemini.ipynb))133 134 ### False Negatives:135 In 2 instances, the model incorrectly predicted that pages lacked the desired information when they actually contained it.136 137 ### False Positives:138 The model incorrectly predicted that 0 pages contained the desired information when they were actually missing it."""139 )140 141 with gr.Column():142 gr.Markdown(143 "#### Figure 2: OpenHermes Confusion Matrix"144 )145 plot_2 = gr.Image("./Benchmark1/openhermes_cm.png", show_label=False)146 # Detailed analysis of OpenHermes's performance147 gr.Markdown(148 """### True Positives:149 Our model correctly identified 12 out of 18 pages lacking the desired information (Payment product, FeeTier, and Rate).150 151 ### True Negatives:152 The model successfully predicted desired information on 21 out of 41 pages with an accuracy ranging from 5% to 66%. (For more details about accuracy, check the Notebook [here](https://huggingface.co/spaces/Effyis/LLms-Benchmark/blob/main/Benchmark1/openhermes.ipynb))153 154 ### False Negatives:155 In 20 instances, the model incorrectly predicted that pages lacked the desired information when they actually contained it.156 157 ### False Positives:158 The model incorrectly predicted that 6 pages contained the desired information when they were actually missing it."""159 )160 161 # Conclusion based on the analysis162 gr.Markdown(163 """## Conclusion\n164 Upon analyzing the performance of Gemini and OpenHermes, it becomes evident that both models exhibit strengths and weaknesses. Gemini demonstrates higher accuracy in identifying pages lacking desired information and also performs better in predicting pages containing the desired information. On the other hand, while OpenHermes shows good results in identifying pages lacking desired information, it achieves only 50% accuracy in predicting pages containing the desired information. Further fine-tuning of both models could lead to enhanced overall performance."""165 )166 167 168 with gr.Tab("🏅 Benchmark 2", id=1):169 arena_table_vals = get_arena_table(model_table_df2)170 md = make_arena_leaderboard_md(len(arena_table_vals))171 gr.Markdown(md, elem_id="leaderboard_markdown")172 gr.Dataframe(173 headers=[174 "Model",175 "Percentage of values (%)",176 "Percentage of keys (%)",177 "Average time (s)",178 "Code",179 "License",180 ],181 datatype=[182 "markdown",183 "number",184 "number",185 "number",186 "markdown",187 "str"188 ],189 value=arena_table_vals,190 elem_id="arena_leaderboard_dataframe",191 height=700,192 column_widths=[200, 150, 150, 130, 100, 140],193 wrap=True,194 ) 195# gr.Markdown(196# f"""197# Note: For this benchmark, only a sample of 100 points from the dataset is utilized. It's evident that the data context is straightforward, yet it includes Arabic names. This could explain the lower performance scores of the models, as they may lack robust capabilities in handling Arabic names.""",198# elem_id="leaderboard_markdown"199# )200 201 else:202 pass203 return [md_1,plot_1, plot_2]204 205block_css = """206#notice_markdown {207 font-size: 104%208}209#notice_markdown th {210 display: none;211}212#notice_markdown td {213 padding-top: 6px;214 padding-bottom: 6px;215}216#leaderboard_markdown {217 font-size: 104%218}219#leaderboard_markdown td {220 padding-top: 6px;221 padding-bottom: 6px;222}223#leaderboard_dataframe td {224 line-height: 0.1em;225}226footer {227 display:none !important228}229.sponsor-image-about img {230 margin: 0 20px;231 margin-top: 20px;232 height: 40px;233 max-height: 100%;234 width: auto;235 float: left;236}237"""238 239def build_demo(leaderboard_table_file1, leaderboard_table_file2):240 text_size = gr.themes.sizes.text_lg241 with gr.Blocks(242 title="LLMS Benchmark",243 theme=gr.themes.Base(text_size=text_size),244 css=block_css,245 ) as demo:246 leader_components = build_leaderboard_tab(247 leaderboard_table_file1,leaderboard_table_file2, show_plot=True248 )249 return demo250 251if __name__ == "__main__":252 parser = argparse.ArgumentParser()253 parser.add_argument("--share", action="store_true")254 args = parser.parse_args()255 256 leaderboard_table_file1 = "./Benchmark1/leaderboard.csv"257 leaderboard_table_file2 = "./Benchmark2/leaderboard.csv"258 demo = build_demo(leaderboard_table_file1,leaderboard_table_file2)259 demo.launch(share=args.share)