caskcsg/LongBench-Pro-Leaderboard
1
1#!/usr/bin/env python32# -*- coding: utf-8 -*-3"""4LongBenchmark Results Visualization5"""6 7import json8import re9import pandas as pd10from pathlib import Path11import gradio as gr12import plotly.graph_objects as go13 14with open('./output/model_info.json', 'r', encoding='utf-8') as f:15 MODLE_INFO_DICT = json.load(f)16 17def get_color(index):18 """Generate color based on index, using golden angle to ensure uniform and infinite color distribution"""19 # Golden angle approx 137.508 degrees20 hue = (index * 137.508) % 36021 # Fixed saturation 70%, lightness 60%22 return f"hsl({hue}, 70%, 60%)"23 24# Custom CSS25CUSTOM_CSS = """26/* Force title center */27h1 {28 text-align: center;29 display: block;30}31 32/* Header center */33#leaderboard_table th, 34#leaderboard_table th button, 35#leaderboard_table th span {36 text-align: center !important;37 justify-content: center !important;38}39 40/* Content column center: starting from 3rd column */41#leaderboard_table td:nth-child(n+3) {42 text-align: center !important;43}44 45/* Make tab labels bold */46button[role="tab"] {47 font-weight: bold !important;48}49"""50 51class ResultParser:52 def __init__(self, output_dir: str):53 self.output_dir = Path(output_dir)54 self.results = []55 56 def parse_filename(self, filename: str):57 """Parse filename to extract context length and thinking status"""58 # Extract context length59 context_match = re.search(r'context-(\d+)', filename)60 context_length = int(context_match.group(1)) if context_match else 061 62 filename_lower = filename.lower()63 # Check nonthinking64 has_nonthinking = 'nonthinking' in filename_lower65 # Check thinking66 has_thinking = 'thinking' in filename_lower and not has_nonthinking67 68 return context_length, has_thinking, has_nonthinking69 70 def parse_result_file(self, model_name: str, file_path: Path):71 """Parse single result file"""72 try:73 with open(file_path, 'r', encoding='utf-8') as f:74 data = json.load(f)75 76 context_length, has_thinking, has_nonthinking = self.parse_filename(file_path.name)77 # Use date field as evaluation date78 eval_date = data.get('date', "Unknown")79 80 # Extract BoN data81 bon_data = {}82 for bon_key in ['BoN-1', 'BoN-2', 'BoN-3']:83 if bon_key in data and 'overall_metric' in data[bon_key]:84 bon_data[bon_key] = data[bon_key]['overall_metric']85 86 result = {87 'model_name': model_name,88 'eval_date': eval_date,89 'context_length': context_length,90 'has_thinking': has_thinking,91 'has_nonthinking': has_nonthinking,92 'overall_metric': data.get('average_overall_metric', 0.0),93 'token_length_metrics': data.get('average_token_length_metric', {}),94 'contextual_requirement': data.get('average_contextual_requirement_metric', {}),95 'difficulty': data.get('average_difficulty_metric', {}),96 'primary_task': data.get('average_primary_task_metric', {}),97 'language': data.get('average_language_metric', {}),98 'bon_data': bon_data, # Store BoN-1, BoN-2, BoN-3 overall_metric99 'pass_at_k': {100 'Pass@1': data.get('pass@1'),101 'Pass@2': data.get('pass@2'),102 'Pass@3': data.get('pass@3')103 }104 }105 106 return result107 108 except Exception as e:109 print(f"Error parsing file {file_path}: {e}")110 return None111 112 def scan_all_results(self):113 """Scan all model result files"""114 self.results = []115 116 if not self.output_dir.exists():117 print(f"Output directory does not exist: {self.output_dir}")118 return119 120 # Traverse all model directories121 for model_dir in self.output_dir.iterdir():122 if not model_dir.is_dir():123 continue124 125 model_name = model_dir.name126 print(f"Scanning model: {model_name}")127 128 # Find all _summary.json files129 for file_path in model_dir.glob("*_summary.json"):130 print(f" Parsing file: {file_path.name}")131 result = self.parse_result_file(model_name, file_path)132 if result:133 self.results.append(result)134 135 print(f"Total parsed {len(self.results)} result files")136 137 def get_leaderboard_data(self):138 """Get leaderboard data"""139 if not self.results:140 return pd.DataFrame()141 142 # Aggregate data by model name143 model_groups = {}144 for result in self.results:145 model_name = result['model_name']146 if model_name not in model_groups:147 model_groups[model_name] = {148 'dates': [],149 'contexts': [],150 'thinking_scores': [],151 'non_thinking_scores': []152 }153 154 group = model_groups[model_name]155 group['dates'].append(result['eval_date'])156 group['contexts'].append(result['context_length'])157 158 score = result['overall_metric']159 if result['has_thinking']:160 group['thinking_scores'].append(score)161 else:162 group['non_thinking_scores'].append(score)163 164 leaderboard_data = []165 for model_name, group in model_groups.items():166 # Get latest date167 valid_dates = [d for d in group['dates'] if d != "Unknown"]168 latest_date = max(valid_dates) if valid_dates else "Unknown"169 170 # Get max Context Window171 max_context = max(group['contexts']) if group['contexts'] else 0172 173 # Format truncated length174 if max_context >= 1000000:175 context_str = f"{max_context/1000000:.0f}M" if max_context % 1000000 == 0 else f"{max_context/1000000:.1f}M"176 elif max_context >= 1000:177 context_str = f"{max_context/1000:.0f}k" if max_context % 1000 == 0 else f"{max_context/1000:.1f}k"178 else:179 context_str = str(max_context)180 181 # Get model type and context length182 model_context = "-"183 model_url = ""184 if model_name in MODLE_INFO_DICT:185 model_info = MODLE_INFO_DICT[model_name]186 if isinstance(model_info, dict):187 model_type = model_info.get("type", "Unknown")188 model_context = model_info.get("context_length", "-")189 model_url = model_info.get("url", "")190 else:191 model_type = str(model_info)192 else:193 model_type = "Unknown"194 195 # Handle model name link and icon196 display_model_name = model_name197 198 if model_url:199 display_model_name = f"[{display_model_name}]({model_url})"200 201 # Calculate average score202 nt_score_val = 0203 nt_score_str = "-"204 if group['non_thinking_scores']:205 nt_score_val = sum(group['non_thinking_scores']) / len(group['non_thinking_scores'])206 nt_score_str = f"{nt_score_val * 100:.2f}"207 208 t_score_val = 0209 t_score_str = "-"210 if group['thinking_scores']:211 t_score_val = sum(group['thinking_scores']) / len(group['thinking_scores'])212 t_score_str = f"{t_score_val * 100:.2f}"213 214 leaderboard_data.append({215 'Model Name': display_model_name,216 'Model Type': model_type,217 'Context Length': model_context,218 'Truncated Length': context_str,219 'Non-Thinking Score': nt_score_str,220 'Thinking Score': t_score_str,221 '_sort_score': max(nt_score_val, t_score_val)222 })223 224 df = pd.DataFrame(leaderboard_data)225 # Sort by highest score descending226 if not df.empty:227 df = df.sort_values('_sort_score', ascending=False).drop(columns=['_sort_score']).reset_index(drop=True)228 229 return df230 231def get_display_name_for_result(result):232 """Get display name for model (append suffix based on thinking/nonthinking)"""233 if result.get('has_nonthinking'):234 return f"{result['model_name']}_nonthinking"235 elif result.get('has_thinking'):236 return f"{result['model_name']}_thinking"237 else:238 return result['model_name']239 240def get_model_color_index(model_name, all_models):241 """Get model index in color list"""242 try:243 return all_models.index(model_name)244 except ValueError:245 return 0246 247def create_contextual_requirement_chart(results, selected_models):248 """Create contextual requirement comparison bar chart"""249 if not selected_models:250 return go.Figure()251 252 # Collect data253 chart_data = {}254 255 for result in results:256 display_name = get_display_name_for_result(result)257 if display_name in selected_models:258 model_name = display_name259 contextual_requirement = result['contextual_requirement']260 261 # Store each model's result directly262 if model_name not in chart_data:263 chart_data[model_name] = {}264 265 for req_type, score in contextual_requirement.items():266 chart_data[model_name][req_type] = score * 100 # multiply by 100267 268 # Create chart269 fig = go.Figure()270 271 # Get all requirement types272 all_req_types = []273 for result in results:274 display_name = get_display_name_for_result(result)275 if display_name in selected_models:276 contextual_requirement = result['contextual_requirement']277 for req_type in contextual_requirement.keys():278 if req_type not in all_req_types:279 all_req_types.append(req_type)280 281 for model_name in selected_models:282 if model_name in chart_data:283 scores = [chart_data[model_name].get(req_type, 0) for req_type in all_req_types]284 color_index = get_model_color_index(model_name, selected_models)285 286 fig.add_trace(go.Bar(287 name=model_name,288 x=all_req_types,289 y=scores,290 marker_color=get_color(color_index),291 text=[f"{score:.2f}" for score in scores], # keep 2 decimal places292 textposition='auto'293 ))294 295 fig.update_layout(296 title='Performance Comparison on Different Context Requirements',297 xaxis_title='Context Requirement Type',298 yaxis_title='Average Score',299 barmode='group',300 autosize=True, # auto size301 legend=dict(302 orientation="h",303 yanchor="top",304 y=-0.25, # adjust lower305 xanchor="center",306 x=0.5307 ),308 margin=dict(b=100) # increase bottom margin309 )310 311 return fig312 313def create_primary_task_radar_chart(results, selected_models):314 """Create primary task radar chart (aggregate by prefix)"""315 if not selected_models:316 return go.Figure()317 318 # Collect all model task prefixes319 prefix_order = []320 # Map prefix -> [scores] for each model321 model_prefix_scores = {}322 323 for result in results:324 display_name = get_display_name_for_result(result)325 if display_name not in selected_models:326 continue327 primary_task = result.get('primary_task', {})328 if display_name not in model_prefix_scores:329 model_prefix_scores[display_name] = {}330 for task_key, score in primary_task.items():331 prefix = task_key.split('.')[0].strip() if isinstance(task_key, str) else str(task_key)332 if prefix not in prefix_order:333 prefix_order.append(prefix)334 if prefix not in model_prefix_scores[display_name]:335 model_prefix_scores[display_name][prefix] = []336 model_prefix_scores[display_name][prefix].append(score * 100)337 338 # Take first 11 prefixes339 categories = prefix_order[:11]340 341 # Create radar chart342 fig = go.Figure()343 344 for model_name in selected_models:345 if model_name not in model_prefix_scores:346 continue347 # Mean aggregation for each prefix348 values = []349 for prefix in categories:350 scores = model_prefix_scores[model_name].get(prefix, [])351 if scores:352 values.append(sum(scores) / len(scores))353 else:354 values.append(0)355 # Close polygon356 r_values = values + ([values[0]] if values else [])357 theta_values = categories + ([categories[0]] if categories else [])358 color_index = get_model_color_index(model_name, selected_models)359 fig.add_trace(go.Scatterpolar(360 r=r_values,361 theta=theta_values,362 mode='lines+markers',363 name=model_name,364 line=dict(color=get_color(color_index), width=3),365 marker=dict(size=6),366 fill='toself'367 ))368 369 fig.update_layout(370 title='Performance Comparison on Different Primary Tasks',371 polar=dict(372 radialaxis=dict(visible=True, range=[0, 100])373 ),374 legend=dict(375 orientation="h",376 yanchor="top",377 y=-0.2,378 xanchor="center",379 x=0.5380 ),381 margin=dict(b=100)382 )383 384 return fig385 386def create_language_chart(results, selected_models):387 """Create language comparison bar chart"""388 if not selected_models:389 return go.Figure()390 391 # Collect data392 chart_data = {}393 394 for result in results:395 display_name = get_display_name_for_result(result)396 if display_name in selected_models:397 model_name = display_name398 language = result['language']399 400 # Store each model's result directly401 if model_name not in chart_data:402 chart_data[model_name] = {}403 404 for lang_type, score in language.items():405 chart_data[model_name][lang_type] = score * 100 # multiply by 100406 407 # Create chart408 fig = go.Figure()409 410 # Get all language types411 all_lang_types = []412 for result in results:413 display_name = get_display_name_for_result(result)414 if display_name in selected_models:415 language = result['language']416 for lang_type in language.keys():417 if lang_type not in all_lang_types:418 all_lang_types.append(lang_type)419 420 for model_name in selected_models:421 if model_name in chart_data:422 scores = [chart_data[model_name].get(lang_type, 0) for lang_type in all_lang_types]423 color_index = get_model_color_index(model_name, selected_models)424 425 fig.add_trace(go.Bar(426 name=model_name,427 x=all_lang_types,428 y=scores,429 marker_color=get_color(color_index),430 text=[f"{score:.2f}" for score in scores], # keep 2 decimal places431 textposition='auto'432 ))433 434 fig.update_layout(435 title='Performance Comparison on Different Languages',436 xaxis_title='Language Type',437 yaxis_title='Average Score',438 barmode='group',439 autosize=True, # auto size440 legend=dict(441 orientation="h",442 yanchor="top",443 y=-0.25, # adjust lower444 xanchor="center",445 x=0.5446 ),447 margin=dict(b=100) # increase bottom margin448 )449 450 return fig451 452def create_difficulty_chart(results, selected_models):453 """Create difficulty comparison bar chart"""454 if not selected_models:455 return go.Figure()456 457 # Collect data458 chart_data = {}459 460 for result in results:461 display_name = get_display_name_for_result(result)462 if display_name in selected_models:463 model_name = display_name464 difficulty = result['difficulty']465 466 # Store each model's result directly467 if model_name not in chart_data:468 chart_data[model_name] = {}469 470 for diff_type, score in difficulty.items():471 chart_data[model_name][diff_type] = score * 100 # multiply by 100472 473 # Create chart474 fig = go.Figure()475 476 # Get all difficulty types477 all_diff_types = []478 for result in results:479 display_name = get_display_name_for_result(result)480 if display_name in selected_models:481 difficulty = result['difficulty']482 for diff_type in difficulty.keys():483 if diff_type not in all_diff_types:484 all_diff_types.append(diff_type)485 486 for model_name in selected_models:487 if model_name in chart_data:488 scores = [chart_data[model_name].get(diff_type, 0) for diff_type in all_diff_types]489 color_index = get_model_color_index(model_name, selected_models)490 491 fig.add_trace(go.Bar(492 name=model_name,493 x=all_diff_types,494 y=scores,495 marker_color=get_color(color_index),496 text=[f"{score:.2f}" for score in scores], # keep 2 decimal places497 textposition='auto'498 ))499 500 fig.update_layout(501 title='Performance Comparison on Different Difficulties',502 xaxis_title='Difficulty Type',503 yaxis_title='Average Score',504 barmode='group',505 autosize=True, # auto size506 legend=dict(507 orientation="h",508 yanchor="top",509 y=-0.25, # adjust lower510 xanchor="center",511 x=0.5512 ),513 margin=dict(b=100) # increase bottom margin514 )515 516 return fig517 518def create_length_heatmap(results, selected_models):519 """Create length heatmap"""520 if not selected_models:521 return go.Figure()522 523 # Standard context lengths524 standard_lengths = [8000, 16000, 32000, 64000, 128000, 256000]525 standard_length_keys = ['8k', '16k', '32k', '64k', '128k', '256k']526 527 # Map results by name528 result_map = {get_display_name_for_result(r): r for r in results}529 530 # Prepare heatmap data531 heatmap_data = []532 model_names = []533 534 for model_name in selected_models:535 if model_name in result_map:536 model_names.append(model_name)537 result = result_map[model_name]538 539 # Get data from token_length_metrics540 token_length_metrics = result.get('token_length_metrics', {})541 row_data = []542 543 for key in standard_length_keys:544 if key in token_length_metrics:545 row_data.append(token_length_metrics[key] * 100) # multiply by 100546 else:547 row_data.append(None) # No data point548 549 heatmap_data.append(row_data)550 551 # Create heatmap552 fig = go.Figure(data=go.Heatmap(553 z=heatmap_data,554 x=[f"{length//1000}k" for length in standard_lengths], # x axis labels555 y=model_names, # y axis labels556 colorscale='RdYlBu_r', # Red is low, Blue is high557 showscale=True,558 text=[[f"{val:.2f}" if val is not None else "N/A" for val in row] for row in heatmap_data], # show values559 texttemplate="%{text}",560 textfont={"size": 10},561 hoverongaps=False562 ))563 564 fig.update_layout(565 title='Performance Heatmap on Different Sample Lengths',566 xaxis_title='Sample Length (tokens)',567 yaxis_title='Model Name',568 autosize=True,569 height=max(400, len(model_names) * 50), # adjust height based on model count570 margin=dict(l=150, r=50, t=80, b=80) # adjust margins571 )572 573 return fig574 575def create_bon_chart(results, selected_models):576 """Create BoN 1-3 line chart"""577 if not selected_models:578 return go.Figure()579 580 # BoN labels581 bon_labels = ['BoN-1', 'BoN-2', 'BoN-3']582 bon_indices = [1, 2, 3]583 584 # Prepare data for each model585 model_data = {}586 for result in results:587 display_name = get_display_name_for_result(result)588 if display_name in selected_models:589 if display_name not in model_data:590 model_data[display_name] = {}591 592 # Get data from bon_data593 bon_data = result.get('bon_data', {})594 for bon_key in bon_labels:595 if bon_key in bon_data:596 bon_index = bon_labels.index(bon_key) + 1597 model_data[display_name][bon_index] = bon_data[bon_key] * 100 # multiply by 100598 599 # Create chart600 fig = go.Figure()601 602 for model_name in selected_models:603 if model_name not in model_data:604 continue605 606 data = model_data[model_name]607 if not data:608 continue609 610 # Prepare data for each BoN611 x_values = []612 y_values = []613 text_values = []614 615 for bon_index in bon_indices:616 x_values.append(bon_index)617 if bon_index in data:618 y_values.append(data[bon_index])619 text_values.append(f"{data[bon_index]:.2f}")620 else:621 y_values.append(None)622 text_values.append("")623 624 # Get model color index625 color_index = get_model_color_index(model_name, selected_models)626 627 fig.add_trace(go.Scatter(628 x=x_values,629 y=y_values,630 mode='lines+markers',631 name=model_name,632 line=dict(color=get_color(color_index), width=3),633 marker=dict(size=10),634 text=text_values,635 textposition='top center',636 connectgaps=False637 ))638 639 # Set x axis640 fig.update_layout(641 title='Performance Comparison on Different Best-of-N',642 xaxis_title='N',643 yaxis_title='Average Score',644 autosize=True,645 xaxis=dict(646 tickmode='array',647 tickvals=bon_indices,648 ticktext=bon_labels,649 tickangle=0650 ),651 legend=dict(652 orientation="h",653 yanchor="top",654 y=-0.25,655 xanchor="center",656 x=0.5657 ),658 margin=dict(b=100)659 )660 661 return fig662 663def create_pass_k_chart(results, selected_models):664 """Create Pass@N line chart"""665 if not selected_models:666 return go.Figure()667 668 # Pass@K labels669 k_labels = ['Pass@1', 'Pass@2', 'Pass@3']670 k_indices = [1, 2, 3]671 672 # Prepare data for each model673 model_data = {}674 for result in results:675 display_name = get_display_name_for_result(result)676 if display_name in selected_models:677 if display_name not in model_data:678 model_data[display_name] = {}679 680 # Get data from pass_at_k681 pass_data = result.get('pass_at_k', {})682 for i, k_key in enumerate(k_labels):683 val = pass_data.get(k_key)684 if val is not None:685 k_index = k_indices[i]686 model_data[display_name][k_index] = val * 100 # multiply by 100687 688 # Create chart689 fig = go.Figure()690 691 for model_name in selected_models:692 if model_name not in model_data:693 continue694 695 data = model_data[model_name]696 if not data:697 continue698 699 # Prepare data for each Pass@K700 x_values = []701 y_values = []702 text_values = []703 704 for k_index in k_indices:705 x_values.append(k_index)706 if k_index in data:707 y_values.append(data[k_index])708 text_values.append(f"{data[k_index]:.2f}")709 else:710 y_values.append(None)711 text_values.append("")712 713 # Get model color index714 color_index = get_model_color_index(model_name, selected_models)715 716 fig.add_trace(go.Scatter(717 x=x_values,718 y=y_values,719 mode='lines+markers',720 name=model_name,721 line=dict(color=get_color(color_index), width=3),722 marker=dict(size=10),723 text=text_values,724 textposition='top center',725 connectgaps=False726 ))727 728 # Set x axis729 fig.update_layout(730 title='Performance Comparison on Different Pass@N',731 xaxis_title='N',732 yaxis_title='Pass@N (%)',733 autosize=True,734 xaxis=dict(735 tickmode='array',736 tickvals=k_indices,737 ticktext=k_labels,738 tickangle=0739 ),740 legend=dict(741 orientation="h",742 yanchor="top",743 y=-0.25,744 xanchor="center",745 x=0.5746 ),747 margin=dict(b=100)748 )749 750 return fig751 752def create_gradio_interface(parser: ResultParser):753 """Create Gradio interface"""754 755 def refresh_data():756 """Refresh data"""757 parser.scan_all_results()758 return parser.get_leaderboard_data()759 760 def get_model_choices():761 """Get model choices (distinguish by suffix for thinking/nonthinking)"""762 if not parser.results:763 return []764 display_names = set()765 for r in parser.results:766 name = get_display_name_for_result(r)767 display_names.add(name)768 models = sorted(list(display_names))769 return models770 771 def update_charts(selected_models):772 """Update all charts"""773 if not selected_models:774 return None, None, None, None, None, None, None775 776 length_heatmap = create_length_heatmap(parser.results, selected_models)777 contextual_chart = create_contextual_requirement_chart(parser.results, selected_models)778 primary_task_radar_chart = create_primary_task_radar_chart(parser.results, selected_models)779 language_chart = create_language_chart(parser.results, selected_models)780 difficulty_chart = create_difficulty_chart(parser.results, selected_models)781 bon_chart = create_bon_chart(parser.results, selected_models)782 pass_k_chart = create_pass_k_chart(parser.results, selected_models)783 784 return length_heatmap, contextual_chart, primary_task_radar_chart, language_chart, difficulty_chart, bon_chart, pass_k_chart785 786 # Create interface787 with gr.Blocks(title="LongBench Pro Results Visualization", theme=gr.themes.Soft(), css=CUSTOM_CSS) as demo:788 gr.Markdown("# LongBench Pro Results Visualization")789 790 gr.HTML("""791 <div style="text-align: center; display: flex; justify-content: center; gap: 10px; margin-bottom: 20px;">792 <a href="https://huggingface.co/datasets/caskcsg/LongBench-Pro" target="_blank"><img src="https://img.shields.io/badge/Dataset-yellow?logo=huggingface&logoColor=yellow&labelColor=white" alt="Dataset"></a>793 <a href="https://github.com/caskcsg/longcontext/tree/main/LongBench-Pro" target="_blank"><img src="https://img.shields.io/badge/Code-181717?logo=github&logoColor=181717&labelColor=white" alt="Code"></a>794 <a href="https://arxiv.org/abs/2601.02872" target="_blank"><img src="https://img.shields.io/badge/Paper-red?logo=arxiv&logoColor=B31B1B&labelColor=white" alt="Paper"></a>795 <a href="https://huggingface.co/spaces/caskcsg/LongBench-Pro-Leaderboard" target="_blank"><img src="https://img.shields.io/badge/🏆-Leaderboard-blue?labelColor=white" alt="Leaderboard"></a>796 </div>797 """)798 799 # Leaderboard area800 gr.Markdown("## 🏆 Overall Performance Leaderboard")801 gr.Markdown("""802 - *Thinking scores for Thinking and Mixed-Thinking models use their own thinking capabilities (Non-Thinking Prompt)*803 - *Thinking scores for Instruct models are obtained using thinking prompts (Thinking Prompt)*804 """)805 leaderboard_df = gr.Dataframe(806 headers=["Model Name", "Model Type", "Context Length", "Truncation Length", "Non-Thinking Score", "Thinking Score"],807 datatype=["markdown", "str", "str", "str", "str", "str"],808 interactive=False,809 wrap=True,810 show_row_numbers=True,811 show_search="filter",812 max_height=800,813 column_widths=["250px", "100px", "100px", "100px", "120px", "120px"],814 elem_id="leaderboard_table"815 )816 817 # Model selection and chart area818 gr.HTML("<br>")819 gr.Markdown("## 📊 Specific Dimension Comparison")820 with gr.Row():821 with gr.Column(scale=4):822 model_selector = gr.Dropdown(823 choices=[],824 label="Select Models",825 value=[],826 multiselect=True,827 interactive=True828 )829 with gr.Column(scale=1):830 update_charts_btn = gr.Button("Update Charts", variant="primary", size="lg")831 832 with gr.Tabs():833 with gr.TabItem("Language"):834 language_plot = gr.Plot(show_label=False)835 836 with gr.TabItem("Difficulty"):837 difficulty_plot = gr.Plot(show_label=False)838 839 with gr.TabItem("Sample Length"):840 length_heatmap = gr.Plot(show_label=False)841 842 with gr.TabItem("Primary Task"):843 primary_task_radar_plot = gr.Plot(show_label=False)844 845 with gr.TabItem("Context Requirement"):846 contextual_plot = gr.Plot(show_label=False)847 848 with gr.TabItem("Best-of-N"):849 bon_plot = gr.Plot(show_label=False)850 851 with gr.TabItem("Pass@N"):852 pass_k_plot = gr.Plot(show_label=False)853 854 # Add bottom spacer855 gr.HTML("<div style='height: 100px;'></div>")856 857 # Event handling858 def update_model_choices():859 models = get_model_choices()860 return gr.Dropdown(choices=models, value=[])861 862 update_charts_btn.click(863 fn=update_charts,864 inputs=[model_selector],865 outputs=[length_heatmap, contextual_plot, primary_task_radar_plot, language_plot, difficulty_plot, bon_plot, pass_k_plot]866 )867 868 # Initialize869 demo.load(870 fn=refresh_data,871 outputs=[leaderboard_df]872 ).then(873 fn=update_model_choices,874 outputs=[model_selector]875 )876 877 return demo878 879def main():880 """Main function"""881 output_dir = "./output"882 883 print("Initializing result parser...")884 parser = ResultParser(output_dir)885 886 print("Scanning result files...")887 parser.scan_all_results()888 889 print("Creating Gradio interface...")890 demo = create_gradio_interface(parser)891 892 print("Starting server...")893 demo.launch()894 895if __name__ == "__main__":896 main()897