OpenHands/openhands-index
19
1"""2Simple data loader for OpenHands Index leaderboard.3Loads JSONL files from local directory or GitHub repository.4Uses pydantic models from openhands-index-results for validation.5"""6import os7import sys8import logging9import pandas as pd10import json11from pathlib import Path12from typing import Optional13 14logger = logging.getLogger(__name__)15 16# Pydantic models will be imported after setup_data adds them to path17_schema_models_loaded = False18Metadata = None19ScoreEntry = None20 21 22def _ensure_schema_models():23 """Lazily import pydantic schema models from openhands-index-results."""24 global _schema_models_loaded, Metadata, ScoreEntry25 26 if _schema_models_loaded:27 return _schema_models_loaded28 29 try:30 # Try importing from the cloned repo's scripts directory31 from validate_schema import Metadata as _Metadata, ScoreEntry as _ScoreEntry32 Metadata = _Metadata33 ScoreEntry = _ScoreEntry34 _schema_models_loaded = True35 logger.info("Successfully loaded pydantic schema models from openhands-index-results")36 except ImportError as e:37 logger.warning(f"Could not import pydantic schema models: {e}")38 logger.warning("Data will be loaded without schema validation")39 _schema_models_loaded = False40 41 return _schema_models_loaded42 43 44def load_and_validate_agent_data(agent_dir: Path) -> tuple[Optional[dict], Optional[list], list[str]]:45 """46 Load and validate agent data using pydantic models if available.47 48 Returns:49 Tuple of (metadata_dict, scores_list, validation_errors)50 """51 errors = []52 metadata_file = agent_dir / "metadata.json"53 scores_file = agent_dir / "scores.json"54 55 if not metadata_file.exists() or not scores_file.exists():56 return None, None, [f"Missing metadata.json or scores.json in {agent_dir}"]57 58 # Load raw JSON59 with open(metadata_file) as f:60 metadata_raw = json.load(f)61 with open(scores_file) as f:62 scores_raw = json.load(f)63 64 # Validate with pydantic if available65 if _ensure_schema_models() and Metadata and ScoreEntry:66 try:67 validated_metadata = Metadata(**metadata_raw)68 # Use mode='json' to serialize enums as strings69 metadata_dict = validated_metadata.model_dump(mode='json')70 except Exception as e:71 errors.append(f"Metadata validation error in {agent_dir.name}: {e}")72 metadata_dict = metadata_raw # Fall back to raw data73 74 validated_scores = []75 for i, score in enumerate(scores_raw):76 try:77 validated_score = ScoreEntry(**score)78 # Use mode='json' to serialize enums as strings79 validated_dict = validated_score.model_dump(mode='json')80 # Preserve any extra fields from raw data (like full_archive)81 for key, value in score.items():82 if key not in validated_dict:83 validated_dict[key] = value84 validated_scores.append(validated_dict)85 except Exception as e:86 errors.append(f"Score entry {i} validation error in {agent_dir.name}: {e}")87 validated_scores.append(score) # Fall back to raw data88 scores_list = validated_scores89 else:90 # No validation, use raw data91 metadata_dict = metadata_raw92 scores_list = scores_raw93 94 return metadata_dict, scores_list, errors95 96 97class SimpleLeaderboardViewer:98 """Simple replacement for agent-eval's LeaderboardViewer."""99 100 AGENT_FILTER_OPENHANDS = "openhands"101 AGENT_FILTER_ALTERNATIVE = "alternative"102 103 def __init__(104 self,105 data_dir: str,106 config: str,107 split: str,108 agent_filter: str = AGENT_FILTER_OPENHANDS,109 ):110 """111 Args:112 data_dir: Path to data directory113 config: Config name (e.g., "1.0.0-dev1")114 split: Split name (e.g., "validation" or "test")115 agent_filter: Which submissions to include.116 ``"openhands"`` (default) loads only the default OpenHands117 agent runs from ``results/{model}/`` — the canonical118 leaderboard. ``"alternative"`` loads only third-party119 harnesses (Claude Code / Codex / Gemini CLI / OpenHands120 Sub-agents) from ``alternative_agents/{type}/{model}/``,121 which power the standalone Alternative Agents page.122 The two are kept on separate pages because their123 cost/runtime numbers aren't apples-to-apples and mixing124 them in one ranking would be misleading.125 """126 if agent_filter not in (self.AGENT_FILTER_OPENHANDS, self.AGENT_FILTER_ALTERNATIVE):127 raise ValueError(128 f"agent_filter must be one of "129 f"{{{self.AGENT_FILTER_OPENHANDS!r}, {self.AGENT_FILTER_ALTERNATIVE!r}}}, "130 f"got {agent_filter!r}"131 )132 self.data_dir = Path(data_dir)133 self.config = config134 self.split = split135 self.agent_filter = agent_filter136 self.config_path = self.data_dir / config137 138 # Benchmark to category mappings (single source of truth)139 self.benchmark_to_categories = {140 'swe-bench': ['Issue Resolution'],141 'swe-bench-multimodal': ['Frontend'],142 'commit0': ['Greenfield'],143 'swt-bench': ['Testing'],144 'gaia': ['Information Gathering'],145 }146 147 # Build tag map (category -> benchmarks)148 self.tag_map = {}149 for benchmark, categories in self.benchmark_to_categories.items():150 for category in categories:151 if category not in self.tag_map:152 self.tag_map[category] = []153 if benchmark not in self.tag_map[category]:154 self.tag_map[category].append(benchmark)155 156 # Default agent_name when metadata.json doesn't carry one. Matches the157 # default-agent value used by push_to_index_from_archive.py so legacy158 # entries (which omit the field) still group cleanly with new entries.159 DEFAULT_AGENT_NAME = "OpenHands"160 161 def _records_from_agent_dir(self, agent_dir: Path, default_agent_name: str | None = None) -> tuple[list[dict], list[str]]:162 """Build per-benchmark records from a single agent directory.163 164 Shared by ``_load_from_agent_dirs`` (default OpenHands results) and165 ``_load_from_alternative_agents_dirs`` (acp-claude / acp-codex / etc.).166 Returns ``(records, validation_errors)``. Returns an empty list of167 records when the directory has no scores or is hidden from the168 leaderboard.169 """170 records: list[dict] = []171 metadata, scores, errors = load_and_validate_agent_data(agent_dir)172 173 if metadata is None or scores is None:174 return records, errors175 176 if metadata.get('hide_from_leaderboard', False):177 logger.info(f"Skipping {agent_dir.name}: hide_from_leaderboard is True")178 return records, errors179 180 # Resolve the agent display name. Prefer the value stamped into181 # metadata.json by push-to-index; fall back to the directory's182 # default (e.g. "Claude Code" for acp-claude/) and finally to183 # "OpenHands" for legacy results/ entries that predate the field.184 agent_name = (185 metadata.get('agent_name')186 or default_agent_name187 or self.DEFAULT_AGENT_NAME188 )189 190 for score_entry in scores:191 record = {192 'agent_name': agent_name,193 'agent_version': metadata.get('agent_version', 'Unknown'),194 'llm_base': metadata.get('model', 'unknown'),195 'openness': metadata.get('openness', 'unknown'),196 'submission_time': score_entry.get('submission_time', metadata.get('submission_time', '')),197 'release_date': metadata.get('release_date', ''),198 'parameter_count_b': metadata.get('parameter_count_b'),199 'active_parameter_count_b': metadata.get('active_parameter_count_b'),200 'score': score_entry.get('score'),201 'metric': score_entry.get('metric', 'unknown'),202 'cost_per_instance': score_entry.get('cost_per_instance'),203 'average_runtime': score_entry.get('average_runtime'),204 'tags': [score_entry.get('benchmark')],205 'full_archive': score_entry.get('full_archive', ''),206 'eval_visualization_page': score_entry.get('eval_visualization_page', ''),207 }208 records.append(record)209 return records, errors210 211 def _load_from_agent_dirs(self):212 """Load agent records based on ``self.agent_filter``.213 214 - ``"openhands"`` (default): only ``{config}/results/{model}/``,215 which is the canonical OpenHands leaderboard. The Home page and216 the per-category subpages use this.217 - ``"alternative"``: only218 ``{config}/alternative_agents/{type}/{model}/`` (acp-claude,219 acp-codex, acp-gemini, openhands_subagents, ...). The dedicated220 Alternative Agents page uses this.221 222 Returns ``None`` if no records were found (which makes the caller223 render an empty-state placeholder).224 """225 all_records = []226 all_validation_errors = []227 228 if self.agent_filter == self.AGENT_FILTER_OPENHANDS:229 # Default OpenHands agent results230 results_dir = self.config_path / "results"231 if results_dir.exists():232 for agent_dir in results_dir.iterdir():233 if not agent_dir.is_dir():234 continue235 records, errors = self._records_from_agent_dir(agent_dir)236 all_records.extend(records)237 all_validation_errors.extend(errors)238 else:239 # Alternative agents (one subdirectory per agent_type, then per model)240 # Default agent_name per agent_type matches the AGENT_NAME_BY_TYPE241 # map in OpenHands/evaluation push_to_index_from_archive.py — keeping242 # it in sync ensures rows are labelled the same way the index repo243 # records them.244 agent_type_default_name = {245 'acp-claude': 'Claude Code',246 'acp-codex': 'Codex',247 'acp-gemini': 'Gemini CLI',248 }249 alt_dir = self.config_path / "alternative_agents"250 if alt_dir.exists():251 for type_dir in alt_dir.iterdir():252 if not type_dir.is_dir():253 continue254 default_name = agent_type_default_name.get(type_dir.name)255 if default_name is None:256 continue # skip unlisted agent types (e.g. openhands_subagents)257 for agent_dir in type_dir.iterdir():258 if not agent_dir.is_dir():259 continue260 records, errors = self._records_from_agent_dir(261 agent_dir, default_agent_name=default_name262 )263 all_records.extend(records)264 all_validation_errors.extend(errors)265 266 # Log validation errors if any267 if all_validation_errors:268 logger.warning(f"Schema validation errors ({len(all_validation_errors)} total):")269 for error in all_validation_errors[:5]: # Show first 5270 logger.warning(f" - {error}")271 if len(all_validation_errors) > 5:272 logger.warning(f" ... and {len(all_validation_errors) - 5} more")273 274 if not all_records:275 return None # Caller will render empty-state placeholder276 277 return pd.DataFrame(all_records)278 279 def _load(self):280 """Load data from agent-centric directories and return DataFrame and tag map."""281 df = self._load_from_agent_dirs()282 283 if df is None:284 # Return empty dataframe with error message285 return pd.DataFrame({286 "Message": [f"No data found for split '{self.split}' in results directory"]287 }), {}288 289 # Process the dataframe290 try:291 292 # Transform to expected format for leaderboard293 # Group by agent (version + model combination) to aggregate results across datasets294 transformed_records = []295 296 # Create a unique identifier per (agent_name, agent_version, model)297 # tuple. Including agent_name keeps an OpenHands run and a Claude298 # Code run on the same SDK version + model from collapsing into299 # one row when both submit to the leaderboard.300 df['agent_name'] = df['agent_name'].fillna(self.DEFAULT_AGENT_NAME)301 df['agent_id'] = (302 df['agent_name'].astype(str)303 + '_' + df['agent_version'].astype(str)304 + '_' + df['llm_base'].astype(str)305 )306 307 for agent_id in df['agent_id'].unique():308 agent_records = df[df['agent_id'] == agent_id]309 310 # Build a single record for this agent311 first_record = agent_records.iloc[0]312 agent_version = first_record['agent_version']313 agent_name = first_record['agent_name']314 315 # Normalize openness to "open" or "closed"316 from aliases import OPENNESS_MAPPING317 raw_openness = first_record['openness']318 normalized_openness = OPENNESS_MAPPING.get(raw_openness, raw_openness)319 320 # All 5 categories for the leaderboard321 ALL_CATEGORIES = ['Issue Resolution', 'Frontend', 'Greenfield', 'Testing', 'Information Gathering']322 323 record = {324 # Core agent info - use final display names325 'agent_name': agent_name, # Will become "Agent"326 'SDK version': agent_version, # Will become "SDK Version"327 'Language model': first_record['llm_base'], # Will become "Language Model"328 'openness': normalized_openness, # Will become "Openness" (simplified to "open" or "closed")329 'date': first_record['submission_time'], # Will become "Date"330 # Model metadata for visualizations331 'release_date': first_record.get('release_date', ''), # Model release date332 'parameter_count_b': first_record.get('parameter_count_b'), # Total params in billions333 'active_parameter_count_b': first_record.get('active_parameter_count_b'), # Active params for MoE334 # Additional columns expected by the transformer335 # Use agent_id (name_version_model) as unique identifier for Pareto frontier calculation336 'id': agent_id,337 'source': first_record.get('source', ''), # Will become "Source"338 'logs': first_record.get('logs', ''), # Will become "Logs"339 'visualization': '', # Will become "Visualization" - populated below340 }341 342 # Add per-dataset scores and costs343 dataset_scores = []344 dataset_costs = []345 346 # Track category-level data for aggregation347 category_data = {} # {category: {'scores': [...], 'costs': [], 'runtimes': []}}348 349 for _, row in agent_records.iterrows():350 tags = row['tags'] if isinstance(row['tags'], list) else [row['tags']]351 for tag in tags:352 # Add columns for this specific dataset/benchmark353 record[f'{tag} score'] = row['score']354 record[f'{tag} cost'] = row['cost_per_instance']355 record[f'{tag} runtime'] = row.get('average_runtime')356 dataset_scores.append(row['score'])357 dataset_costs.append(row['cost_per_instance'])358 359 # Store the full_archive URL for this benchmark (for benchmark-specific download)360 full_archive_url = row.get('full_archive', '') if hasattr(row, 'get') else row['full_archive'] if 'full_archive' in row.index else ''361 if full_archive_url:362 record[f'{tag} download'] = full_archive_url363 364 # Store the eval_visualization_page URL for this benchmark (for Laminar visualization)365 viz_url = row.get('eval_visualization_page', '') if hasattr(row, 'get') else row['eval_visualization_page'] if 'eval_visualization_page' in row.index else ''366 if viz_url:367 record[f'{tag} visualization'] = viz_url368 369 # Track category-level data for aggregation370 if tag in self.benchmark_to_categories:371 for category in self.benchmark_to_categories[tag]:372 if category not in category_data:373 category_data[category] = {'scores': [], 'costs': [], 'runtimes': []}374 category_data[category]['scores'].append(row['score'])375 category_data[category]['costs'].append(row['cost_per_instance'])376 category_data[category]['runtimes'].append(row.get('average_runtime'))377 378 # Calculate category-level aggregates and track average cost/runtime379 all_costs = []380 all_runtimes = []381 categories_with_scores = 0382 for category in ALL_CATEGORIES:383 if category in category_data and category_data[category]['scores']:384 data = category_data[category]385 avg_score = sum(data['scores']) / len(data['scores'])386 record[f'{category} score'] = avg_score387 categories_with_scores += 1388 if data['costs']:389 valid_costs = [c for c in data['costs'] if c is not None]390 if valid_costs:391 avg_cost = sum(valid_costs) / len(valid_costs)392 record[f'{category} cost'] = avg_cost393 all_costs.extend(valid_costs)394 if data['runtimes']:395 valid_runtimes = [r for r in data['runtimes'] if r is not None]396 if valid_runtimes:397 avg_runtime = sum(valid_runtimes) / len(valid_runtimes)398 record[f'{category} runtime'] = avg_runtime399 all_runtimes.extend(valid_runtimes)400 else:401 # Category not submitted - will show as NA402 pass403 404 # Calculate average score: always divide by 5 (treating missing categories as 0)405 # This penalizes incomplete submissions406 score_sum = sum(407 record.get(f'{cat} score', 0) or 0 408 for cat in ALL_CATEGORIES409 )410 record['average score'] = score_sum / 5411 412 # Average cost per instance across all benchmarks413 record['average cost'] = sum(all_costs) / len(all_costs) if all_costs else None414 415 # Average runtime per instance across all benchmarks416 record['average runtime'] = sum(all_runtimes) / len(all_runtimes) if all_runtimes else None417 418 # Track how many categories were completed419 record['categories_completed'] = categories_with_scores420 421 transformed_records.append(record)422 423 transformed_df = pd.DataFrame(transformed_records)424 425 # Build tag map if not already built426 if not self.tag_map:427 # Create simple tag map from the data428 all_tags = set()429 for _, row in df.iterrows():430 tags = row['tags'] if isinstance(row['tags'], list) else [row['tags']]431 all_tags.update(tags)432 433 # Simple mapping: each tag maps to itself434 self.tag_map = {tag: [tag] for tag in sorted(all_tags)}435 436 # DEBUG: Print sample of loaded data437 print(f"[DATA_LOADER] Loaded {len(transformed_df)} agents")438 if len(transformed_df) > 0:439 sample_cols = ['agent_name', 'overall_score', 'overall_cost']440 available_cols = [c for c in sample_cols if c in transformed_df.columns]441 print(f"[DATA_LOADER] Sample row: {transformed_df[available_cols].iloc[0].to_dict()}")442 443 return transformed_df, self.tag_map444 except Exception as e:445 import traceback446 traceback.print_exc()447 return pd.DataFrame({448 "Message": [f"Error loading data: {e}"]449 }), {}450 451 def get_dataframe(self):452 """Get the raw dataframe."""453 df, _ = self._load()454 return df455 456 457def load_mock_data_locally(data_dir: str = "mock_results"):458 """459 Load mock data from local directory for testing.460 461 Args:462 data_dir: Path to mock results directory463 464 Returns:465 Dictionary mapping split names to SimpleLeaderboardViewer instances466 """467 viewers = {}468 data_path = Path(data_dir)469 470 if not data_path.exists():471 print(f"Warning: Mock data directory '{data_dir}' not found")472 return viewers473 474 # Find all config directories475 for config_dir in data_path.iterdir():476 if config_dir.is_dir():477 config_name = config_dir.name478 479 # Find all JSONL files (each represents a split)480 for jsonl_file in config_dir.glob("*.jsonl"):481 split_name = jsonl_file.stem482 viewer = SimpleLeaderboardViewer(483 data_dir=str(data_path),484 config=config_name,485 split=split_name486 )487 viewers[split_name] = viewer488 489 return viewers490