OpenHands/openhands-index
19
1"""2REST API for OpenHands Index leaderboard data.3 4This module provides API endpoints that use the same data loading functions5as the Gradio UI, ensuring consistency between the web interface and API responses.6"""7 8import logging9import math10from datetime import datetime11from typing import Optional, Any12 13from fastapi import FastAPI, Query, HTTPException14from fastapi.middleware.cors import CORSMiddleware15from fastapi.responses import HTMLResponse16 17from simple_data_loader import SimpleLeaderboardViewer18from config import CONFIG_NAME, EXTRACTED_DATA_DIR19from setup_data import _last_fetch_time, CACHE_TTL_SECONDS20import os21 22 23def _sanitize_value(val: Any) -> Any:24 """Convert NaN/inf values to None for JSON serialization."""25 if val is None:26 return None27 if isinstance(val, float):28 if math.isnan(val) or math.isinf(val):29 return None30 return val31 32 33def _sanitize_dict(d: dict) -> dict:34 """Recursively sanitize a dictionary for JSON serialization."""35 result = {}36 for k, v in d.items():37 if isinstance(v, dict):38 result[k] = _sanitize_dict(v)39 elif isinstance(v, list):40 result[k] = [_sanitize_dict(i) if isinstance(i, dict) else _sanitize_value(i) for i in v]41 else:42 result[k] = _sanitize_value(v)43 return result44 45logger = logging.getLogger(__name__)46 47# Create FastAPI app for API endpoints48api_app = FastAPI(49 title="OpenHands Index API",50 description="""51REST API for accessing OpenHands Index benchmark results.52 53The OpenHands Index is a comprehensive benchmark for evaluating AI coding agents 54across real-world software engineering tasks. It assesses models across five categories:55 56- **Issue Resolution**: Fixing bugs (SWE-Bench)57- **Greenfield**: Building new applications (Commit0)58- **Frontend**: UI development (SWE-Bench Multimodal)59- **Testing**: Test generation (SWT-Bench)60- **Information Gathering**: Research tasks (GAIA)61 62This API provides the same data that powers the leaderboard UI.63 """,64 version="1.0.0",65 docs_url="/docs",66 redoc_url="/redoc",67)68 69api_app.add_middleware(70 CORSMiddleware,71 allow_origins=["*"],72 allow_credentials=True,73 allow_methods=["*"],74 allow_headers=["*"],75)76 77# Benchmark to category mappings (same as simple_data_loader.py)78BENCHMARK_TO_CATEGORIES = {79 'swe-bench': ['Issue Resolution'],80 'swe-bench-multimodal': ['Frontend'],81 'commit0': ['Greenfield'],82 'swt-bench': ['Testing'],83 'gaia': ['Information Gathering'],84}85 86ALL_CATEGORIES = ['Issue Resolution', 'Frontend', 'Greenfield', 'Testing', 'Information Gathering']87 88CATEGORY_DESCRIPTIONS = {89 "Issue Resolution": "Fixing bugs in real GitHub issues (SWE-Bench)",90 "Greenfield": "Building new applications from scratch (Commit0)",91 "Frontend": "UI development with visual context (SWE-Bench Multimodal)",92 "Testing": "Test generation and quality (SWT-Bench)",93 "Information Gathering": "Research and information retrieval (GAIA)",94}95 96# Openness mapping (same as aliases.py)97OPENNESS_MAPPING = {98 'open': 'open',99 'open_weights': 'open',100 'open_weights_open_data': 'open',101 'closed': 'closed',102 'closed_api_available': 'closed',103 'closed_api_unavailable': 'closed',104}105 106 107def _get_leaderboard_data() -> dict:108 """109 Load leaderboard data using the same SimpleLeaderboardViewer used by the UI.110 This ensures API responses match what's displayed in the Gradio interface.111 """112 try:113 data_dir = EXTRACTED_DATA_DIR if os.path.exists(EXTRACTED_DATA_DIR) else "mock_results"114 viewer = SimpleLeaderboardViewer(115 data_dir=data_dir,116 config=CONFIG_NAME,117 split="test"118 )119 120 raw_df, tag_map = viewer._load()121 122 if raw_df is None or raw_df.empty or "Message" in raw_df.columns:123 return {"entries": [], "error": "No data available"}124 125 entries = []126 for _, row in raw_df.iterrows():127 # Normalize openness128 raw_openness = row.get('openness', 'unknown')129 normalized_openness = OPENNESS_MAPPING.get(raw_openness, raw_openness)130 131 entry = {132 "id": row.get('id'),133 "language_model": row.get('Language model'),134 "sdk_version": row.get('SDK version'),135 "openness": normalized_openness,136 "average_score": row.get('average score'),137 "average_cost": row.get('average cost'),138 "average_runtime": row.get('average runtime'),139 "categories_completed": row.get('categories_completed', 0),140 "release_date": row.get('release_date'),141 "benchmarks": {},142 "categories": {},143 }144 145 # Add benchmark-level data146 for benchmark in BENCHMARK_TO_CATEGORIES.keys():147 score_col = f'{benchmark} score'148 cost_col = f'{benchmark} cost'149 runtime_col = f'{benchmark} runtime'150 download_col = f'{benchmark} download'151 viz_col = f'{benchmark} visualization'152 153 if score_col in row and row[score_col] is not None:154 entry["benchmarks"][benchmark] = {155 "score": row.get(score_col),156 "cost": row.get(cost_col),157 "runtime": row.get(runtime_col),158 "download_url": row.get(download_col),159 "visualization_url": row.get(viz_col),160 }161 162 # Add category-level data163 for category in ALL_CATEGORIES:164 score_col = f'{category} score'165 cost_col = f'{category} cost'166 runtime_col = f'{category} runtime'167 168 if score_col in row and row[score_col] is not None:169 entry["categories"][category] = {170 "score": row.get(score_col),171 "cost": row.get(cost_col),172 "runtime": row.get(runtime_col),173 }174 175 # Sanitize the entry to handle NaN values176 entries.append(_sanitize_dict(entry))177 178 # Sort by average score descending179 entries.sort(key=lambda x: x.get('average_score') or 0, reverse=True)180 181 return {182 "entries": entries,183 "total_count": len(entries),184 "fetched_at": _last_fetch_time.isoformat() if _last_fetch_time else None,185 }186 187 except Exception as e:188 logger.error(f"Error loading leaderboard data: {e}")189 return {"entries": [], "error": str(e)}190 191 192@api_app.get("/", tags=["Info"])193async def api_root():194 """API information and available endpoints."""195 return {196 "name": "OpenHands Index API",197 "version": "1.0.0",198 "description": "REST API for accessing OpenHands Index benchmark results",199 "leaderboard_ui": "/",200 "documentation": "/api/docs",201 "endpoints": {202 "/api/": "API information (this page)",203 "/api/health": "Health check endpoint",204 "/api/leaderboard": "Get the full leaderboard with scores and metadata",205 "/api/leaderboard/models": "List all language models in the leaderboard",206 "/api/leaderboard/model/{model_name}": "Get data for a specific model",207 "/api/categories": "List all benchmark categories",208 "/api/benchmarks": "List all benchmarks",209 "/api/docs": "Interactive Swagger UI documentation",210 }211 }212 213 214@api_app.get("/health", tags=["Health"])215async def health_check():216 """Check API health status and cache information."""217 cache_age = None218 if _last_fetch_time is not None:219 cache_age = (datetime.now() - _last_fetch_time).total_seconds()220 221 return {222 "status": "healthy",223 "version": "1.0.0",224 "cache_ttl_seconds": CACHE_TTL_SECONDS,225 "cache_age_seconds": cache_age,226 "last_fetch_time": _last_fetch_time.isoformat() if _last_fetch_time else None,227 }228 229 230@api_app.get("/leaderboard", tags=["Leaderboard"])231async def get_leaderboard(232 openness: Optional[str] = Query(None, description="Filter by openness (open/closed)"),233 min_categories: Optional[int] = Query(None, description="Minimum categories completed"),234 sort_by: str = Query("average_score", description="Sort field (average_score, average_cost, average_runtime)"),235 limit: Optional[int] = Query(None, description="Limit number of results"),236):237 """238 Get the full leaderboard with benchmark scores and metadata.239 240 Returns the same data displayed in the OpenHands Index UI leaderboard.241 """242 data = _get_leaderboard_data()243 244 if "error" in data and data.get("entries") == []:245 raise HTTPException(status_code=503, detail=data["error"])246 247 entries = data.get("entries", [])248 249 # Apply filters250 if openness:251 entries = [e for e in entries if e.get("openness") == openness]252 253 if min_categories is not None:254 entries = [e for e in entries if (e.get("categories_completed") or 0) >= min_categories]255 256 # Apply sorting257 reverse = True258 if sort_by in ["average_cost", "average_runtime"]:259 reverse = False # Lower is better260 261 entries.sort(262 key=lambda x: x.get(sort_by) if x.get(sort_by) is not None else (float('inf') if not reverse else float('-inf')),263 reverse=reverse264 )265 266 # Apply limit267 if limit:268 entries = entries[:limit]269 270 return {271 "entries": entries,272 "total_count": len(entries),273 "categories": ALL_CATEGORIES,274 "benchmarks": list(BENCHMARK_TO_CATEGORIES.keys()),275 "fetched_at": data.get("fetched_at"),276 }277 278 279@api_app.get("/leaderboard/models", tags=["Leaderboard"])280async def list_models(281 openness: Optional[str] = Query(None, description="Filter by openness (open/closed)"),282):283 """List all language models available in the leaderboard."""284 data = _get_leaderboard_data()285 entries = data.get("entries", [])286 287 if openness:288 entries = [e for e in entries if e.get("openness") == openness]289 290 models = [291 {292 "language_model": e.get("language_model"),293 "sdk_version": e.get("sdk_version"),294 "openness": e.get("openness"),295 "average_score": e.get("average_score"),296 "categories_completed": e.get("categories_completed"),297 }298 for e in entries299 ]300 301 return {302 "models": models,303 "total_count": len(models),304 }305 306 307@api_app.get("/leaderboard/model/{model_name}", tags=["Leaderboard"])308async def get_model(model_name: str):309 """Get detailed data for a specific language model."""310 data = _get_leaderboard_data()311 entries = data.get("entries", [])312 313 # Find entries matching the model name (case-insensitive)314 matching = [e for e in entries if (e.get("language_model") or "").lower() == model_name.lower()]315 316 if not matching:317 raise HTTPException(status_code=404, detail=f"Model '{model_name}' not found")318 319 return {320 "model_name": model_name,321 "entries": matching,322 "count": len(matching),323 }324 325 326@api_app.get("/categories", tags=["Metadata"])327async def list_categories():328 """List all benchmark categories with their associated benchmarks."""329 category_to_benchmarks = {}330 for benchmark, categories in BENCHMARK_TO_CATEGORIES.items():331 for category in categories:332 if category not in category_to_benchmarks:333 category_to_benchmarks[category] = []334 category_to_benchmarks[category].append(benchmark)335 336 return {337 "categories": [338 {339 "name": category,340 "description": CATEGORY_DESCRIPTIONS.get(category, ""),341 "benchmarks": category_to_benchmarks.get(category, [])342 }343 for category in ALL_CATEGORIES344 ]345 }346 347 348@api_app.get("/benchmarks", tags=["Metadata"])349async def list_benchmarks():350 """List all benchmarks with their category mappings."""351 return {352 "benchmarks": [353 {354 "name": benchmark,355 "categories": categories356 }357 for benchmark, categories in BENCHMARK_TO_CATEGORIES.items()358 ]359 }360