bigcode/bigcodebench-interaction

1import gradio as gr2import io3import sys4import logging5import multiprocessing6import os7import pickle8import threading9import time10from collections import Counter, defaultdict11from concurrent.futures import ProcessPoolExecutor, as_completed, wait, FIRST_COMPLETED12from datetime import datetime13from typing import Any, Dict, List, Tuple14from warnings import warn15from contextlib import redirect_stdout, redirect_stderr16 17import numpy as np18from huggingface_hub import HfApi19from bigcodebench.data.utils import CACHE_DIR20from bigcodebench.eval import PASS, compatible_eval_result, estimate_pass_at_k, untrusted_check21from bigcodebench.gen.util import trusted_check22from apscheduler.schedulers.background import BackgroundScheduler23from datasets import load_datase24 25REPO_ID = "bigcode/bigcodebench-interaction"26HF_TOKEN = os.environ.get("HF_TOKEN", None)27API = HfApi(token=HF_TOKEN)28Result = Tuple[str, List[bool]]29 30dataset = load_dataset("bigcode/bigcodebench-tool")31tasks = {32    _id: task["mixed_tool_implementation"]33    for _id, task in dataset.items()34}35 36 37def run_code(code: str, _id: str) -> str:38    # Create string buffers to capture output39    stdout_buffer = io.StringIO()40    stderr_buffer = io.StringIO()41    pre_code = tasks[_id]42    # Capture both stdout and stderr43    with redirect_stdout(stdout_buffer), redirect_stderr(stderr_buffer):44        try:45            # Execute the code46            exec(pre_code + code)47            48            # Get the output49            output = stdout_buffer.getvalue()50            errors = stderr_buffer.getvalue()51            52            # Combine stdout and stderr53            result = output54            if errors:55                result += "\n--- Errors ---\n" + errors56                57        except Exception as e:58            # Capture any execution errors59            result = f"Error: {str(e)}"60    61    return result62 63# Create the Gradio interface with better styling64interface = gr.Interface(65    fn=run_code,66    inputs=[67        gr.Code(label="Python Code", language="python"),68        gr.Dropdown(label="Task", choices=list(tasks.keys())),69    ],70    outputs=[71        gr.Textbox(label="Output")72    ],73)74interface.queue(default_concurrency_limit=None)75 76 77def restart_space():78    logging.info(f"Restarting space with repo ID: {REPO_ID}")79    try:80        # Now restart the space81        API.restart_space(repo_id=REPO_ID, token=HF_TOKEN)82        logging.info("Space restarted successfully.")83    except Exception as e:84        logging.error(f"Failed to restart space: {e}")85 86 87scheduler = BackgroundScheduler()88scheduler.add_job(restart_space, "interval", hours=5)  # Restart every 5hs89scheduler.start()90interface.launch(show_error=True)91