ykumar2020/gaia-final-assignment

1"""Public Space UI: reviewable dry-runs and a separate explicit submission."""2 3from __future__ import annotations4 5import json6import os7from pathlib import Path8from typing import Any9 10import gradio as gr11import pandas as pd12from huggingface_hub import get_token13 14from agent import GaiaAgent15from agent_code import resolve_agent_code16from cache import AnswerCache, ResultStore17from config import Settings18from evaluation import run_evaluation, submission_answers19from gaia_client import GaiaClient20 21SETTINGS = Settings.from_env()22PROJECT_ROOT = Path(__file__).resolve().parent23RESULT_STORE = ResultStore(Path(os.getenv("GAIA_RESULTS_PATH", "results/results.json")))24 25 26def has_huggingface_login() -> bool:27    """Detect Space OAuth or local CLI auth without retaining or logging the token."""28    if os.getenv("SPACE_ID") or os.getenv("HF_TOKEN"):29        return True30    try:31        return bool(get_token())32    except OSError:33        return False34 35 36def _frame(results: list[dict[str, Any]]) -> pd.DataFrame:37    return pd.DataFrame(38        [39            {40                "Task ID": row.get("task_id", ""),41                "Task Type": row.get("task_type", ""),42                "Question": row.get("question", ""),43                "Generated Answer": row.get("answer", ""),44                "Seconds": row.get("duration_seconds", 0),45                "Status": row.get("status", ""),46                "Error": row.get("error", ""),47                "Evidence": json.dumps(row.get("evidence", []), ensure_ascii=False),48            }49            for row in results50        ]51    )52 53 54def _ordered_all(client: GaiaClient) -> list[dict[str, Any]]:55    ids = [str(task["task_id"]) for task in client.get_questions()]56    return RESULT_STORE.ordered(ids)57 58 59def run_dry_evaluation(force: bool = False) -> tuple[str, pd.DataFrame]:60    """Answer every task and checkpoint results; this function cannot submit."""61    try:62        client = GaiaClient(SETTINGS)63        results = run_evaluation(64            client,65            GaiaAgent(SETTINGS),66            RESULT_STORE,67            force=bool(force),68        )69        completed = sum(row["status"] == "ok" for row in results)70        failed = len(results) - completed71        status = (72            f"Dry run complete: {completed}/{len(results)} answered; {failed} failed. "73            "No answers were submitted."74        )75        if len(results) == 20 and failed == 0:76            status += " All 20 unique tasks are ready for explicit submission."77        return status, _frame(results)78    except Exception as exc:79        return f"Dry run failed: {type(exc).__name__}: {exc}", _frame(80            RESULT_STORE.ordered()81        )82 83 84def rerun_task(task_id: str) -> tuple[str, pd.DataFrame]:85    """Force one task to run again while retaining other checkpoints."""86    task_id = str(task_id or "").strip()87    if not task_id:88        return "Enter a Task ID to rerun.", _frame(RESULT_STORE.ordered())89    try:90        client = GaiaClient(SETTINGS)91        run_evaluation(92            client,93            GaiaAgent(94                SETTINGS,95                AnswerCache(SETTINGS.cache_dir / "answers.json", enabled=False),96            ),97            RESULT_STORE,98            force=True,99            task_ids={task_id},100        )101        results = _ordered_all(client)102        row = RESULT_STORE.load()[task_id]103        return f"Rerun {task_id}: {row['status']}", _frame(results)104    except Exception as exc:105        return f"Rerun failed: {type(exc).__name__}: {exc}", _frame(106            RESULT_STORE.ordered()107        )108 109 110def submit_cached_answers(profile: gr.OAuthProfile | None) -> tuple[str, pd.DataFrame]:111    """The sole explicit route to the official submission POST."""112    try:113        client = GaiaClient(SETTINGS)114        results = _ordered_all(client)115    except Exception as exc:116        return f"Could not validate cached run: {type(exc).__name__}: {exc}", _frame(117            RESULT_STORE.ordered()118        )119    if profile is None:120        return "Please log in to Hugging Face before submitting.", _frame(results)121    try:122        answers = submission_answers(results, expected_count=20)123        agent_code = resolve_agent_code(124            space_id=os.getenv("SPACE_ID"),125            configured_url=SETTINGS.agent_code_url,126            allow_inline=SETTINGS.allow_inline_agent_code,127            root=PROJECT_ROOT,128        )129        response = client.submit_answers(130            username=str(profile.username).strip(),131            agent_code=agent_code,132            answers=answers,133        )134        status = (135            "Submission successful.\n"136            f"User: {response.get('username', profile.username)}\n"137            f"Overall Score: {response.get('score', 'N/A')}% "138            f"({response.get('correct_count', '?')}/{response.get('total_attempted', '?')} correct)\n"139            f"Message: {response.get('message', 'No message received.')}"140        )141        return status, _frame(results)142    except Exception as exc:143        return f"Submission not sent: {type(exc).__name__}: {exc}", _frame(results)144 145 146with gr.Blocks() as demo:147    gr.Markdown("# Modular GAIA Level-1 Agent")148    gr.Markdown(149        "Run or resume a dry evaluation, review all answers/evidence, and rerun individual "150        "tasks. Submission is a distinct action and is enabled logically only when exactly 20 "151        "unique tasks have non-empty successful answers. Keep this Space public."152    )153    # Outside Spaces, Gradio mocks OAuth from the token saved by `hf auth login`.154    # Only test whether one exists; model/ASR credentials remain environment-only.155    if has_huggingface_login():156        gr.LoginButton()157    else:158        gr.Markdown(159            "Local mode: run `hf auth login`, then expose `HF_TOKEN` to this process for "160            "inference. Configure `GAIA_AGENT_CODE_URL`, or explicitly enable inline source."161        )162    with gr.Row():163        force_all = gr.Checkbox(label="Force rerun all tasks", value=False)164        run_button = gr.Button("Run / Resume Dry Evaluation", variant="primary")165        submit_button = gr.Button("Submit Reviewed Complete Run", variant="secondary")166    with gr.Row():167        rerun_id = gr.Textbox(label="Task ID to rerun")168        rerun_button = gr.Button("Rerun Selected Task")169    status_output = gr.Textbox(label="Status", lines=5, interactive=False)170    results_table = gr.DataFrame(label="20-question review", wrap=True)171    run_button.click(172        run_dry_evaluation, inputs=[force_all], outputs=[status_output, results_table]173    )174    rerun_button.click(175        rerun_task, inputs=[rerun_id], outputs=[status_output, results_table]176    )177    submit_button.click(submit_cached_answers, outputs=[status_output, results_table])178 179 180if __name__ == "__main__":181    demo.queue(default_concurrency_limit=1).launch(debug=False, share=False)182