flutterdev555/mymodel
0
1"""GAIA Benchmark Evaluation Runner — smolagents CodeAgent"""2import os3import re4import gradio as gr5import requests6import pandas as pd7from smolagents import (8 CodeAgent,9 DuckDuckGoSearchTool,10)11 12# Import the correct model class (name changed across versions)13try:14 from smolagents import InferenceClientModel as ModelClass15except ImportError:16 try:17 from smolagents import HfApiModel as ModelClass18 except ImportError:19 from smolagents import ApiModel as ModelClass20import yaml21 22from tools.final_answer import FinalAnswerTool23from tools.visit_webpage import VisitWebpageTool24from tools.web_search import DuckDuckGoSearchTool as CustomSearchTool25 26# --- Constants ---27DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"28 29 30def build_agent():31 """Build a smolagents CodeAgent equipped for GAIA benchmark tasks."""32 33 # Model — Use the HF Inference API34 # Try multiple models in order of preference35 model_id = os.getenv(36 "MODEL_ID",37 "Qwen/Qwen2.5-Coder-32B-Instruct"38 )39 model = ModelClass(40 max_tokens=4096,41 temperature=0.1,42 model_id=model_id,43 custom_role_conversions=None,44 )45 46 # Tools47 final_answer = FinalAnswerTool()48 visit_webpage = VisitWebpageTool()49 search_tool = CustomSearchTool(max_results=5)50 51 # Load prompt templates52 with open("prompts.yaml", "r") as stream:53 prompt_templates = yaml.safe_load(stream)54 55 agent = CodeAgent(56 model=model,57 tools=[search_tool, visit_webpage, final_answer],58 max_steps=12,59 verbosity_level=1,60 name="gaia_agent",61 description="An agent designed for GAIA benchmark question answering.",62 prompt_templates=prompt_templates,63 )64 return agent65 66 67def extract_answer(raw_answer) -> str:68 """Aggressively clean agent output to extract only the final answer value."""69 if raw_answer is None:70 return ""71 answer = str(raw_answer).strip()72 73 # If the answer contains final_answer("..."), extract the argument74 fa_match = re.search(r'final_answer\(["\'](.+?)["\']\)', answer, re.DOTALL)75 if fa_match:76 answer = fa_match.group(1).strip()77 78 # Remove code blocks (```py ... ```)79 answer = re.sub(r'```[\s\S]*?```', '', answer).strip()80 81 # Remove <end_code> tags and surrounding artifacts82 answer = re.sub(r'<end_code>.*', '', answer, flags=re.DOTALL).strip()83 84 # Remove Calling tools: [...] JSON metadata85 answer = re.sub(r'Calling tools:.*', '', answer, flags=re.DOTALL).strip()86 87 # Remove "Using the `final_answer` tool:" and similar88 answer = re.sub(r'Using the `final_answer` tool:.*', '', answer, flags=re.DOTALL).strip()89 90 # Remove Thought: / Code: sections if they leaked through91 answer = re.sub(r'^Thought:.*?(?=\S)', '', answer, flags=re.DOTALL).strip()92 93 # Remove common prefixes94 prefixes = [95 "FINAL ANSWER:", "Final Answer:", "final answer:",96 "The final answer is:", "The final answer is ",97 "The answer is:", "The answer is ",98 "Answer:", "Final answer:",99 ]100 for prefix in prefixes:101 if answer.lower().startswith(prefix.lower()):102 answer = answer[len(prefix):].strip()103 104 # Remove surrounding quotes if present105 if len(answer) >= 2:106 if (answer[0] == '"' and answer[-1] == '"') or \107 (answer[0] == "'" and answer[-1] == "'"):108 answer = answer[1:-1].strip()109 110 # Remove trailing periods (unless it's a decimal number)111 if answer.endswith('.') and not re.match(r'^\d+\.$', answer):112 answer = answer[:-1].strip()113 114 return answer115 116 117def run_and_submit_all(profile: gr.OAuthProfile | None):118 """119 Fetches all questions, runs the agent on them, submits all answers,120 and displays the results.121 """122 space_id = os.getenv("SPACE_ID")123 124 if profile:125 username = f"{profile.username}"126 print(f"User logged in: {username}")127 else:128 print("User not logged in.")129 return "Please Login to Hugging Face with the button.", None130 131 api_url = DEFAULT_API_URL132 questions_url = f"{api_url}/questions"133 submit_url = f"{api_url}/submit"134 135 # 1. Instantiate Agent136 try:137 agent = build_agent()138 except Exception as e:139 print(f"Error instantiating agent: {e}")140 return f"Error initializing agent: {e}", None141 142 agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"143 print(f"Agent code URL: {agent_code}")144 145 # 2. Fetch Questions146 print(f"Fetching questions from: {questions_url}")147 try:148 response = requests.get(questions_url, timeout=15)149 response.raise_for_status()150 questions_data = response.json()151 if not questions_data:152 print("Fetched questions list is empty.")153 return "Fetched questions list is empty or invalid format.", None154 print(f"Fetched {len(questions_data)} questions.")155 except requests.exceptions.RequestException as e:156 print(f"Error fetching questions: {e}")157 return f"Error fetching questions: {e}", None158 except requests.exceptions.JSONDecodeError as e:159 print(f"Error decoding JSON response: {e}")160 return f"Error decoding server response: {e}", None161 except Exception as e:162 print(f"An unexpected error occurred fetching questions: {e}")163 return f"An unexpected error occurred: {e}", None164 165 # 3. Run Agent on each question166 results_log = []167 answers_payload = []168 print(f"Running agent on {len(questions_data)} questions...")169 for i, item in enumerate(questions_data):170 task_id = item.get("task_id")171 question_text = item.get("question")172 if not task_id or question_text is None:173 print(f"Skipping item with missing task_id or question: {item}")174 continue175 try:176 print(f"\n{'='*60}")177 print(f"Question {i+1}/{len(questions_data)} (task_id: {task_id})")178 print(f"Q: {question_text[:100]}...")179 raw_answer = agent.run(question_text, reset=True)180 submitted_answer = extract_answer(raw_answer)181 print(f"A: {submitted_answer}")182 answers_payload.append({183 "task_id": task_id,184 "submitted_answer": submitted_answer,185 })186 results_log.append({187 "Task ID": task_id,188 "Question": question_text,189 "Submitted Answer": submitted_answer,190 })191 except Exception as e:192 print(f"Error running agent on task {task_id}: {e}")193 results_log.append({194 "Task ID": task_id,195 "Question": question_text,196 "Submitted Answer": f"AGENT ERROR: {e}",197 })198 199 if not answers_payload:200 print("Agent did not produce any answers to submit.")201 return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)202 203 # 4. Prepare Submission204 submission_data = {205 "username": username.strip(),206 "agent_code": agent_code,207 "answers": answers_payload,208 }209 status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."210 print(status_update)211 212 # 5. Submit213 print(f"Submitting {len(answers_payload)} answers to: {submit_url}")214 try:215 response = requests.post(submit_url, json=submission_data, timeout=120)216 response.raise_for_status()217 result_data = response.json()218 final_status = (219 f"Submission Successful!\n"220 f"User: {result_data.get('username')}\n"221 f"Overall Score: {result_data.get('score', 'N/A')}% "222 f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"223 f"Message: {result_data.get('message', 'No message received.')}"224 )225 print("Submission successful.")226 results_df = pd.DataFrame(results_log)227 return final_status, results_df228 except requests.exceptions.HTTPError as e:229 error_detail = f"Server responded with status {e.response.status_code}."230 try:231 error_json = e.response.json()232 error_detail += f" Detail: {error_json.get('detail', e.response.text)}"233 except requests.exceptions.JSONDecodeError:234 error_detail += f" Response: {e.response.text[:500]}"235 status_message = f"Submission Failed: {error_detail}"236 print(status_message)237 results_df = pd.DataFrame(results_log)238 return status_message, results_df239 except requests.exceptions.Timeout:240 status_message = "Submission Failed: The request timed out."241 print(status_message)242 results_df = pd.DataFrame(results_log)243 return status_message, results_df244 except requests.exceptions.RequestException as e:245 status_message = f"Submission Failed: Network error - {e}"246 print(status_message)247 results_df = pd.DataFrame(results_log)248 return status_message, results_df249 except Exception as e:250 status_message = f"An unexpected error occurred during submission: {e}"251 print(status_message)252 results_df = pd.DataFrame(results_log)253 return status_message, results_df254 255 256# --- Build Gradio Interface ---257with gr.Blocks() as demo:258 gr.Markdown("# GAIA Benchmark Agent Evaluation")259 gr.Markdown(260 """261 **Instructions:**262 1. Log in to your Hugging Face account using the button below.263 2. Click 'Run Evaluation & Submit All Answers' to fetch questions,264 run the agent, submit answers, and see the score.265 266 ---267 **Note:** This may take several minutes as the agent processes all questions.268 """269 )270 271 gr.LoginButton()272 273 run_button = gr.Button("Run Evaluation & Submit All Answers")274 275 status_output = gr.Textbox(276 label="Run Status / Submission Result", lines=5, interactive=False277 )278 results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)279 280 run_button.click(281 fn=run_and_submit_all,282 outputs=[status_output, results_table],283 )284 285if __name__ == "__main__":286 print("\n" + "-" * 30 + " App Starting " + "-" * 30)287 space_host = os.getenv("SPACE_HOST")288 space_id = os.getenv("SPACE_ID")289 if space_host:290 print(f"✅ SPACE_HOST: {space_host}")291 else:292 print("ℹ️ SPACE_HOST not found (running locally?).")293 if space_id:294 print(f"✅ SPACE_ID: {space_id}")295 else:296 print("ℹ️ SPACE_ID not found (running locally?).")297 print("-" * (60 + len(" App Starting ")) + "\n")298 print("Launching Gradio Interface for GAIA Evaluation...")299 demo.launch(debug=True, share=False, server_name="0.0.0.0", server_port=7860, ssr_mode=False)