npmaker/Final_Assignment
0
1import os2from dotenv import load_dotenv3import gradio as gr4import requests5import pandas as pd6from smolagents import CodeAgent, VisitWebpageTool, WebSearchTool, OpenAIServerModel7from smolagents.memory import ActionStep8from selenium_wiki_tool import WikipediaSeleniumTool9from google_search import GoogleSearchTool10from gemini_image_tool import GeminiImageTool11from gemini_audio_tool import GeminiAudioTool12import tracer13import yaml14import google.genai as genai15import time16from typing import Generator, Any, List17 18load_dotenv()19 20# --- Constants ---21DEFAULT_API_URL = os.getenv('HF_UNIT4_URL') #, "https://agents-course-unit4-scoring.hf.space")22 23class Agent:24 max_steps = 1225 temperature = 0.326 #model = 'gpt-4.1'27 model_name = "gemini-2.0-flash"28 request_timestamps = []29 max_requests_per_minute = 1030 31 def __init__(self):32 websearch_tool = WebSearchTool()33 webpage_tool = VisitWebpageTool(max_output_length=1000000)34 wiki_selenium_tool = WikipediaSeleniumTool()35 google_search = GoogleSearchTool()36 gemini_image_tool = GeminiImageTool()37 gemini_audio_tool = GeminiAudioTool()38 self.tools = [webpage_tool, wiki_selenium_tool, google_search, gemini_image_tool, gemini_audio_tool]39 40 with open("prompts.yaml", 'r') as stream:41 prompt_templates = yaml.safe_load(stream)42 prompt_templates['system_prompt'] += f"\n\n{prompt_templates['gaia_requirements']}"43 44 """self.model = InferenceClientModel(45 max_tokens=2096,46 temperature=0.2,47 model_id='Qwen/Qwen2.5-Coder-32B-Instruct',48 custom_role_conversions=None,49 )50 # OpenAI model51 self.model = OpenAIServerModel(52 api_key=os.getenv("OPENAI_API_KEY"),53 api_base="https://api.openai.com/v1",54 model_id=self.model,55 temperature=self.temperature,56 max_tokens=409657 )"""58 59 self.model = OpenAIServerModel(60 model_id=self.model_name,61 api_key=os.getenv("GOOGLE_API_KEY"),62 # Google Gemini OpenAI-compatible API base URL63 api_base="https://generativelanguage.googleapis.com/v1beta/openai/",64 )65 66 self.agent = CodeAgent(67 name="Final_Assignment_Agent",68 description="This is the Final Assignment agent.",69 tools=self.tools,70 model=self.model,71 max_steps=self.max_steps,72 prompt_templates=prompt_templates,73 )74 self.agent._check_rate_limit = self._check_rate_limit75 self.agent._execute_step = self._execute_step76 77 print("Agent initialized.")78 79 def _check_rate_limit(self):80 # Check if we need to throttle requests81 current_time = time.time()82 # Remove timestamps older than 60 seconds83 self.request_timestamps = [t for t in self.request_timestamps if current_time - t < 60]84 # If we've made too many requests in the last minute, wait85 if len(self.request_timestamps) >= self.max_requests_per_minute:86 # Wait until the oldest request is more than a minute old87 wait_time = 60 - (current_time - self.request_timestamps[0])88 if wait_time > 0:89 print(f"Wait for {round(wait_time,1)} seconds\n")90 time.sleep(wait_time)91 92 # Add the current request timestamp93 self.request_timestamps.append(time.time())94 print(f"Requests in last minute: {len(self.request_timestamps)}\n")95 96 def _execute_step(self, memory_step: ActionStep) -> Generator[Any, None, None]:97 #self.logger.log_rule(f"Step {self.step_number}", level=LogLevel.INFO)98 99 # Apply rate limiting before executing the step100 self.agent._check_rate_limit()101 102 final_answer = None103 for el in self.agent._step_stream(memory_step):104 final_answer = el105 yield el106 if final_answer is not None and self.agent.final_answer_checks:107 self.agent._validate_final_answer(final_answer)108 yield final_answer109 110 def __call__(self, question: str, task_id: str) -> str:111 112 #print(f"system_prompt: {self.agent.prompt_templates['system_prompt']}")113 print(f"Agent received question (first 50 chars): {question[:50]}...")114 print(f"Task ID: {task_id}")115 load_file_prompt = """When you need to download a file that is associated with the task you can download the file at the following location: """116 load_file_prompt += f"{DEFAULT_API_URL}/files/{task_id}"117 question = load_file_prompt + "\n\n" + question118 agent_answer = self.agent.run(question)119 120 print(f"Agent returning answer: {agent_answer}")121 122 return agent_answer123 124def run_and_submit_all( profile: gr.OAuthProfile | None):125 """126 Fetches all questions, runs the Agent on them, submits all answers,127 and displays the results.128 """129 # --- Determine HF Space Runtime URL and Repo URL ---130 space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code131 132 if profile:133 username= f"{profile.username}"134 print(f"User logged in: {username}")135 else:136 print("User not logged in.")137 return "Please Login to Hugging Face with the button.", None138 139 api_url = DEFAULT_API_URL140 questions_url = f"{api_url}/questions"141 submit_url = f"{api_url}/submit"142 143 # 1. Instantiate Agent ( modify this part to create your agent)144 try:145 agent = Agent()146 except Exception as e:147 print(f"Error instantiating agent: {e}")148 return f"Error initializing agent: {e}", None149 # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)150 agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"151 print(agent_code)152 153 # 2. Fetch Questions154 print(f"Fetching questions from: {questions_url}")155 try:156 response = requests.get(questions_url, timeout=15)157 response.raise_for_status()158 questions_data = response.json()159 if not questions_data:160 print("Fetched questions list is empty.")161 return "Fetched questions list is empty or invalid format.", None162 print(f"Fetched {len(questions_data)} questions.")163 except requests.exceptions.RequestException as e:164 print(f"Error fetching questions: {e}")165 return f"Error fetching questions: {e}", None166 except requests.exceptions.JSONDecodeError as e:167 print(f"Error decoding JSON response from questions endpoint: {e}")168 print(f"Response text: {response.text[:500]}")169 return f"Error decoding server response for questions: {e}", None170 except Exception as e:171 print(f"An unexpected error occurred fetching questions: {e}")172 return f"An unexpected error occurred fetching questions: {e}", None173 174 # 3. Run your Agent175 results_log = []176 answers_payload = []177 print(f"Running agent on {len(questions_data)} questions...")178 for item in questions_data:179 task_id = item.get("task_id")180 question_text = item.get("question")181 if not task_id or question_text is None:182 print(f"Skipping item with missing task_id or question: {item}")183 continue184 try:185 submitted_answer = agent(question_text, task_id)186 answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})187 results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})188 except Exception as e:189 print(f"Error running agent on task {task_id}: {e}")190 results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})191 192 if not answers_payload:193 print("Agent did not produce any answers to submit.")194 return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)195 196 # 4. Prepare Submission 197 submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}198 status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."199 print(status_update)200 201 # 5. Submit202 print(f"Submitting {len(answers_payload)} answers to: {submit_url}")203 try:204 response = requests.post(submit_url, json=submission_data, timeout=60)205 response.raise_for_status()206 result_data = response.json()207 final_status = (208 f"Submission Successful!\n"209 f"User: {result_data.get('username')}\n"210 f"Overall Score: {result_data.get('score', 'N/A')}% "211 f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"212 f"Message: {result_data.get('message', 'No message received.')}"213 )214 print("Submission successful.")215 results_df = pd.DataFrame(results_log)216 return final_status, results_df217 except requests.exceptions.HTTPError as e:218 error_detail = f"Server responded with status {e.response.status_code}."219 try:220 error_json = e.response.json()221 error_detail += f" Detail: {error_json.get('detail', e.response.text)}"222 except requests.exceptions.JSONDecodeError:223 error_detail += f" Response: {e.response.text[:500]}"224 status_message = f"Submission Failed: {error_detail}"225 print(status_message)226 results_df = pd.DataFrame(results_log)227 return status_message, results_df228 except requests.exceptions.Timeout:229 status_message = "Submission Failed: The request timed out."230 print(status_message)231 results_df = pd.DataFrame(results_log)232 return status_message, results_df233 except requests.exceptions.RequestException as e:234 status_message = f"Submission Failed: Network error - {e}"235 print(status_message)236 results_df = pd.DataFrame(results_log)237 return status_message, results_df238 except Exception as e:239 status_message = f"An unexpected error occurred during submission: {e}"240 print(status_message)241 results_df = pd.DataFrame(results_log)242 return status_message, results_df243 244 245# --- Build Gradio Interface using Blocks ---246with gr.Blocks() as demo:247 gr.Markdown("# Basic Agent Evaluation Runner")248 gr.Markdown(249 """250 **Instructions:**251 252 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...253 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.254 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.255 256 ---257 **Disclaimers:**258 Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).259 This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.260 """261 )262 263 gr.LoginButton()264 265 run_button = gr.Button("Run Evaluation & Submit All Answers")266 267 status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)268 # Removed max_rows=10 from DataFrame constructor269 results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)270 271 run_button.click(272 fn=run_and_submit_all,273 outputs=[status_output, results_table]274 )275 276if __name__ == "__main__":277 print("\n" + "-"*30 + " App Starting " + "-"*30)278 # Check for SPACE_HOST and SPACE_ID at startup for information279 space_host_startup = os.getenv("SPACE_HOST")280 space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup281 282 if space_host_startup:283 print(f"✅ SPACE_HOST found: {space_host_startup}")284 print(f" Runtime URL should be: https://{space_host_startup}.hf.space")285 else:286 print("ℹ️ SPACE_HOST environment variable not found (running locally?).")287 288 if space_id_startup: # Print repo URLs if SPACE_ID is found289 print(f"✅ SPACE_ID found: {space_id_startup}")290 print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")291 print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")292 else:293 print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")294 295 print("-"*(60 + len(" App Starting ")) + "\n")296 297 print("Launching Gradio Interface for Basic Agent Evaluation...")298 demo.launch(debug=True, share=False, server_name="0.0.0.0", server_port=7860)