AlirezaDelavari/Final_Assignment_Template
0
1# import subprocess2 3# # pull the Qwen 2.5 weights via Ollama 4# subprocess.run(["ollama", "pull", "qwen2.5:7b-instruct"], check=True)5 6import os7import re8import gradio as gr9import requests10import pandas as pd11from typing import TypedDict, Annotated12 13from langgraph.graph.message import add_messages14from langgraph.graph import START, StateGraph15from langgraph.prebuilt import ToolNode, tools_condition16from langchain_core.messages import AnyMessage, HumanMessage, AIMessage, SystemMessage17from langchain_core.tools import tool18 19from langchain_community.tools import DuckDuckGoSearchRun20from langchain_community.document_loaders import WikipediaLoader, ArxivLoader21# from langchain_community.tools.tavily_search import TavilySearchResults22 23from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline24from langchain_openai import ChatOpenAI25from langchain_ollama import ChatOllama26from langgraph.prebuilt import create_react_agent27from langchain_core.messages.ai import AIMessage28# Create specialized agents29from langchain_community.tools import DuckDuckGoSearchRun30from langchain_community.document_loaders import WikipediaLoader, ArxivLoader31# from langchain_community.tools.tavily_search import TavilySearchResults32from datetime import datetime33 34 35 36# (Keep Constants as is)37# --- Constants ---38DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"39 40 41HUGGINGFACEHUB_API_TOKEN = os.getenv("HUGGINGFACEHUB_API_TOKEN")42if HUGGINGFACEHUB_API_TOKEN is None:43 raise ValueError("HUGGINGFACEHUB_API_TOKEN not set in environment")44 45 46def multiply(a: int, b: int) -> int:47 """Multiply two numbers.48 Args:49 a: first int50 b: second int51 """52 return a * b53 54 55 56def add(a: int, b: int) -> int:57 """Add two numbers.58 Args:59 a: first int60 b: second int61 """62 return a + b63 64 65 66def subtract(a: int, b: int) -> int:67 """Subtract two numbers.68 Args:69 a: first int70 b: second int71 """72 return a - b73 74 75 76def divide(a: int, b: int) -> int:77 """Divide two numbers.78 Args:79 a: first int80 b: second int81 """82 if b == 0:83 raise ValueError("Cannot divide by zero.")84 return a / b85 86 87 88def modulus(a: int, b: int) -> int:89 """Get the modulus of two numbers.90 Args:91 a: first int92 b: second int93 """94 return a % b95 96 97 98def wiki_search(query: str) -> str:99 """Search Wikipedia for a query and return maximum 2 results.100 Args:101 query: The search query."""102 search_docs = WikipediaLoader(query=query, load_max_docs=2).load()103 formatted_search_docs = "\n\n---\n\n".join(104 [105 f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'106 for doc in search_docs107 ])108 return {"wiki_results": formatted_search_docs}109 110 111 112# def web_search(query: str) -> str:113# """Search Tavily for a query and return maximum 3 results.114# Args:115# query: The search query."""116# search_docs = TavilySearchResults(max_results=3).invoke(query=query)117# formatted_search_docs = "\n\n---\n\n".join(118# [119# f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'120# for doc in search_docs121# ])122# return {"web_results": formatted_search_docs}123 124web_search = DuckDuckGoSearchRun()125 126def arvix_search(query: str) -> str:127 """Search Arxiv for a query and return maximum 3 result.128 Args:129 query: The search query."""130 search_docs = ArxivLoader(query=query, load_max_docs=3).load()131 formatted_search_docs = "\n\n---\n\n".join(132 [133 f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content[:1000]}\n</Document>'134 for doc in search_docs135 ])136 return {"arvix_results": formatted_search_docs}137 138 139def reverse_string(text: str) -> str:140 """Returns the reverse of the input string141 Args:142 text: The input stirng."""143 return text[::-1]144 145 146def get_now(format: str = "%Y-%m-%d %H:%M:%S") -> str:147 """148 Returns the current time formatted according to the `format` string.149 Args:150 format: the desired time format eg "%Y-%m-%d %H:%M:%S" """151 return datetime.now().strftime(format)152 153 154 155local_llm = "qwen2.5:7b-instruct"156# local_llm = "qwen2.5:0.5b-instruct"157model = ChatOllama(model=local_llm, temperature=0.0)158 159 160# --- Basic Agent Definition ---161# ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------162class BasicAgent:163 164 def __init__(self):165 self.agent = create_react_agent(166 model=model,167 tools=[add, multiply , subtract, divide , modulus , web_search , wiki_search, arvix_search, reverse_string, get_now],168 name="research_expert",169 # prompt="You are a general AI assistant with access to tools, I will ask you a question. Report your thoughts, and finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER]. YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string."170 prompt="""171 You are a general AI assistant that ALWAYS thinks step‐by‐step, chooses the correct tool, and then returns a final answer in exactly this format:172 173 Thought 1: <your reasoning here, e.g. I need to find which episode, so I should search “Teal’c ‘Isn’t that hot’ Stargate SG-1 transcript.”> 174 Action 1: duckduckgo_search[query="Teal’c ‘Isn’t that hot’ Stargate SG-1 transcript"] 175 Observation 1: <whatever DuckDuckGo returns> 176 Thought 2: <now I see a transcript snippet...> 177 ... 178 FINAL ANSWER: <the exact line Teal’c says>179 180 If you use a search tool, your query should include enough context words (e.g. “Teal’c,” “Stargate SG-1,” “transcript”). Always finish with “FINAL ANSWER: …” 181 """182 183 )184 185 186 def __call__(self, question: str) -> str:187 print(f"Agent received question (first 50 chars): {question[:50]}...")188 189 result = self.agent.invoke({190 "messages": [191 {192 "role": "user",193 "content": question194 }195 ]196 })197 198 199 return result['messages'][-1].content200 201 202 203def run_and_submit_all( profile: gr.OAuthProfile | None):204 """205 Fetches all questions, runs the BasicAgent on them, submits all answers,206 and displays the results.207 """208 # --- Determine HF Space Runtime URL and Repo URL ---209 space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code210 211 if profile:212 username= f"{profile.username}"213 print(f"User logged in: {username}")214 else:215 print("User not logged in.")216 return "Please Login to Hugging Face with the button.", None217 218 api_url = DEFAULT_API_URL219 questions_url = f"{api_url}/questions"220 submit_url = f"{api_url}/submit"221 222 # 1. Instantiate Agent ( modify this part to create your agent)223 try:224 agent = BasicAgent()225 except Exception as e:226 print(f"Error instantiating agent: {e}")227 return f"Error initializing agent: {e}", None228 # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)229 agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"230 print(agent_code)231 232 # 2. Fetch Questions233 print(f"Fetching questions from: {questions_url}")234 try:235 response = requests.get(questions_url, timeout=15)236 response.raise_for_status()237 questions_data = response.json()238 if not questions_data:239 print("Fetched questions list is empty.")240 return "Fetched questions list is empty or invalid format.", None241 print(f"Fetched {len(questions_data)} questions.")242 except requests.exceptions.RequestException as e:243 print(f"Error fetching questions: {e}")244 return f"Error fetching questions: {e}", None245 except requests.exceptions.JSONDecodeError as e:246 print(f"Error decoding JSON response from questions endpoint: {e}")247 print(f"Response text: {response.text[:500]}")248 return f"Error decoding server response for questions: {e}", None249 except Exception as e:250 print(f"An unexpected error occurred fetching questions: {e}")251 return f"An unexpected error occurred fetching questions: {e}", None252 253 # 3. Run your Agent254 results_log = []255 answers_payload = []256 print(f"Running agent on {len(questions_data)} questions...")257 for item in questions_data:258 task_id = item.get("task_id")259 question_text = item.get("question")260 if not task_id or question_text is None:261 print(f"Skipping item with missing task_id or question: {item}")262 continue263 try:264 submitted_answer = agent(question_text)265 answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})266 results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})267 except Exception as e:268 print(f"Error running agent on task {task_id}: {e}")269 results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})270 271 if not answers_payload:272 print("Agent did not produce any answers to submit.")273 return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)274 275 # 4. Prepare Submission276 submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}277 status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."278 print(status_update)279 280 # 5. Submit281 print(f"Submitting {len(answers_payload)} answers to: {submit_url}")282 try:283 response = requests.post(submit_url, json=submission_data, timeout=60)284 response.raise_for_status()285 result_data = response.json()286 final_status = (287 f"Submission Successful!\n"288 f"User: {result_data.get('username')}\n"289 f"Overall Score: {result_data.get('score', 'N/A')}% "290 f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"291 f"Message: {result_data.get('message', 'No message received.')}"292 )293 print("Submission successful.")294 results_df = pd.DataFrame(results_log)295 return final_status, results_df296 except requests.exceptions.HTTPError as e:297 error_detail = f"Server responded with status {e.response.status_code}."298 try:299 error_json = e.response.json()300 error_detail += f" Detail: {error_json.get('detail', e.response.text)}"301 except requests.exceptions.JSONDecodeError:302 error_detail += f" Response: {e.response.text[:500]}"303 status_message = f"Submission Failed: {error_detail}"304 print(status_message)305 results_df = pd.DataFrame(results_log)306 return status_message, results_df307 except requests.exceptions.Timeout:308 status_message = "Submission Failed: The request timed out."309 print(status_message)310 results_df = pd.DataFrame(results_log)311 return status_message, results_df312 except requests.exceptions.RequestException as e:313 status_message = f"Submission Failed: Network error - {e}"314 print(status_message)315 results_df = pd.DataFrame(results_log)316 return status_message, results_df317 except Exception as e:318 status_message = f"An unexpected error occurred during submission: {e}"319 print(status_message)320 results_df = pd.DataFrame(results_log)321 return status_message, results_df322 323 324# --- Build Gradio Interface using Blocks ---325with gr.Blocks() as demo:326 gr.Markdown("# Basic Agent Evaluation Runner")327 gr.Markdown(328 """329 **Instructions:**330 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...331 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.332 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.333 ---334 **Disclaimers:**335 Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).336 This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.337 """338 )339 340 gr.LoginButton()341 342 run_button = gr.Button("Run Evaluation & Submit All Answers")343 344 status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)345 # Removed max_rows=10 from DataFrame constructor346 results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)347 348 run_button.click(349 fn=run_and_submit_all,350 outputs=[status_output, results_table]351 )352 353if __name__ == "__main__":354 print("\n" + "-"*30 + " App Starting " + "-"*30)355 # Check for SPACE_HOST and SPACE_ID at startup for information356 space_host_startup = os.getenv("SPACE_HOST")357 space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup358 359 if space_host_startup:360 print(f"✅ SPACE_HOST found: {space_host_startup}")361 print(f" Runtime URL should be: https://{space_host_startup}.hf.space")362 else:363 print("ℹ️ SPACE_HOST environment variable not found (running locally?).")364 365 if space_id_startup: # Print repo URLs if SPACE_ID is found366 print(f"✅ SPACE_ID found: {space_id_startup}")367 print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")368 print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")369 else:370 print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")371 372 print("-"*(60 + len(" App Starting ")) + "\n")373 374 print("Launching Gradio Interface for Basic Agent Evaluation...")375 demo.launch(debug=True, share=False)