hchtao/Final_Assignment_Template
0
1import os2import gradio as gr3import requests4import inspect5import pandas as pd6import time7 8import base649from typing import List, TypedDict, Annotated, Optional, Literal10from langchain_openai import ChatOpenAI11from langchain_core.messages import AnyMessage, SystemMessage, HumanMessage12from langgraph.graph.message import add_messages13from langgraph.graph import START, END, StateGraph14from langgraph.prebuilt import ToolNode, tools_condition15from langchain_community.tools import DuckDuckGoSearchRun16from langchain_community.document_loaders import TextLoader17from youtube_transcript_api import YouTubeTranscriptApi18 19# (Keep Constants as is)20# --- Constants ---21DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"22 23# --- Basic Agent Definition ---24# ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------25class BasicAgent:26 def __init__(self):27 print("BasicAgent initialized.")28 def __call__(self, question: str, file_name: str) -> str:29 30 def online_search(query: str) -> str:31 '''32 search for information online using DuckDuckSearch/ChatGPT search33 '''34 search = DuckDuckGoSearchRun()35 response = search.invoke(query)36 print(f"search keywords:{query}")37 # sllm = ChatOpenAI(model="gpt-4o-search-preview")38 # response = sllm.invoke(query).content.strip()39 time.sleep(0.3)40 return response41 42 def read_img(file_name: str) -> str:43 '''44 read the image with file_name and convert the information into text45 '''46 vllm = ChatOpenAI(model="gpt-4o-mini")47 with open(file_name, "rb") as image_file:48 image_bytes = image_file.read()49 base64_image = base64.b64encode(image_bytes).decode("utf-8")50 input = [51 {52 "role": "user",53 "content": [54 { "type": "text", "text": "Describe in detail what's in this image." },55 {56 "type": "image_url",57 "image_url": {58 "url": f"data:image/png;base64,{base64_image}"59 },60 },61 ],62 }63 ]64 response = vllm.invoke(input).content.strip()65 print(f"image: {response[:50]}")66 return 'The image has been turned into text:' + response67 68 def read_spreadsheet(file_name: str) -> str:69 '''70 read spreadsheet/Excel with file_name using CSVLoader into text71 '''72 data = pd.read_excel(file_name, dtype=str).to_string()73 print(f"data from excel:{data}")74 return 'The Excel has been turned into text:' + data75 76 def read_textfile(file_name: str) -> str:77 '''78 read any text files including code files with file_name using TextLoader into text79 '''80 loader = TextLoader(file_name)81 documents = loader.load()82 data = " ".join(doc.page_content for doc in documents)83 print(f"data from textfile:{data}")84 return 'The file has been turned into text:' + data85 86 def read_audio(file_name: str) -> str:87 '''88 transcribe any audio files with file_name into text89 '''90 allm = ChatOpenAI(model="gpt-4o-mini-audio-preview")91 with open(file_name, "rb") as audio_file:92 audio_bytes = audio_file.read()93 encoded_string = base64.b64encode(audio_bytes).decode('utf-8')94 input = [95 {96 "role": "user",97 "content": [98 { 99 "type": "text",100 "text": "Transcribe this recording word by word."101 },102 {103 "type": "input_audio",104 "input_audio": {105 "data": encoded_string,106 "format": "mp3"107 }108 }109 ]110 },111 ]112 response = allm.invoke(input).content.strip()113 print(f"audio transcript: {response[:50]}")114 return 'The audio has been turned into text:' + response115 116 def read_ytb(url: str) -> str:117 '''118 get the transcript of youtube video119 '''120 ytt_api = YouTubeTranscriptApi()121 documents = ytt_api.fetch(url.split('?v=')[-1])122 data = " ".join(doc.text for doc in documents)123 print(f"video transcript: {data}")124 return 'The youtube video has been turned into text:' + data125 126 tools = [read_img, read_audio, read_spreadsheet, read_textfile, read_ytb, online_search]127 llm = ChatOpenAI(model="gpt-4.1")128 llm_with_tools = llm.bind_tools(tools, parallel_tool_calls=False)129 130 class AgentState(TypedDict):131 messages: Annotated[list[AnyMessage], add_messages]132 file_name: Optional[str]133 agent_call_cnt: int134 135 def assistant(state: AgentState):136 read_file_tool_desc = """137 read_img(file_name: str) -> str:138 read the image with file_name and convert the information into text139 read_spreadsheet(file_name: str) -> str:140 read spreadsheet/Excel with file_name using CSVLoader into text141 read_textfile(file_name: str) -> str:142 read any text files including code files with file_name using TextLoader into text143 read_audio(file_name: str) -> str:144 transcribe any audio files with file_name into text145 """146 147 ytb_tool_desc = """148 read_ytb(url: str) -> str:149 get the transcript of youtube video with url150 """151 152 search_tool_desc = '''153 online_search(query: str) -> str:154 search for information online using DuckDuckSearch/ChatGPT search155 '''156 157 # print(state["messages"])158 sys_msg = SystemMessage(content=f"""You are a smart AI that is good at answering complicated questions with all the tools you have. You always think step-by-step.159 I will ask you a question. Based on your own knowledge and information from tool use return, try to finish your answer. 160 General rules for tool use are explained in angle brackets:161 <If a file exists, but has not been turned into text yet, choose appropriate tool from {read_file_tool_desc} to read the file. The 'file_name' attribute for the tool should be '{state["file_name"]}'>162 <If a youtube url link exists, but has not been turned into text yet, use {ytb_tool_desc} to transcribe the youtube video of the url link.>163 <If the information you have is insufficient to finish your answer to the question, craft a search query for online search tool {search_tool_desc} to gather more online information towards the question.> 164 Here is my question:165 """)166 print(f"cnt:{state['agent_call_cnt']}")167 return {168 "messages": [llm_with_tools.invoke([sys_msg] + state["messages"])],169 "file_name": state["file_name"],170 "agent_call_cnt": state["agent_call_cnt"] + 1171 }172 173 174 # The graph175 builder = StateGraph(AgentState)176 177 # Define nodes: these do the work178 builder.add_node("assistant", assistant)179 builder.add_node("tools", ToolNode(tools))180 181 # Define edges: these determine how the control flow moves182 builder.add_edge(START, "assistant")183 184 def router(state: AgentState) -> Literal["tools", "__end__"]:185 if state['agent_call_cnt'] > 5:186 return "__end__"187 else:188 return tools_condition(state)189 190 builder.add_conditional_edges(191 "assistant",192 router,193 )194 builder.add_edge("tools", "assistant")195 196 react_graph = builder.compile()197 198 print(f"Agent received question: {question}")199 print(f"Agent received file: {file_name}")200 201 messages = [HumanMessage(content=question)]202 answers = react_graph.invoke({"messages": messages, "file_name": file_name, "agent_call_cnt": 0})['messages']203 answer_str = " ".join(ans.content for ans in answers)204 print(f"full answer:{answer_str}")205 final_answer = llm.invoke(f"""For the question {question}, summarize your answer in <{answer_str}> and rephrase it to YOUR FINAL ANSWER. ALWAYS give YOUR FINAL ANSWER to the question. When you are unsure, ALWAYS give your best educated guess.206 YOUR FINAL ANSWER need to STRICTLY meet ALL the following 4 requirements:207 1. YOUR FINAL ANSWER should be a number if possible, OR as few words as possible, OR a comma separated list of numbers and/or strings. 208 2. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. 209 3. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. 210 4. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.211 YOUR FINAL ANSWER:""").content.split('ANSWER:')[-1].strip()212 print(f"final answer:{final_answer}")213 return final_answer214 215 # fixed_answer = "This is a default answer."216 # print(f"Agent returning fixed answer: {fixed_answer}")217 # return fixed_answer218 219def run_and_submit_all(profile: gr.OAuthProfile | None):220 """221 Fetches all questions, runs the BasicAgent on them, submits all answers,222 and displays the results.223 """224 # --- Determine HF Space Runtime URL and Repo URL ---225 space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code226 227 if profile:228 username= f"{profile.username}"229 print(f"User logged in: {username}")230 else:231 print("User not logged in.")232 return "Please Login to Hugging Face with the button.", None233 234 api_url = DEFAULT_API_URL235 questions_url = f"{api_url}/questions"236 submit_url = f"{api_url}/submit"237 238 # 1. Instantiate Agent ( modify this part to create your agent)239 try:240 agent = BasicAgent()241 except Exception as e:242 print(f"Error instantiating agent: {e}")243 return f"Error initializing agent: {e}", None244 # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)245 agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"246 print(agent_code)247 248 # 2. Fetch Questions249 print(f"Fetching questions from: {questions_url}")250 try:251 response = requests.get(questions_url, timeout=15)252 response.raise_for_status()253 questions_data = response.json()254 if not questions_data:255 print("Fetched questions list is empty.")256 return "Fetched questions list is empty or invalid format.", None257 print(f"Fetched {len(questions_data)} questions.")258 except requests.exceptions.RequestException as e:259 print(f"Error fetching questions: {e}")260 return f"Error fetching questions: {e}", None261 except requests.exceptions.JSONDecodeError as e:262 print(f"Error decoding JSON response from questions endpoint: {e}")263 print(f"Response text: {response.text[:500]}")264 return f"Error decoding server response for questions: {e}", None265 except Exception as e:266 print(f"An unexpected error occurred fetching questions: {e}")267 return f"An unexpected error occurred fetching questions: {e}", None268 269 # 3. Run your Agent270 results_log = []271 answers_payload = []272 print(f"Running agent on {len(questions_data)} questions...")273 for item in questions_data:274 task_id = item.get("task_id")275 question_text = item.get("question")276 if not task_id or question_text is None:277 print(f"Skipping item with missing task_id or question: {item}")278 continue279 print(f"task_id:{task_id}")280 file_name = item.get("file_name")281 if file_name != "":282 file_url = f"{api_url}/files/{task_id}"283 resp = requests.get(file_url, timeout=15)284 with open(file_name, 'wb') as f:285 f.write(resp.content)286 try:287 submitted_answer = agent(question_text, file_name)288 answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})289 results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})290 except Exception as e:291 print(f"Error running agent on task {task_id}: {e}")292 results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})293 294 if not answers_payload:295 print("Agent did not produce any answers to submit.")296 return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)297 298 # 4. Prepare Submission 299 submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}300 status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."301 print(status_update)302 303 # 5. Submit304 print(f"Submitting {len(answers_payload)} answers to: {submit_url}")305 try:306 response = requests.post(submit_url, json=submission_data, timeout=60)307 response.raise_for_status()308 result_data = response.json()309 final_status = (310 f"Submission Successful!\n"311 f"User: {result_data.get('username')}\n"312 f"Overall Score: {result_data.get('score', 'N/A')}% "313 f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"314 f"Message: {result_data.get('message', 'No message received.')}"315 )316 print("Submission successful.")317 results_df = pd.DataFrame(results_log)318 return final_status, results_df319 except requests.exceptions.HTTPError as e:320 error_detail = f"Server responded with status {e.response.status_code}."321 try:322 error_json = e.response.json()323 error_detail += f" Detail: {error_json.get('detail', e.response.text)}"324 except requests.exceptions.JSONDecodeError:325 error_detail += f" Response: {e.response.text[:500]}"326 status_message = f"Submission Failed: {error_detail}"327 print(status_message)328 results_df = pd.DataFrame(results_log)329 return status_message, results_df330 except requests.exceptions.Timeout:331 status_message = "Submission Failed: The request timed out."332 print(status_message)333 results_df = pd.DataFrame(results_log)334 return status_message, results_df335 except requests.exceptions.RequestException as e:336 status_message = f"Submission Failed: Network error - {e}"337 print(status_message)338 results_df = pd.DataFrame(results_log)339 return status_message, results_df340 except Exception as e:341 status_message = f"An unexpected error occurred during submission: {e}"342 print(status_message)343 results_df = pd.DataFrame(results_log)344 return status_message, results_df345 346 347# --- Build Gradio Interface using Blocks ---348with gr.Blocks() as demo:349 gr.Markdown("# Basic Agent Evaluation Runner")350 gr.Markdown(351 """352 **Instructions:**353 354 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...355 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.356 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.357 358 ---359 **Disclaimers:**360 Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).361 This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.362 """363 )364 365 gr.LoginButton()366 367 run_button = gr.Button("Run Evaluation & Submit All Answers")368 369 status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)370 # Removed max_rows=10 from DataFrame constructor371 results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)372 373 run_button.click(374 fn=run_and_submit_all,375 outputs=[status_output, results_table]376 )377 378if __name__ == "__main__":379 print("\n" + "-"*30 + " App Starting " + "-"*30)380 # Check for SPACE_HOST and SPACE_ID at startup for information381 space_host_startup = os.getenv("SPACE_HOST")382 space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup383 384 if space_host_startup:385 print(f"✅ SPACE_HOST found: {space_host_startup}")386 print(f" Runtime URL should be: https://{space_host_startup}.hf.space")387 else:388 print("ℹ️ SPACE_HOST environment variable not found (running locally?).")389 390 if space_id_startup: # Print repo URLs if SPACE_ID is found391 print(f"✅ SPACE_ID found: {space_id_startup}")392 print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")393 print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")394 else:395 print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")396 397 print("-"*(60 + len(" App Starting ")) + "\n")398 399 print("Launching Gradio Interface for Basic Agent Evaluation...")400 demo.launch(debug=True, share=False)