Akjava/open_Deep-Research-DuckDuckGo

1import argparse2import json3import os4import time5import threading6from concurrent.futures import ThreadPoolExecutor, as_completed7from datetime import datetime8from pathlib import Path9from typing import List, Optional10 11import datasets12import pandas as pd13from dotenv import load_dotenv14from huggingface_hub import login15import gradio as gr16 17from scripts.reformulator import prepare_response18from scripts.run_agents import (19    get_single_file_description,20    get_zip_description,21)22from scripts.text_inspector_tool import TextInspectorTool23from scripts.text_web_browser import (24    ArchiveSearchTool,25    FinderTool,26    FindNextTool,27    PageDownTool,28    PageUpTool,29    SimpleTextBrowser,30    VisitTool,31)32from scripts.visual_qa import visualizer33from tqdm import tqdm34 35from smolagents import (36    CodeAgent,37    HfApiModel,38    LiteLLMModel,39    Model,40    ToolCallingAgent,41    DuckDuckGoSearchTool42)43from smolagents.agent_types import AgentText, AgentImage, AgentAudio44from smolagents.gradio_ui import pull_messages_from_step, handle_agent_output_types45 46from smolagents import Tool47 48from huggingface_hub import InferenceClient49def hf_chat(api_key, model, text):50    client = InferenceClient(api_key=api_key)51    messages = [52        {53            "role": "user",54            "content": text,55        }56    ]57 58    stream = client.chat.completions.create(59        model=model, messages=messages, max_tokens=6000, stream=False60    )61 62    return stream.choices[0].message.content63 64AUTHORIZED_IMPORTS = [65    "requests",66    "zipfile",67    "os",68    "pandas",69    "numpy",70    "sympy",71    "json",72    "bs4",73    "pubchempy",74    "xml",75    "yahoo_finance",76    "Bio",77    "sklearn",78    "scipy",79    "pydub",80    "io",81    "PIL",82    "chess",83    "PyPDF2",84    "pptx",85    "torch",86    "datetime",87    "fractions",88    "csv",89]90load_dotenv(override=True)91#login(os.getenv("HF_TOKEN"))92 93append_answer_lock = threading.Lock()94 95custom_role_conversions = {"tool-call": "assistant", "tool-response": "user"}96 97user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0"98 99BROWSER_CONFIG = {100    "viewport_size": 1024 * 5,101    "downloads_folder": "downloads_folder",102    "request_kwargs": {103        "headers": {"User-Agent": user_agent},104        "timeout": 300,105    },106    "serpapi_key": os.getenv("SERPAPI_API_KEY"),107}108 109os.makedirs(f"./{BROWSER_CONFIG['downloads_folder']}", exist_ok=True)110 111 112 113model = HfApiModel(114max_tokens=9000,115temperature=0.5,116#model_id='meta-llama/Llama-3.3-70B-Instruct',117model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B',   118custom_role_conversions=None,119)120 121text_limit = 15000122ti_tool = TextInspectorTool(model, text_limit)123 124browser = SimpleTextBrowser(**BROWSER_CONFIG)125 126WEB_TOOLS = [127    DuckDuckGoSearchTool(),128    #GoogleSearchTool(),129    VisitTool(browser),130    PageUpTool(browser),131    PageDownTool(browser),132    FinderTool(browser),133    FindNextTool(browser),134    ArchiveSearchTool(browser),135    TextInspectorTool(model, text_limit),136]137 138# Agent creation in a factory function139def create_agent():140    """Creates a fresh agent instance for each session"""141    return CodeAgent(142        model=model,143        tools=[visualizer] + WEB_TOOLS,144        max_steps=10,145        verbosity_level=1,146        additional_authorized_imports=AUTHORIZED_IMPORTS,147        planning_interval=10,148    )149 150document_inspection_tool = TextInspectorTool(model, text_limit)151 152def stream_to_gradio(153    agent,154    task: str,155    reset_agent_memory: bool = False,156    additional_args: Optional[dict] = None,157):158    """Runs an agent with the given task and streams the messages from the agent as gradio ChatMessages."""159    for step_log in agent.run(task, stream=True, reset=reset_agent_memory, additional_args=additional_args):160        for message in pull_messages_from_step(161            step_log,162        ):163            yield message164 165 166 167    final_answer = step_log  # Last log is the run's final_answer168    final_answer = handle_agent_output_types(final_answer)169 170    if isinstance(final_answer, AgentText):171        jp=hf_chat(None,"google/gemma-2-27b-it",f"以下を日本語に翻訳して:{final_answer.to_string()}")172        173        yield gr.ChatMessage(174            role="assistant",175            content=f"**Final answer:**\n{final_answer.to_string()}\n\n**日本語訳:**\n{jp}",176        )177    elif isinstance(final_answer, AgentImage):178        yield gr.ChatMessage(179            role="assistant",180            content={"path": final_answer.to_string(), "mime_type": "image/png"},181        )182    elif isinstance(final_answer, AgentAudio):183        yield gr.ChatMessage(184            role="assistant",185            content={"path": final_answer.to_string(), "mime_type": "audio/wav"},186        )187    else:188        yield gr.ChatMessage(role="assistant", content=f"**Final answer:** {str(final_answer)}")189 190 191class GradioUI:192    """A one-line interface to launch your agent in Gradio"""193 194    def __init__(self, file_upload_folder: str | None = None):195        196        self.file_upload_folder = file_upload_folder197        if self.file_upload_folder is not None:198            if not os.path.exists(file_upload_folder):199                os.mkdir(file_upload_folder)200 201    def interact_with_agent(self, prompt, messages, session_state):202        # Get or create session-specific agent203        if 'agent' not in session_state:204            session_state['agent'] = create_agent()205            206        messages.append(gr.ChatMessage(role="user", content=prompt))207        yield messages208 209        # Use session's agent instance210        for msg in stream_to_gradio(session_state['agent'], task=prompt, reset_agent_memory=False):211            messages.append(msg)212            yield messages213        yield messages214 215    def upload_file(216        self,217        file,218        file_uploads_log,219        allowed_file_types=[220            "application/pdf",221            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",222            "text/plain",223        ],224    ):225        """226        Handle file uploads, default allowed types are .pdf, .docx, and .txt227        """228        if file is None:229            return gr.Textbox("No file uploaded", visible=True), file_uploads_log230 231        try:232            mime_type, _ = mimetypes.guess_type(file.name)233        except Exception as e:234            return gr.Textbox(f"Error: {e}", visible=True), file_uploads_log235 236        if mime_type not in allowed_file_types:237            return gr.Textbox("File type disallowed", visible=True), file_uploads_log238 239        # Sanitize file name240        original_name = os.path.basename(file.name)241        sanitized_name = re.sub(242            r"[^\w\-.]", "_", original_name243        )  # Replace any non-alphanumeric, non-dash, or non-dot characters with underscores244 245        type_to_ext = {}246        for ext, t in mimetypes.types_map.items():247            if t not in type_to_ext:248                type_to_ext[t] = ext249 250        # Ensure the extension correlates to the mime type251        sanitized_name = sanitized_name.split(".")[:-1]252        sanitized_name.append("" + type_to_ext[mime_type])253        sanitized_name = "".join(sanitized_name)254 255        # Save the uploaded file to the specified folder256        file_path = os.path.join(self.file_upload_folder, os.path.basename(sanitized_name))257        shutil.copy(file.name, file_path)258 259        return gr.Textbox(f"File uploaded: {file_path}", visible=True), file_uploads_log + [file_path]260 261    def log_user_message(self, text_input, file_uploads_log):262        return (263            text_input264            + (265                f"\nYou have been provided with these files, which might be helpful or not: {file_uploads_log}"266                if len(file_uploads_log) > 0267                else ""268            ),269            "",270        )271 272    def launch(self, **kwargs):273        with gr.Blocks(theme="ocean", fill_height=True) as demo:274            gr.Markdown("""# open Deep Research - free the AI agents!275 276DuckDuckGo + Huggingface(DeepSeek-R1-Distill-Qwen-32B) + 日本語訳(gemma2-27b-it)  Maybe Duplicate space and add your hf_token,improve response time.277 278_Built with [smolagents](https://github.com/huggingface/smolagents)_279 280OpenAI just published [Deep Research](https://openai.com/index/introducing-deep-research/), a very nice assistant that can perform deep searches on the web to answer user questions.281 282However, their agent has a huge downside: it's not open. So we've started a 24-hour rush to replicate and open-source it. Our resulting [open-Deep-Research agent](https://github.com/huggingface/smolagents/tree/main/examples/open_deep_research) took the #1 rank of any open submission on the GAIA leaderboard! ✨283 284You can try a simplified version below. 👇""")285            # Add session state to store session-specific data286            session_state = gr.State({})  # Initialize empty state for each session287            stored_messages = gr.State([])288            file_uploads_log = gr.State([])289            chatbot = gr.Chatbot(290                label="open-Deep-Research",291                type="messages",292                avatar_images=(293                    None,294                    "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/smolagents/mascot_smol.png",295                ),296                resizeable=True,297                scale=1,298            )299            # If an upload folder is provided, enable the upload feature300            if self.file_upload_folder is not None:301                upload_file = gr.File(label="Upload a file")302                upload_status = gr.Textbox(label="Upload Status", interactive=False, visible=False)303                upload_file.change(304                    self.upload_file,305                    [upload_file, file_uploads_log],306                    [upload_status, file_uploads_log],307                )308            text_input = gr.Textbox(lines=1, label="Your request")309            text_input.submit(310                self.log_user_message,311                [text_input, file_uploads_log],312                [stored_messages, text_input],313            ).then(self.interact_with_agent,314                # Include session_state in function calls315                [stored_messages, chatbot, session_state],316                [chatbot]317            )318 319        demo.launch(debug=True, share=True, **kwargs)320 321GradioUI().launch()