Pacama95/chatbot_agent

1from pydantic import BaseModel, Field2from langchain.schema import SystemMessage3from langchain.tools import BaseTool4from langchain_core.messages import SystemMessage, HumanMessage5from langchain_openai import ChatOpenAI6 7from typing import Type, Final, Any8    9 10DEFAULT_MULTIMODAL_LLM: Final[str] = 'gpt-4o-mini'11 12MULTIMODAL_LLM: Final[str] = 'multimodal_llm'13 14class ImageAnalyzerInput(BaseModel):15    image_url: str = Field(description='URL to the image to analyze.')16 17class ImageAnalyzer(BaseTool):18 19    name: str = "image_analyzer"20    description: str = (21        "Given an image URL it returns a general description of what appears in the image."22    )23    args_schema: Type[BaseModel] = ImageAnalyzerInput24 25    def __init__(self, **kwargs: Any) -> None:26        super().__init__()27 28        if MULTIMODAL_LLM in kwargs:29            self._vllm = kwargs[MULTIMODAL_LLM]30        else:31            self._vllm = ChatOpenAI(model=DEFAULT_MULTIMODAL_LLM)32 33    def _run(self, image_url: str) -> str:34        """35            Given the URL to an image, it return a general description of what appears in it.36            Args:37                image_url (str): The URL to the image to analyze38 39            Returns:40                str: An answer for the query based on the image content41        """42        system_prompt = SystemMessage(content="""43            You are a highly capable multimodal language model specialized in visual analysis. Your task is to interpret and analyze the provided image. Carefully consider the visual content to deliver a clear, accurate, and relevant response.44            Always prioritize factual and useful insights based on the image, and avoid making assumptions beyond what is visible.45        """)46        query = 'Describe what you see in the provided image'47        user_message = HumanMessage(48            content=[49                {"type": "text", "text": query},50                {51                    "type": "image_url",52                    "image_url": {"url": image_url}53                },54            ],55        )56 57        response = self._vllm.invoke([system_prompt, user_message])58 59        return response60