Pacama95/chatbot_agent

1from pydantic import BaseModel, Field2from langchain.tools import BaseTool3from PIL import Image4from transformers import pipeline5import requests6from io import BytesIO7 8from typing import Type, Final, Any9 10VISUAL_REASONING_MODEL: Final[str] = 'Salesforce/blip-vqa-capfilt-large'11 12class SimpleImageQuestionAnsweringInput(BaseModel):13    query: str = Field(description='Your SIMPLE question about the image')14    image_url: str = Field(description='URL to the image.')15 16class SimpleImageQuestionAnsweringTool(BaseTool):17 18    name: str = "simple_image_question_answering_tool"19    description: str = """20        Use this tool when asked to answer simple questions about images.21        This tool is designed to perform basic visual question answering on images accessible via a public URL. It supports simple, factual questions about the content of the image, such as:22            - Counting objects (e.g., “How many cars are in the image?”)23            - Identifying colors (e.g., “What is the color of the bird?”)24            - Recognizing general objects or scenes (e.g., “Is there a tree in the picture?”)25        It is not intended for complex reasoning, OCR (text reading), facial recognition, or detailed scene interpretation.26        It can only answer questions that can be answered directly from the visual content of the image.27        It requires a valid and accessible image URL.28    """29    args_schema: Type[BaseModel] = SimpleImageQuestionAnsweringInput30 31    def __init__(self, **kwargs: Any) -> None:32        super().__init__()33 34    def _run(self, image_url: str, query: str) -> str:35        """36            Perform basic visual question answering on images accessible via a public URL.37            Args:38                image_url (str): The URL to the image to analyze.39                query (str): Your question about the provided image.40 41            Returns:42                str: An answer for the query based on the image content43        """44        vqa_pipeline = pipeline("visual-question-answering", model=VISUAL_REASONING_MODEL, use_fast=True)45 46        # Load image from URL47        response = requests.get(image_url)48        image = Image.open(BytesIO(response.content))49 50        image_response = vqa_pipeline(image, query, top_k=1)51        52        return image_response[0]['answer']53