eadx/LocateAnything-3B-MLX

1# coding=utf-82# Copyright 2024 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""16Processor class for LocateAnything.17"""18 19import math20import os21from typing import Iterable, List, Union, Literal22import base6423import sys24import time25import warnings26from functools import lru_cache27from io import BytesIO28import re29import requests30import torch31import torchvision32from packaging import version33from PIL import Image34from torchvision import io35from torchvision import transforms36from torchvision.transforms import InterpolationMode37from typing import Optional, Any38import numpy as np39 40from transformers.feature_extraction_utils import BatchFeature41from transformers.image_utils import ImageInput42try:43    from transformers.image_utils import VideoInput44except ImportError:45    VideoInput = None46from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack47from transformers.tokenization_utils_base import PreTokenizedInput, TextInput48from transformers.utils import logging49import lmdb50import cv251import pickle52import decord53 54logger = logging.get_logger(__name__)55 56FPS = 2.057MAX_FRAMES = 64 58VIDEO_TOTAL_PIXELS = int(float(os.environ.get('VIDEO_MAX_PIXELS', 32000 * 28 * 28 * 0.9)))59logger.info(f"set VIDEO_TOTAL_PIXELS: {VIDEO_TOTAL_PIXELS}")60 61 62def to_rgb(pil_image: Image.Image) -> Image.Image:63      if pil_image.mode == 'RGBA':64          white_background = Image.new("RGB", pil_image.size, (255, 255, 255))65          white_background.paste(pil_image, mask=pil_image.split()[3])  # Use alpha channel as mask66          return white_background67      else:68          return pil_image.convert("RGB")69 70def read_img_from_lmdb_v2(image_data):71    # special case for AgiBotWorld72    lmdb_file, lmdb_key = image_data['lmdb_file'], image_data['lmdb_key']73    key = lmdb_key.encode('ascii')74    env = lmdb.open(lmdb_file, max_readers=10240, readonly=True, lock=False, readahead=False, meminit=False)75    txn = env.begin()76    value = txn.get(key)77    if value is None:78        print(f"Warning: Key {key} not found.")79        return None80    record = pickle.loads(value)81    image_bgr = cv2.imdecode(np.frombuffer(record['image'], dtype=np.uint8), cv2.IMREAD_COLOR)82    image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)83    image = Image.fromarray(image_rgb)84        85    return image86 87def parse_lmdb_image_data(image_data):88    lmdb_file = image_data['lmdb_file']89    if not os.path.exists(lmdb_file):90        if "/home/zhidingy/workspace/libs/eagle/Eagle2/" in lmdb_file:91            image_data['lmdb_file'] = lmdb_file.replace("/home/zhidingy/workspace/libs/eagle/Eagle2/", "")92        else:93            raise ValueError(f"LMDB file {lmdb_file} does not exist")94    # special case for AgiBotWorld95    if 'AgiBotWorld' in image_data['lmdb_file']:96        return read_img_from_lmdb_v2(image_data)97    98    try:99        env = lmdb.open(image_data['lmdb_file'], readonly=True, lock=False, max_readers=10240)100    except Exception as e:101        print(f"Failed to open lmdb file {image_data['lmdb_file']}. Error message: {e}", flush=True)102        raise e103 104    with env.begin(write=False) as txn:105        try:106            image_bin = txn.get(image_data['lmdb_key'].encode('ascii'))107            buf = BytesIO(image_bin)108        except Exception as e:109            print(f"Failed to get image from lmdb file {image_data['lmdb_file']}. Error message: {e}", flush=True)110            raise e111    try:112        image = Image.open(buf)113    except Exception as e:114        image_np = np.frombuffer(image_bin, dtype=np.uint8)115        image_bgr = cv2.imdecode(image_np, cv2.IMREAD_COLOR)116        image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)117        image = Image.fromarray(image_rgb)118    return image119 120def fetch_image(ele: dict[str, str | Image.Image]) -> Image.Image:121    if "image" in ele:122        image = ele["image"]123    else:124        image = ele["image_url"]125    image_obj = None126    if isinstance(image, Image.Image):127        image_obj = image128    elif isinstance(image, dict) and 'lmdb_file' in image:129        image_obj = parse_lmdb_image_data(image)130    elif image.startswith("http://") or image.startswith("https://"):131        response = requests.get(image, stream=True)132        image_obj = Image.open(BytesIO(response.content))133    elif image.startswith("file://"):134        image_obj = Image.open(image[7:])135    elif image.startswith("data:image"):136        if "base64," in image:137            _, base64_data = image.split("base64,", 1)138            data = base64.b64decode(base64_data)139            image_obj = Image.open(BytesIO(data))140    else:141        image_obj = Image.open(image)142    if image_obj is None:143        raise ValueError(f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}")144    image = to_rgb(image_obj)145 146    return image147 148 149def get_video_frame_indices(150    ele: dict,151    total_frames: int,152    video_fps: int | float,153) -> tuple[torch.Tensor, float]:154    target_fps = ele.get("fps", FPS)155    max_frames = ele.get("max_frames", MAX_FRAMES)156    157    nframes = (total_frames / video_fps) * target_fps158    nframes = int(round(nframes))159    nframes = max(1, nframes)160    161    if nframes > max_frames:162        nframes = max_frames163    164    nframes = min(nframes, total_frames)165    166    if nframes == total_frames:167        idx = torch.arange(total_frames).long()168    else:169        idx = torch.linspace(0, total_frames - 1, nframes).round().long()170        171    sample_fps = nframes / max(total_frames, 1e-6) * video_fps172    173    return idx, sample_fps174 175def _read_video_torchvision(176    ele: dict,177) -> (torch.Tensor, float, list):178    """read video using torchvision.io.read_video and return also per-frame timestamps"""179    video_path = ele["video"]180    if version.parse(torchvision.__version__) < version.parse("0.19.0"):181        if "http://" in video_path or "https://" in video_path:182            warnings.warn("torchvision < 0.19.0 does not support http/https video path, please upgrade to 0.19.0.")183        if "file://" in video_path:184            video_path = video_path[7:]185    st = time.time()186    187    video, audio, info = io.read_video(188        video_path,189        start_pts=ele.get("video_start", 0.0),190        end_pts=ele.get("video_end", None),191        pts_unit="sec",192        output_format="TCHW",193    )194    total_frames, video_fps = video.size(0), info["video_fps"]195    logger.info(f"torchvision:  {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s")196    197    idx, sample_fps = get_video_frame_indices(ele, total_frames, video_fps)198    199    start_time = ele.get("video_start", 0.0)200    timestamps = (start_time + idx.to(torch.float32) / video_fps).tolist()201    202    video = video[idx]203    return video, sample_fps, timestamps204 205 206def is_decord_available() -> bool:207    import importlib.util208    return importlib.util.find_spec("decord") is not None209 210def _read_video_decord(211    ele: dict,212) -> (torch.Tensor, float, list):213    """read video using decord.VideoReader and return also per-frame timestamps"""214    video_path = ele["video"]215    st = time.time()216    vr = decord.VideoReader(video_path)217    218    total_frames, video_fps = len(vr), vr.get_avg_fps()219    logger.info(f"decord:  {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s")220    221    idx_tensor, sample_fps = get_video_frame_indices(ele, total_frames, video_fps)222    idx = idx_tensor.tolist()223    224    start_time = ele.get("video_start", 0.0)225    timestamps = [start_time + i / video_fps for i in idx]226    227    video = vr.get_batch(idx).asnumpy()228    video = torch.tensor(video).permute(0, 3, 1, 2)  # Convert to TCHW format229    230    return video, sample_fps, timestamps231 232 233VIDEO_READER_BACKENDS = {234    "decord": _read_video_decord,235    "torchvision": _read_video_torchvision,236}237 238 239@lru_cache(maxsize=1)240def get_video_reader_backend() -> str:241    if is_decord_available():242        video_reader_backend = "decord"243    else:244        video_reader_backend = "torchvision"245    return video_reader_backend246 247 248def fetch_video(ele: dict, return_video_sample_fps: bool = False, video_reader_backend: str = "torchvision") -> torch.Tensor | list[Image.Image]:249    """250    Fetches video, samples frames, resizes based on video_total_pixels, and returns as Tensor (TCHW).251    """252    if isinstance(ele["video"], str):253        video_reader_backend = video_reader_backend if video_reader_backend is not None else get_video_reader_backend()254        try:255            video, sample_fps, timestamps = VIDEO_READER_BACKENDS[video_reader_backend](ele)256        except Exception as e:257            logger.warning(f"video_reader_backend {video_reader_backend} error, use torchvision as default, msg: {e}")258            video, sample_fps, timestamps = VIDEO_READER_BACKENDS["torchvision"](ele)259 260        nframes, _, height, width = video.shape261        262        video_total_pixels = ele.get("video_total_pixels", VIDEO_TOTAL_PIXELS)263        current_pixels = nframes * height * width264        265        if current_pixels > video_total_pixels:266            scale_factor = math.sqrt(video_total_pixels / current_pixels)267            new_height = int(height * scale_factor)268            new_width = int(width * scale_factor)269            270            video = transforms.functional.resize(271                video,272                [new_height, new_width],273                interpolation=InterpolationMode.BICUBIC,274                antialias=True,275            ).float()276        else:277            video = video.float()278 279        if return_video_sample_fps:280            return video, sample_fps, timestamps281        return video282 283    else:284        assert isinstance(ele["video"], (list, tuple))285        process_info = ele.copy()286        process_info.pop("type", None)287        process_info.pop("video", None)288        289        images = [290            fetch_image({"image": video_element, **process_info})291            for video_element in ele["video"]292        ]293        294        nframes = len(images)295        timestamps = [-1 for i in range(nframes)] 296        297        # For list of images, we return list of PIL images directly, 298        # the processor will handle conversion to tensor later.299        if return_video_sample_fps:300            return images, process_info.get("fps", 2.0), timestamps301        return images302 303class LocateAnythingProcessorKwargs(ProcessingKwargs, total=False):304    _defaults = {305        "text_kwargs": {306            "padding": False,307        },308        "images_kwargs": {},309        "videos_kwargs": {},310    }311 312 313class LocateAnythingProcessor(ProcessorMixin):314    attributes = ["image_processor", "tokenizer"]315    valid_kwargs = [316        "chat_template",317        "num_image_tokens",318        "image_token",319        "video_token",320        "images_kwargs",321        "videos_kwargs",322        "text_kwargs",323    ]324    image_processor_class = "AutoImageProcessor" 325    tokenizer_class = "AutoTokenizer"326 327    def __init__(328        self,329        image_processor=None,330        tokenizer=None,331        chat_template=None,332        image_token='<IMG_CONTEXT>',333        video_token='<IMG_CONTEXT>',334        merge_kernel_size=[2, 2], # Note: This might need adjustment based on your patch_size (14*14)335        image_placeholder='image',336        video_placeholder='video',337        image_start_token='<img>',338        image_end_token='</img>',339        **kwargs,340    ):  341        self.image_token = tokenizer.image_token if hasattr(tokenizer, "image_token") else image_token342        self.video_token = tokenizer.video_token if hasattr(tokenizer, "video_token") else video_token343        self.image_token_id = (344            tokenizer.image_token_id345            if getattr(tokenizer, "image_token_id", None)346            else tokenizer.convert_tokens_to_ids(self.image_token)347        )348        self.video_token_id = (349            tokenizer.video_token_id350            if getattr(tokenizer, "video_token_id", None)351            else tokenizer.convert_tokens_to_ids(self.video_token)352        )353        self.image_placeholder = image_placeholder354        self.video_placeholder = video_placeholder355        self.merge_kernel_size = merge_kernel_size356        self.image_start_token = image_start_token357        self.image_end_token = image_end_token358        if 'auto_map' in kwargs:359            self.auto_map = kwargs['auto_map']360        super().__init__(image_processor, tokenizer, chat_template=chat_template)361 362    363    def replace_media_placeholder(self, text, image_list, video_list, timestamps_list, fps_list, **output_kwargs):364 365        num_of_images_in_this_sample = 0366        num_of_videos_in_this_sample = 0367        pattern = re.compile(rf"<({self.image_placeholder}|{self.video_placeholder})-(\d+)>")368        unified_frame_list = []369        370        def replace_in_text(text):371            def repl(match):372                nonlocal unified_frame_list373                nonlocal num_of_images_in_this_sample374                nonlocal num_of_videos_in_this_sample375                media_type = match.group(1)376                idx_in_list = int(match.group(2)) - 1377                idx_mapper = {0: "first", 1: "second", 2: "third", 3: "fourth", 4: "fifth", 5: "sixth", 6: "seventh", 7: "eighth", 8: "ninth", 9: "tenth"}  378                379                if media_type == 'image':380                    # Call LocateAnythingImageProcessor with a single image in a list381                    image_inputs = self.image_processor(images=[image_list[idx_in_list]], **output_kwargs["images_kwargs"])382                    383                    num_of_tokens_list = [int(h * w) // (self.image_processor.merge_kernel_size[0] * self.image_processor.merge_kernel_size[1]) for h, w in image_inputs['image_grid_hws']]384                    385                    special_placeholder = f"<image {idx_in_list+1}>{self.image_start_token}{self.image_token * num_of_tokens_list[0]}{self.image_end_token}"386                    unified_frame_list.append(image_inputs)387                    num_of_images_in_this_sample += 1388                    389                elif media_type == 'video':390                    video_obj = video_list[idx_in_list]391                    392                    # Convert Tensor TCHW to list of PIL Images for the ImageProcessor393                    if isinstance(video_obj, torch.Tensor):394                        # video_obj is [T, C, H, W], float, likely 0-255 or standardized395                        # LocateAnythingImageProcessor expects PIL or 0-255 inputs usually.396                        # We need to convert back to PIL or List[Tensor] compatible with make_list_of_images397                        video_frames = []398                        for i in range(video_obj.shape[0]):399                            frame = video_obj[i] # [C, H, W]400                            # Assuming fetch_video returns float tensors.401                            # If they are 0-255, convert to uint8.402                            if frame.dtype.is_floating_point and frame.max() > 1.0:403                                frame = frame.byte()404                            elif frame.dtype.is_floating_point:405                                frame = (frame * 255).byte()406                            407                            img = transforms.ToPILImage()(frame)408                            video_frames.append(img)409                    elif isinstance(video_obj, list):410                        # Already list of PIL images411                        video_frames = video_obj412                    else:413                        raise ValueError("Unsupported video format")414 415                    # Call ImageProcessor with list of frames416                    video_inputs = self.image_processor(images=video_frames, **output_kwargs["videos_kwargs"])417                    418                    # Calculate tokens per frame419                    num_of_tokens_list = [int(h * w) // (self.image_processor.merge_kernel_size[0] * self.image_processor.merge_kernel_size[1]) for h, w in video_inputs['image_grid_hws']]420                    421                    if timestamps_list is not None and -1 not in timestamps_list:422                        frame_timestamps = timestamps_list[idx_in_list]423                    else:424                        frame_timestamps = None425                    sampled_fps = fps_list[idx_in_list] if fps_list is not None else None426                    427                    if frame_timestamps is not None:428                        # Ensure lengths match (sometimes rounding might cause off-by-one if not careful, but usually safe here)429                        if len(frame_timestamps) != len(num_of_tokens_list):430                             logger.warning(f"Timestamp mismatch: {len(frame_timestamps)} vs {len(num_of_tokens_list)}")431                             min_len = min(len(frame_timestamps), len(num_of_tokens_list))432                             frame_timestamps = frame_timestamps[:min_len]433                             num_of_tokens_list = num_of_tokens_list[:min_len]434 435                        special_placeholder = [f"Frame-{i+1}-{frame_timestamps[i]:.2f}s: {self.image_start_token}{self.image_token * num_of_tokens}{self.image_end_token}" for i, num_of_tokens in enumerate(num_of_tokens_list)]436                    else:437                        special_placeholder = [f"Frame-{i+1}: {self.image_start_token}{self.image_token * num_of_tokens}{self.image_end_token}" for i, num_of_tokens in enumerate(num_of_tokens_list)]438                    439                    if sampled_fps is not None:440                        special_placeholder = f"The {idx_mapper[idx_in_list]} video sampled with {sampled_fps:.2f} fps: " + "".join(special_placeholder)441                    else:442                        special_placeholder = f"The {idx_mapper[idx_in_list]} video: " + "".join(special_placeholder)443                    444                    unified_frame_list.append(video_inputs)445                    num_of_videos_in_this_sample += 1446                else:447                    raise ValueError(f'Unknown media type: {media_type}')448                return special_placeholder449            return pattern.sub(repl, text)450        451        text = replace_in_text(text)452        453        if len(unified_frame_list) > 0:454            # Concatenate all pixel values from all images/videos in this sample455            pvs = []456            for frame in unified_frame_list:457                pv = frame['pixel_values']458                if isinstance(pv, np.ndarray):459                    pv = torch.from_numpy(pv)460                pvs.append(pv)461            pixel_values = torch.cat(pvs, dim=0)462            # Concatenate grid hws463            image_grid_hws = np.concatenate([frame['image_grid_hws'] for frame in unified_frame_list], axis=0)464        else:465            pixel_values = torch.empty(0)466            image_grid_hws = np.empty(0)467            468        return text, pixel_values, image_grid_hws, num_of_images_in_this_sample, num_of_videos_in_this_sample469    470    def __call__(471        self,472        images: ImageInput = None,473        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,474        audio=None,475        videos: VideoInput = None,476        **kwargs: Unpack[LocateAnythingProcessorKwargs],477    ) -> BatchFeature:478        output_kwargs = self._merge_kwargs(479            LocateAnythingProcessorKwargs,480            tokenizer_init_kwargs=self.tokenizer.init_kwargs,481            **kwargs,482        )483        484        if isinstance(text, str):485            text_list = [text]486        elif not isinstance(text, list) and not isinstance(text[0], str):487            raise ValueError("Invalid input text. Please provide a string, or a list of strings")488        elif isinstance(text, list) and isinstance(text[0], str):489            text_list = text490        491        if images is None: images = []492        if videos is None: videos = []493        494        pixel_values_list = []495        image_grid_hws_list = []496        new_sample_list = []497        image_start_idx = 0498        video_start_idx = 0499        timestamps_batch = output_kwargs['videos_kwargs'].pop("timestamps", None)500        fps_batch = output_kwargs['videos_kwargs'].pop("fps", None)501        502        for sample in text_list:503            timestamps_list = timestamps_batch[video_start_idx:] if timestamps_batch is not None else None504            fps_list = fps_batch[video_start_idx:] if fps_batch is not None else None505            506            sample, pixel_values, image_grid_hws, num_of_images_in_this_sample, num_of_videos_in_this_sample = self.replace_media_placeholder(507                sample, images[image_start_idx:], videos[video_start_idx:], timestamps_list, fps_list, **output_kwargs508            )509            new_sample_list.append(sample)510            511            if pixel_values.numel() > 0:512                pixel_values_list.append(pixel_values)513                image_grid_hws_list.append(image_grid_hws)514 515            image_start_idx += num_of_images_in_this_sample516            video_start_idx += num_of_videos_in_this_sample517 518        image_inputs = {}519        if len(pixel_values_list) > 0:520            # Concatenate across the batch521            pts = [torch.from_numpy(pv) if isinstance(pv, np.ndarray) else pv for pv in pixel_values_list]522            image_inputs['pixel_values'] = torch.cat(pts, dim=0)523            image_inputs['image_grid_hws'] = np.concatenate(image_grid_hws_list, axis=0)524        525        video_inputs = {} # Video data is merged into image_inputs now526        text_inputs = self.tokenizer(new_sample_list, **output_kwargs["text_kwargs"])527        528        return BatchFeature(data={**text_inputs, **image_inputs, **video_inputs})529 530    def batch_decode(self, *args, **kwargs):531        return self.tokenizer.batch_decode(*args, **kwargs)532 533    def decode(self, *args, **kwargs):534        return self.tokenizer.decode(*args, **kwargs)535 536    @property537    def model_input_names(self):538        tokenizer_input_names = self.tokenizer.model_input_names539        image_processor_input_names = self.image_processor.model_input_names540        return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))541 542    def save_pretrained(self, save_directory, **kwargs):543        if os.path.isfile(save_directory):544            raise ValueError(f"Provided path ({save_directory}) should be a directory, not a file")545        os.makedirs(save_directory, exist_ok=True)546        outputs = super().save_pretrained(save_directory, **kwargs)547        return outputs548 549    @classmethod550    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):551        processor = super().from_pretrained(pretrained_model_name_or_path, **kwargs)552        if isinstance(processor, tuple):553            processor = processor[0]554        return processor555 556    def process_vision_info(557        self,558        conversations: list[dict] | list[list[dict]],559        return_video_kwargs: bool = False,560        video_reader_backend: str = "torchvision",561    ) -> tuple[list[Image.Image] | None, list[torch.Tensor | list[Image.Image]] | None, Optional[dict]]:562 563        vision_infos = self.extract_vision_info(conversations)564        image_inputs = []565        video_inputs = []566        video_sample_fps_list = []567        video_timestamps_list = []568        569        for vision_info in vision_infos:570            if "image" in vision_info or "image_url" in vision_info:571                image_inputs.append(fetch_image(vision_info))572            elif "video" in vision_info:573                video_input, video_sample_fps, video_timestamps = fetch_video(vision_info, return_video_sample_fps=True, video_reader_backend=video_reader_backend)574                video_sample_fps_list.append(video_sample_fps)575                video_inputs.append(video_input)576                video_timestamps_list.append(video_timestamps)577            else:578                raise ValueError("image, image_url or video should in content.")579        580        if len(image_inputs) == 0:581            image_inputs = None582        if len(video_inputs) == 0:583            video_inputs = None584            585        if return_video_kwargs:586            return image_inputs, video_inputs, {'fps': video_sample_fps_list, 'timestamps': video_timestamps_list}587        return image_inputs, video_inputs588 589    def extract_vision_info(self, conversations: list[dict] | list[list[dict]]) -> list[dict]:590        vision_infos = []591        if isinstance(conversations[0], dict):592            conversations = [conversations]593        for conversation in conversations:594            for message in conversation:595                if isinstance(message["content"], list):596                    for ele in message["content"]:597                        if (598                            "image" in ele599                            or "image_url" in ele600                            or "video" in ele601                            or ele["type"] in ("image", "image_url", "video")602                        ):603                            vision_infos.append(ele)604        return vision_infos605    606    def py_apply_chat_template(self, messages, tokenize=False, add_generation_prompt=False):607        assert tokenize == False, "tokenize is not supported yet"608        result = ""609        image_count = 0610        video_count = 0611        612        message_text = ""613        for idx, message in enumerate(messages):614            if message.get('role') != 'user': continue615            content = message.get('content')616            if isinstance(content, str):617                message_text += content618            elif isinstance(content, list):619                for item in content:620                    if isinstance(item, dict) and "text" in item:621                        message_text += item["text"]622                    elif isinstance(item, str):623                        message_text += item624                        625        for idx, message in enumerate(messages):626            if idx == 0 and message.get('role') != 'system':627                result += "<|im_start|>system\n"628                result += "You are a helpful assistant.\n"629                result += "<|im_end|>\n"630 631            result += f"<|im_start|>{message.get('role', '')}\n"632            content = message.get('content')633 634            if isinstance(content, str):635                result += content636                result += "<|im_end|>\n"637            else:638                for item in content:639                    if (isinstance(item, dict) and (item.get('type') == 'image' or 'image' in item or 'image_url' in item)):640                        image_count += 1641                        candidate_token = f"<image-{image_count}>"642                        if candidate_token not in message_text:643                            result += candidate_token644                    elif (isinstance(item, dict) and (item.get('type') == 'video' or 'video' in item)):645                        video_count += 1646                        candidate_token = f"<video-{video_count}>"647                        if candidate_token not in message_text:648                            result += candidate_token649                    elif isinstance(item, dict) and 'text' in item:650                        result += item['text']651                    elif isinstance(item, str):652                        result += item653                result += "<|im_end|>\n"654 655        if add_generation_prompt:656            result += "<|im_start|>assistant\n"657 658        return result659 660 661    @classmethod662    def from_args_and_dict(cls, args, processor_dict: dict[str, Any], **kwargs):663        processor_dict = processor_dict.copy()664        return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)665 666        if "processor_class" in processor_dict:667            del processor_dict["processor_class"]668 669        for key in ["image_processor", "tokenizer"]:670            processor_dict.pop(key, None)671 672        unused_kwargs = cls.validate_init_kwargs(processor_config=processor_dict, valid_kwargs=cls.valid_kwargs)673        processor = cls(*args, **processor_dict)674 675        for key in set(kwargs.keys()):676            if hasattr(processor, key):677                setattr(processor, key, kwargs.pop(key))678 679        if isinstance(unused_kwargs, dict):680            kwargs.update(unused_kwargs)681        logger.info(f"Processor {processor}")682        if return_unused_kwargs:683            return processor, kwargs684        else:685            return processor686        687        688__all__ = ["LocateAnythingProcessor"]