cbipok/VideoLLaMA3-2B-fork
033
1"""Processor class for VideoLLaMA3."""2 3import copy4import importlib.util5import os6import os.path as osp7import warnings8from collections import defaultdict9from typing import Any, List, Union, Dict, Optional, Tuple, TypedDict10 11import cv212import ffmpeg13import imageio14import json15import numpy as np16import torch17import transformers18from decord import VideoReader, cpu19from PIL import Image20from transformers.feature_extraction_utils import BatchFeature21from transformers.image_utils import ImageInput22from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack23from transformers.tokenization_utils_base import PreTokenizedInput, TextInput24 25try:26 from . import image_processing_videollama327 from .image_processing_videollama3 import (28 is_valid_image, is_valid_video,29 )30except ModuleNotFoundError:31 spec = importlib.util.spec_from_file_location(32 "image_processing_videollama3",33 osp.join(osp.dirname(__file__), "image_processing_videollama3.py"),34 )35 image_processing_videollama3 = importlib.util.module_from_spec(spec)36 spec.loader.exec_module(image_processing_videollama3)37 is_valid_image = getattr(image_processing_videollama3, "is_valid_image")38 is_valid_video = getattr(image_processing_videollama3, "is_valid_video")39 40# constants41DEFAULT_IMAGE_TOKEN = "<image>"42IGNORE_INDEX = -10043 44# Type aliases45Conversation = List[Dict[str, Any]]46SingleImage = Union[Image.Image, np.ndarray, torch.Tensor]47SingleVideo = Union[List[SingleImage], np.ndarray, torch.Tensor]48BatchedImage = List[Union[SingleImage, SingleVideo]]49BatchedNamedImage = List[Tuple[str, Union[SingleImage, SingleVideo]]]50 51 52def _custom_import(class_name: str):53 try:54 attribute_class = getattr(transformers, class_name)55 except AttributeError:56 attribute_class = getattr(image_processing_videollama3, class_name)57 return attribute_class58 59 60def is_named_image(image) -> bool:61 return isinstance(image, (list, tuple)) and \62 len(image) == 2 and \63 isinstance(image[0], str) and \64 image[0] in ["image", "video"] and \65 (is_valid_image(image[1]) or is_valid_video(image[1]))66 67 68def make_batched_images(images) -> List[List[ImageInput]]:69 if isinstance(images, (list, tuple)) and all(is_named_image(image) for image in images):70 # list of named images71 return [image[0] for image in images], [image[1] for image in images]72 elif isinstance(images, (list, tuple)) and all(is_valid_image(image) or is_valid_video(image) for image in images):73 # list of images/videos74 batch = []75 for image in images:76 if is_valid_video(image):77 batch.append(("video", image))78 elif is_valid_image(image):79 batch.append(("image", image))80 else:81 raise ValueError(f"Could not make batched images from {images}")82 return [x[0] for x in batch], [x[1] for x in batch]83 elif is_named_image(images):84 # named images85 return [images[0]], [image[1]]86 elif is_valid_video(images):87 # single video88 return ["video"], [images]89 elif is_valid_image(images):90 # single image91 return ["image"], [images]92 93 raise ValueError(f"Could not make batched images from {images}")94 95 96def frame_sample(duration, mode='uniform', num_frames=None, vid_fps=None, fps=None):97 if mode == 'uniform':98 assert num_frames is not None, "Number of frames must be provided for uniform sampling."99 if duration <= num_frames:100 return np.arange(duration).astype(int)101 # NOTE: v1 version102 # Calculate the size of each segment from which a frame will be extracted103 # if duration <= num_frames:104 # return np.arange(duration).astype(int)105 # seg_size = float(duration - 1) / num_frames106 107 # frame_ids = []108 # for i in range(num_frames):109 # # Calculate the start and end indices of each segment110 # start = seg_size * i111 # end = seg_size * (i + 1)112 # # Append the middle index of the segment to the list113 # frame_ids.append((start + end) / 2)114 115 # return np.round(np.array(frame_ids) + 1e-6).astype(int)116 # NOTE: v0 version117 return np.linspace(0, duration-1, num_frames, dtype=int)118 elif mode == 'fps':119 assert vid_fps is not None, "FPS must be provided for FPS sampling."120 assert fps is not None, "FPS must be provided for FPS sampling."121 segment_len = min(vid_fps // fps, duration)122 return np.arange(segment_len // 2, duration, segment_len, dtype=int)123 else:124 raise ImportError(f'Unsupported frame sampling mode: {mode}')125 126 127def load_video_from_ids(video_path, s=None, e=None, fps=None, max_frames=128, temporal_factor=1):128 if s is not None and e is not None:129 s = s if s >= 0. else 0.130 e = e if e >= 0. else 0.131 if s > e:132 s, e = e, s133 elif s == e:134 e = s + 1135 136 # 1. Loading Video137 if os.path.isdir(video_path):138 frame_files = sorted(os.listdir(video_path))139 140 vid_fps = 3141 num_frames_of_video = len(frame_files)142 elif video_path.endswith('.gif'):143 gif_reader = imageio.get_reader(video_path)144 145 vid_fps = 25146 num_frames_of_video = len(gif_reader)147 else:148 vreader = VideoReader(video_path, ctx=cpu(0), num_threads=2)149 # vreader = VideoReader(video_path, ctx=cpu(0), num_threads=1)150 151 vid_fps = vreader.get_avg_fps()152 num_frames_of_video = len(vreader)153 154 # 2. Determine frame range & Calculate frame indices155 f_start = 0 if s is None else max(int(s * vid_fps) - 1, 0)156 f_end = num_frames_of_video - 1 if e is None else min(int(e * vid_fps) - 1, num_frames_of_video - 1)157 frame_indices = list(range(f_start, f_end + 1))158 159 duration = len(frame_indices)160 # 3. Sampling frame indices161 if fps is not None and duration / vid_fps < max_frames:162 sampled_frame_indices = [frame_indices[i] for i in frame_sample(duration, mode='fps', vid_fps=vid_fps, fps=fps)]163 else:164 sampled_frame_indices = [frame_indices[i] for i in frame_sample(duration, mode='uniform', num_frames=max_frames)]165 166 # 4. Acquire frame data167 if os.path.isdir(video_path):168 frames = np.array([cv2.cvtColor(cv2.imread(os.path.join(video_path, frame_files[frame_idx])), cv2.COLOR_BGR2RGB) for frame_idx in sampled_frame_indices])169 elif video_path.endswith('.gif'):170 frames = np.array([cv2.cvtColor(frame, cv2.COLOR_RGBA2RGB) for idx, frame in enumerate(gif_reader) if idx in sampled_frame_indices])171 else:172 frames = vreader.get_batch(sampled_frame_indices).asnumpy()173 174 frames = frames.transpose(0, 3, 1, 2)175 timestamps = [x / vid_fps for x in sampled_frame_indices]176 177 if temporal_factor > 1:178 pad_length = temporal_factor - len(frames) % temporal_factor179 frames = np.concatenate([frames, frames[-1:].repeat(pad_length, axis=0)])180 [timestamps.append(timestamps[-1] + 1 / fps) for _ in range(pad_length)]181 182 frames = [frame for frame in frames]183 184 return frames, timestamps185 186 187class ChatTemplateKwargs(TypedDict, total=False):188 189 chat_template: Optional[str]190 add_system_prompt: Optional[bool]191 add_generation_prompt: Optional[bool]192 193 194class Videollama3Qwen2ProcessorKwargs(ProcessingKwargs, ChatTemplateKwargs, total=False):195 196 chat_template_kwargs: ChatTemplateKwargs = {197 **ChatTemplateKwargs.__annotations__,198 }199 200 _defaults = {201 "text_kwargs": {202 "padding": False,203 },204 "image_kwargs": {205 "merge_size": None,206 },207 "chat_template_kwargs": {208 "chat_template": None,209 "add_system_prompt": False,210 "add_generation_prompt": False,211 },212 }213 214 215class Videollama3Qwen2Processor(ProcessorMixin):216 217 attributes = ["image_processor", "tokenizer"]218 image_processor_class = "Videollama3ImageProcessor"219 tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")220 valid_kwargs = ["chat_template", "image_merge_size", "video_merge_size", "fps", "max_frames"]221 222 def __init__(223 self,224 image_processor=None,225 tokenizer=None,226 chat_template: str = None,227 image_merge_size: int = 1,228 video_merge_size: int = 2,229 fps: Optional[int] = 1,230 max_frames: Optional[int] = 128,231 ):232 self.image_processor = image_processor233 self.tokenizer = tokenizer234 if chat_template is None:235 chat_template = self.tokenizer.chat_template236 self.chat_template = chat_template237 238 self.image_merge_size = image_merge_size239 self.video_merge_size = video_merge_size240 self.fps = fps241 self.max_frames = max_frames242 243 self.generation_prompt = self._infer_generation_prompt()244 self.generation_prompt_ids = self.tokenizer.encode(self.generation_prompt, return_tensors="pt")245 self.generation_prompt_length = len(self.generation_prompt_ids[0])246 self.image_token_id = self.tokenizer.convert_tokens_to_ids(DEFAULT_IMAGE_TOKEN)247 self.eos_token_id = self.tokenizer.eos_token_id248 249 @classmethod250 def _get_arguments_from_pretrained(cls, pretrained_model_name_or_path, **kwargs):251 args = []252 for attribute_name in cls.attributes:253 class_name = getattr(cls, f"{attribute_name}_class")254 if isinstance(class_name, tuple):255 classes = tuple(_custom_import(n) if n is not None else None for n in class_name)256 use_fast = kwargs.get("use_fast", True)257 if use_fast and classes[1] is not None:258 attribute_class = classes[1]259 else:260 attribute_class = classes[0]261 else:262 attribute_class = _custom_import(class_name)263 264 args.append(attribute_class.from_pretrained(pretrained_model_name_or_path, **kwargs))265 return args266 267 def get_generation_prompt(self):268 return self.generation_prompt269 270 def get_generation_prompt_ids(self):271 return self.generation_prompt_ids272 273 def _infer_generation_prompt(self):274 pseudo_message = [{"role": "user", "content": ""}]275 instruction = self.apply_chat_template(pseudo_message, tokenize=False, add_generation_prompt=True)276 conversation = self.apply_chat_template(pseudo_message, tokenize=False, add_generation_prompt=False)277 return instruction.replace(conversation, "")278 279 def _get_downsampled_grid_sizes(self, image_inputs: Dict[str, Any]):280 grid_sizes = []281 for grid_size, merge_size in zip(image_inputs.get("grid_sizes", []), image_inputs.get("merge_sizes", [])):282 if not torch.all(grid_size[1:] % merge_size == 0):283 warnings.warn(f"Grid size {grid_size} is not divisible by merge size. Some undesired errors may occur.")284 if grid_size[0] == 1:285 grid_sizes.append(grid_size[1:] / merge_size)286 elif grid_size[0] > 1:287 grid_sizes.extend([grid_size[1:] / merge_size] * grid_size[0])288 return grid_sizes289 290 def _get_visual_seq_len(self, grid_size: torch.Tensor):291 num_tokens = int(grid_size.prod().item())292 return num_tokens293 294 def load_images(self, image_path: Union[str, List[str], Image.Image, List[Image.Image]]):295 def load_single_image(image_path):296 if isinstance(image_path, str) and os.path.isfile(image_path):297 # images = [cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)]298 images = Image.open(image_path).convert('RGB')299 elif isinstance(image_path, str) and image_path.startswith("http://") or image_path.startswith("https://"):300 images = Image.open(requests.get(image_path, stream=True).raw)301 elif isinstance(image_path, Image.Image):302 images = np.array(image_path)303 else:304 raise ValueError(f"Unsupported image path type: {type(image_path)}")305 return images306 307 try:308 if isinstance(image_path, list):309 images = [load_single_image(f) for f in image_path]310 elif isinstance(image_path, str) and os.path.isdir(image_path):311 images = [Image.open(os.path.join(image_path, f)).convert('RGB') for f in sorted(os.listdir(image_path))]312 else:313 images = [load_single_image(image_path)]314 return images315 except:316 raise ValueError(f"Error when loading images: {type(image_path)}")317 318 def load_video(319 self,320 video_path: str,321 start_time: Optional[float] = None,322 end_time: Optional[float] = None,323 fps: Optional[float] = None,324 max_frames: Optional[float] = None,325 size: Optional[int] = None,326 size_divisible: int = 1,327 precise_time: bool = False,328 verbose: bool = False,329 temporal_factor: int = 1330 ):331 """332 Load and process a video file and return the frames and the timestamps of each frame.333 334 Args:335 video_path (str): Path to the video file.336 start_time (float, optional): Start time in seconds. Defaults to None.337 end_time (float, optional): End time in seconds. Defaults to None.338 fps (float, optional): Frames per second. Defaults to None.339 num_frames (float, optional): Number of frames to sample. Defaults to None.340 size (int, optional): Size of the shortest side. Defaults to None.341 size_divisible (int, optional): Size divisible by this number. Defaults to 1.342 precise_time (bool, optional): Whether to use precise time. Defaults to False.343 verbose (bool, optional): Print ffmpeg output. Defaults to False.344 345 Returns:346 frames (List[PIL.Image]): List of frames.347 timestamps (List[float]): List of timestamps.348 """349 fps = self.fps if fps is None else fps350 max_frames = self.max_frames if max_frames is None else max_frames351 352 if start_time is not None and end_time is not None and end_time - start_time < 1:353 return load_video_from_ids(video_path, start_time, end_time, fps=fps, max_frames=max_frames)354 if os.path.isdir(video_path):355 return load_video_from_ids(video_path, start_time, end_time, fps=fps, max_frames=max_frames)356 if video_path.endswith('.gif'):357 return load_video_from_ids(video_path, start_time, end_time, fps=fps, max_frames=max_frames)358 probe = ffmpeg.probe(video_path)359 duration = float(probe['format']['duration'])360 video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)361 w, h = int(video_stream['width']), int(video_stream['height'])362 363 kwargs, input_kwargs, output_kwargs = {}, {}, {}364 do_trim = start_time is not None or end_time is not None365 if start_time is not None:366 new_start_time = max(float(video_stream['start_time']), start_time)367 duration -= new_start_time - start_time368 start_time = new_start_time369 else:370 start_time = float(video_stream['start_time'])371 if end_time is not None:372 duration = min(duration, end_time - start_time)373 else:374 duration = duration375 if do_trim:376 kwargs = {'ss': start_time, 't': duration}377 if precise_time:378 output_kwargs.update(kwargs)379 else:380 input_kwargs.update(kwargs)381 382 if size is not None:383 scale_factor = size / min(w, h)384 new_w, new_h = round(w * scale_factor), round(h * scale_factor)385 else:386 new_w, new_h = w, h387 new_w = new_w // size_divisible * size_divisible388 new_h = new_h // size_divisible * size_divisible389 390 # NOTE: It may result in unexpected number of frames in ffmpeg391 # if calculate the fps directly according to max_frames392 # if max_frames is not None and (fps is None or duration * fps > 2 * max_frames):393 # fps = round(max_frames / duration * 2)394 395 stream = ffmpeg.input(video_path, **input_kwargs)396 if fps is not None:397 stream = ffmpeg.filter(stream, "fps", fps=fps, round="down")398 if new_w != w or new_h != h:399 stream = ffmpeg.filter(stream, 'scale', new_w, new_h)400 stream = ffmpeg.output(stream, "pipe:", format="rawvideo", pix_fmt="rgb24", **output_kwargs)401 out, _ = ffmpeg.run(stream, capture_stdout=True, quiet=not verbose)402 403 frames = np.frombuffer(out, np.uint8).reshape([-1, new_h, new_w, 3]).transpose([0, 3, 1, 2])404 405 if fps is not None:406 timestamps = np.arange(start_time, start_time + duration + 1 / fps, 1 / fps)[:len(frames)]407 else:408 timestamps = np.linspace(start_time, start_time + duration, len(frames))409 410 if max_frames is not None and len(frames) > max_frames:411 indices = np.linspace(0, len(frames) - 1, max_frames, dtype=int)412 frames = frames[indices]413 timestamps = timestamps[indices]414 415 if temporal_factor > 1:416 pad_length = temporal_factor - len(frames) % temporal_factor417 frames = np.concatenate([frames, frames[-1:].repeat(pad_length, axis=0)])418 timestamps = np.concatenate([timestamps, timestamps[-1:].repeat(pad_length) + np.arange(1, pad_length + 1) / fps])419 420 frames = [frame for frame in frames]421 timestamps = [timestamp for timestamp in timestamps]422 423 return frames, timestamps424 425 def _load_multimodal_data(self, conversation: Conversation):426 multimodal_info = defaultdict(list)427 new_conversation = []428 for message in conversation:429 new_message = {"role": message["role"]}430 if not isinstance(message["content"], (list, tuple)):431 new_message["content"] = message["content"]432 new_conversation.append(new_message)433 continue434 435 new_contents = []436 for content in message["content"]:437 if not isinstance(content, dict):438 new_contents.append(content)439 continue440 assert "type" in content, "Content must have 'type' field."441 if content["type"] in ["image", "video"] and content["type"] in content and isinstance(content[content["type"]], dict):442 # TODO: support other types which are not compatible with json443 load_args = content[content["type"]]444 data_id = json.dumps({k: v for k, v in load_args.items() if not k in ["start_time", "end_time"]})445 new_content = copy.deepcopy(content)446 multimodal_info[data_id].append(new_content)447 new_contents.append(new_content)448 else:449 new_contents.append(content)450 451 new_message["content"] = new_contents452 new_conversation.append(new_message)453 454 for data_id, contents in multimodal_info.items():455 data_type = contents[0]["type"]456 if data_type == "image":457 image = self.load_images(contents[0][data_type]["image_path"])[0]458 for content in contents:459 content["image"] = [image.copy()]460 461 elif data_type == "video":462 # TODO: start_time is None?463 start_times = [content["video"].get("start_time", 0.) for content in contents]464 end_times = [content["video"].get("end_time", float("inf")) for content in contents]465 466 load_args = contents[0][data_type]467 start_time, end_time = min(start_times), max(end_times)468 if start_time > 0:469 load_args["start_time"] = start_time470 if end_time < float("inf"):471 load_args["end_time"] = end_time472 images, timestamps = self.load_video(**load_args)473 474 for content, start_time, end_time in zip(contents, start_times, end_times):475 cur_images, cur_timestamps = [], []476 for image, timestamp in zip(images, timestamps):477 if start_time <= timestamp <= end_time:478 cur_images.append(image.copy())479 cur_timestamps.append(timestamp)480 481 content[data_type] = cur_images482 content["num_frames"] = len(cur_images)483 content["timestamps"] = cur_timestamps484 485 return new_conversation486 487 def _gather_multimodal_data(self, conversation: Conversation):488 images = []489 for message in conversation:490 if not isinstance(message["content"], (list, tuple)):491 continue492 for content in message["content"]:493 if not isinstance(content, dict):494 continue495 if content["type"] == "video":496 video = content["video"]497 assert is_valid_video(video), f"Invalid video data: {video}."498 images.append(("video", video))499 if content["type"] == "image":500 image = content["image"]501 images.append(("image", image))502 images = images if len(images) > 0 else None503 return images504 505 def _process_conversation_with_label(506 self,507 conversation: Conversation,508 image_inputs: Dict[str, Any],509 **kwargs,510 ):511 assert kwargs.pop("return_tensors", "pt") == "pt", "Only PyTorch tensors are supported when return_labels=True."512 assert not "add_generation_prompt" in kwargs, "'add_generation_prompt' argument is not supported when return_labels=True."513 514 output_kwargs = self._merge_kwargs(515 Videollama3Qwen2ProcessorKwargs,516 tokenizer_init_kwargs=self.tokenizer.init_kwargs,517 **kwargs,518 )519 output_kwargs["chat_template_kwargs"].pop("add_generation_prompt")520 521 grid_sizes = self._get_downsampled_grid_sizes(image_inputs)522 text_inputs = {"input_ids": [], "labels": []}523 sample_types_list = []524 image_idx = 0525 526 for message_idx, message in enumerate(conversation):527 prompt = self.apply_chat_template(528 [message],529 tokenize=False,530 add_generation_prompt=False,531 **output_kwargs["chat_template_kwargs"],532 )533 prompt_chunks = prompt.split(DEFAULT_IMAGE_TOKEN)534 prompt = []535 for chunk_idx in range(len(prompt_chunks) - 1):536 prompt.append(prompt_chunks[chunk_idx])537 num_tokens = self._get_visual_seq_len(grid_sizes[image_idx])538 prompt.append(DEFAULT_IMAGE_TOKEN * num_tokens)539 image_idx += 1540 prompt.append(prompt_chunks[-1])541 prompt = "".join(prompt)542 543 # TODO: support attention_mask, position_ids, etc.544 input_ids = self.tokenizer.encode(prompt, return_tensors="pt", **output_kwargs["text_kwargs"])[0]545 text_inputs["input_ids"].append(input_ids)546 547 targets = torch.full_like(input_ids, IGNORE_INDEX)548 sample_types = torch.full_like(input_ids, IGNORE_INDEX)549 if message["role"] == "assistant":550 targets[self.generation_prompt_length:-1] = input_ids[self.generation_prompt_length:-1].clone()551 # elif message["role"] == "stream":552 # diff = torch.diff((input_ids == self.image_token_id).float())553 # image_end_indices = torch.nonzero(diff < 0)[:, 0]554 # targets[image_end_indices + 1] = input_ids[image_end_indices + 1]555 # sample_types = targets.clone()556 # sample_types[torch.logical_and(sample_types > 0, sample_types != self.eos_token_id)] = 0557 # targets[-2] = input_ids[-2] # <|im_end|>558 559 if message_idx > 0 and conversation[message_idx - 1]["role"] == "stream":560 targets[0] = input_ids[0]561 # TODO: consider non-special tokens562 sample_types[0] = input_ids[0]563 564 text_inputs["labels"].append(targets)565 sample_types_list.append(sample_types)566 567 # Negative sampling for streaming data568 text_inputs = {k: torch.cat(v) for k, v in text_inputs.items()}569 sample_types = torch.cat(sample_types_list)570 types, counts = torch.unique(sample_types[sample_types > -1], return_counts=True)571 572 if len(types) > 0:573 target_num_samples = counts.amin()574 for type_id, type_count in zip(types, counts):575 if type_count > target_num_samples:576 indices = torch.nonzero(sample_types == type_id)[:, 0]577 random_selector = torch.randperm(indices.size(0))[:-target_num_samples]578 text_inputs["labels"][indices[random_selector]] = IGNORE_INDEX579 # sample_types[indices[random_selector]] = -1580 581 assert len(grid_sizes) == image_idx, "Number of images does not match the number of image tokens in the text."582 583 return text_inputs584 585 def _process_conversation_without_label(586 self,587 conversation: Conversation,588 image_inputs: Dict[str, Any],589 **kwargs,590 ):591 output_kwargs = self._merge_kwargs(592 Videollama3Qwen2ProcessorKwargs,593 tokenizer_init_kwargs=self.tokenizer.init_kwargs,594 **kwargs,595 )596 prompt = self.apply_chat_template(597 conversation,598 tokenize=False,599 **output_kwargs["chat_template_kwargs"],600 )601 return self.process_text(prompt, image_inputs, **output_kwargs["text_kwargs"])602 603 def _process_conversation(604 self,605 conversation: Conversation,606 images: Optional[Union[BatchedImage, BatchedNamedImage]] = None,607 return_labels: bool = False,608 **kwargs: Unpack[Videollama3Qwen2ProcessorKwargs],609 ) -> BatchFeature:610 assert isinstance(conversation, list), "Conversation must be a list of messages."611 612 if images is None:613 conversation = self._load_multimodal_data(conversation)614 images = self._gather_multimodal_data(conversation)615 616 output_kwargs = self._merge_kwargs(617 Videollama3Qwen2ProcessorKwargs,618 tokenizer_init_kwargs=self.tokenizer.init_kwargs,619 **kwargs,620 )621 622 if images is not None:623 image_inputs = self.process_images(images, **output_kwargs["images_kwargs"])624 else:625 image_inputs = {}626 627 if return_labels:628 text_inputs = self._process_conversation_with_label(conversation, image_inputs, **kwargs)629 else:630 text_inputs = self._process_conversation_without_label(conversation, image_inputs, **kwargs)631 632 return BatchFeature(data={**text_inputs, **image_inputs})633 634 def _process_plain(635 self,636 text: Union[TextInput, PreTokenizedInput] = None,637 images: Optional[Union[BatchedImage, BatchedNamedImage]] = None,638 return_labels: bool = False,639 **kwargs: Unpack[Videollama3Qwen2ProcessorKwargs],640 ) -> BatchFeature:641 if text is None:642 raise ValueError("You must provide 'text' or 'message'.")643 if return_labels:644 raise ValueError("return_labels is not supported for plain text processing.")645 646 output_kwargs = self._merge_kwargs(647 Videollama3Qwen2ProcessorKwargs,648 tokenizer_init_kwargs=self.tokenizer.init_kwargs,649 **kwargs,650 )651 652 if images is not None:653 image_inputs = self.process_images(images, **output_kwargs["images_kwargs"])654 else:655 image_inputs = {}656 657 text_inputs = self.process_text(text, image_inputs, **output_kwargs["text_kwargs"])658 659 return BatchFeature(data={**text_inputs, **image_inputs})660 661 def process_images(self, images: Union[BatchedImage, BatchedNamedImage], **kwargs):662 modals, images = make_batched_images(images)663 if not "merge_size" in kwargs:664 kwargs["merge_size"] = [665 self.image_merge_size if modal == "image" else self.video_merge_size666 for modal in modals667 ]668 image_inputs = self.image_processor(images=images, **kwargs)669 image_inputs["modals"] = modals670 return image_inputs671 672 def process_text(673 self,674 text: TextInput,675 image_inputs: Dict[str, Any],676 **kwargs,677 ):678 grid_sizes = self._get_downsampled_grid_sizes(image_inputs)679 680 kwargs.pop("padding")681 kwargs.pop("padding_side")682 683 if len(grid_sizes) > 0:684 image_idx = 0685 while DEFAULT_IMAGE_TOKEN in text:686 num_tokens = self._get_visual_seq_len(grid_sizes[image_idx])687 text = text.replace(DEFAULT_IMAGE_TOKEN, "<placeholder>" * num_tokens, 1)688 image_idx += 1689 text = text.replace("<placeholder>", DEFAULT_IMAGE_TOKEN)690 691 assert len(grid_sizes) == image_idx, "Number of images does not match the number of image tokens in the text."692 693 text_inputs = self.tokenizer(text, **kwargs)694 return text_inputs695 696 def __call__(697 self,698 text: Optional[TextInput] = None,699 conversation: Optional[Conversation] = None,700 images: Optional[Union[BatchedImage, BatchedNamedImage]] = None,701 return_labels: bool = False,702 **kwargs: Unpack[Videollama3Qwen2ProcessorKwargs],703 ) -> BatchFeature:704 if conversation is not None:705 if text is not None:706 raise ValueError("You cannot provide 'message' with 'text'.")707 return self._process_conversation(conversation, images, return_labels, **kwargs)708 return self._process_plain(text, images, return_labels, **kwargs)709 710 def batch_decode(self, *args, **kwargs):711 return self.tokenizer.batch_decode(*args, **kwargs)712 713 def decode(self, *args, **kwargs):714 return self.tokenizer.decode(*args, **kwargs)715 716 def apply_chat_template(717 self,718 conversation: Conversation,719 chat_template: Optional[str] = None,720 tokenize: bool = False,721 add_system_prompt: bool = False,722 add_generation_prompt: bool = False,723 image_token: Optional[str] = DEFAULT_IMAGE_TOKEN,724 **kwargs,725 ) -> str:726 """727 Similar to the `apply_chat_template` method on tokenizers, this method applies a Jinja template to input728 conversations to turn them into a single tokenizable string.729 730 Args:731 conversation (`List[Dict, str, str]`):732 The conversation to format.733 chat_template (`Optional[str]`, *optional*):734 The Jinja template to use for formatting the conversation. If not provided, the tokenizer's735 chat template is used.736 tokenize (`bool`, *optional*, defaults to `False`):737 Whether to tokenize the output or not.738 add_system_prompt (`bool`, *optional*, defaults to `False`):739 Whether to add the system prompt to the output or not.740 add_generation_prompt (`bool`, *optional*, defaults to `False`):741 Whether to add the generation prompt to the output or not.742 image_token (`Optional[str]`, *optional*, defaults to `<image>`):743 The token to use for indicating images in the conversation.744 **kwargs:745 Additional keyword arguments746 """747 748 if chat_template is None:749 if self.chat_template is not None:750 chat_template = self.chat_template751 else:752 raise ValueError(753 "No chat template is set for this processor. Please either set the `chat_template` attribute, "754 "or provide a chat template as an argument. See "755 "https://huggingface.co/docs/transformers/main/en/chat_templating for more information."756 )757 return self.tokenizer.apply_chat_template(758 conversation,759 chat_template=chat_template,760 tokenize=tokenize,761 add_system_prompt=add_system_prompt,762 add_generation_prompt=add_generation_prompt,763 image_token=image_token,764 **kwargs765 )766 767 @property768 def model_input_names(self):769 tokenizer_input_names = self.tokenizer.model_input_names770 image_processor_input_names = self.image_processor.model_input_names771 return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) + ["modals"]772 773 # modified from transformers.ProcessorMixin774 def _merge_kwargs(775 self,776 ModelProcessorKwargs: ProcessingKwargs,777 tokenizer_init_kwargs: Optional[Dict] = None,778 **kwargs,779 ) -> Dict[str, Dict]:780 """781 Method to merge dictionaries of kwargs cleanly separated by modality within a Processor instance.782 The order of operations is as follows:783 1) kwargs passed as before have highest priority to preserve BC.784 ```python785 high_priority_kwargs = {"crop_size" = {"height": 222, "width": 222}, "padding" = "max_length"}786 processor(..., **high_priority_kwargs)787 ```788 2) kwargs passed as modality-specific kwargs have second priority. This is the recommended API.789 ```python790 processor(..., text_kwargs={"padding": "max_length"}, images_kwargs={"crop_size": {"height": 222, "width": 222}}})791 ```792 3) kwargs passed during instantiation of a modality processor have fourth priority.793 ```python794 tokenizer = tokenizer_class(..., {"padding": "max_length"})795 image_processor = image_processor_class(...)796 processor(tokenizer, image_processor) # will pass max_length unless overriden by kwargs at call797 ```798 4) defaults kwargs specified at processor level have lowest priority.799 ```python800 class MyProcessingKwargs(ProcessingKwargs, CommonKwargs, TextKwargs, ImagesKwargs, total=False):801 _defaults = {802 "text_kwargs": {803 "padding": "max_length",804 "max_length": 64,805 },806 }807 ```808 Args:809 ModelProcessorKwargs (`ProcessingKwargs`):810 Typed dictionary of kwargs specifically required by the model passed.811 tokenizer_init_kwargs (`Dict`, *optional*):812 Dictionary of kwargs the tokenizer was instantiated with and need to take precedence over defaults.813 814 Returns:815 output_kwargs (`Dict`):816 Dictionary of per-modality kwargs to be passed to each modality-specific processor.817 818 """819 # Initialize dictionaries820 output_kwargs = {821 "text_kwargs": {},822 "images_kwargs": {},823 "audio_kwargs": {},824 "videos_kwargs": {},825 "chat_template_kwargs": {},826 "common_kwargs": {},827 }828 829 default_kwargs = {830 "text_kwargs": {},831 "images_kwargs": {},832 "audio_kwargs": {},833 "videos_kwargs": {},834 "chat_template_kwargs": {},835 "common_kwargs": {},836 }837 838 used_keys = set()839 840 # get defaults from set model processor kwargs if they exist841 for modality in default_kwargs:842 default_kwargs[modality] = ModelProcessorKwargs._defaults.get(modality, {}).copy()843 # update defaults with arguments from tokenizer init844 for modality_key in ModelProcessorKwargs.__annotations__[modality].__annotations__.keys():845 # init with tokenizer init kwargs if necessary846 if modality_key in tokenizer_init_kwargs:847 value = (848 getattr(self.tokenizer, modality_key)849 if hasattr(self.tokenizer, modality_key)850 else tokenizer_init_kwargs[modality_key]851 )852 default_kwargs[modality][modality_key] = value853 # now defaults kwargs are updated with the tokenizers defaults.854 # pass defaults to output dictionary855 output_kwargs.update(default_kwargs)856 857 # update modality kwargs with passed kwargs858 non_modality_kwargs = set(kwargs) - set(output_kwargs)859 for modality in output_kwargs:860 for modality_key in ModelProcessorKwargs.__annotations__[modality].__annotations__.keys():861 # check if we received a structured kwarg dict or not to handle it correctly862 if modality in kwargs:863 kwarg_value = kwargs[modality].pop(modality_key, "__empty__")864 # check if this key was passed as a flat kwarg.865 if kwarg_value != "__empty__" and modality_key in non_modality_kwargs:866 raise ValueError(867 f"Keyword argument {modality_key} was passed two times:\n"868 f"in a dictionary for {modality} and as a **kwarg."869 )870 elif modality_key in kwargs:871 # we get a modality_key instead of popping it because modality-specific processors872 # can have overlapping kwargs873 kwarg_value = kwargs.get(modality_key, "__empty__")874 else:875 kwarg_value = "__empty__"876 if kwarg_value != "__empty__":877 output_kwargs[modality][modality_key] = kwarg_value878 used_keys.add(modality_key)879 880 # Determine if kwargs is a flat dictionary or contains nested dictionaries881 if any(key in default_kwargs for key in kwargs):882 # kwargs is dictionary-based, and some keys match modality names883 for modality, subdict in kwargs.items():884 if modality in default_kwargs:885 for subkey, subvalue in subdict.items():886 if subkey not in used_keys:887 output_kwargs[modality][subkey] = subvalue888 used_keys.add(subkey)889 else:890 # kwargs is a flat dictionary891 for key in kwargs:892 if key not in used_keys:893 output_kwargs["common_kwargs"][key] = kwargs[key]894 895 # all modality-specific kwargs are updated with common kwargs896 for modality in output_kwargs:897 output_kwargs[modality].update(output_kwargs["common_kwargs"])898 return output_kwargs899