Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2025 Meta Platforms, Inc. and the HuggingFace Inc. team. All rights reserved.3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""PerceptionLM model configuration"""15 16from ...configuration_utils import PretrainedConfig17from ...utils import logging18from ..auto import CONFIG_MAPPING, AutoConfig19from ..timm_wrapper.configuration_timm_wrapper import TimmWrapperConfig20 21 22logger = logging.get_logger(__name__)23 24 25class PerceptionLMConfig(PretrainedConfig):26    r"""27    This is the configuration class to store the configuration of a [`PerceptionLMForConditionalGeneration`]. It is used to instantiate an28    PerceptionLM model according to the specified arguments, defining the model architecture.29 30    Example models:31    -  [facebook/Perception-LM-1B](https://huggingface.co/facebook/Perception-LM-1B).32    -  [facebook/Perception-LM-3B](https://huggingface.co/facebook/Perception-LM-3B).33    -  [facebook/Perception-LM-8B](https://huggingface.co/facebook/Perception-LM-8B).34 35    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the36    documentation from [`PretrainedConfig`] for more information.37 38    Args:39        vision_config (`Union[TimmWrapperConfig, dict]`, *optional*, defaults to `TimmWrapperConfig()`):40            The config object or dictionary of the vision backbone.41        text_config (`Union[PretrainedConfig, dict]`, *optional*, defaults to `LlamaConfig()`):42            The config object or dictionary of the text backbone.43        vision_use_cls_token (`bool`, *optional*, defaults to `True`):44            Whether CLS token is used in the vision backbone. If used, we remove CLS token embedding from vision output.45        projector_pooling_ratio (`int`, *optional*, defaults to 1):46            The pooling ratio used in the multimodal projector.47        image_token_id (`int`, *optional*, defaults to 128002):48            The image token index to encode the image prompt.49        video_token_id (`int`, *optional*, defaults to 128003):50            The video token index to encode the video prompt.51    """52 53    model_type = "perception_lm"54    sub_configs = {"text_config": AutoConfig, "vision_config": TimmWrapperConfig}55 56    def __init__(57        self,58        vision_config=None,59        text_config=None,60        vision_use_cls_token=True,61        projector_pooling_ratio=1,62        image_token_id=128002,63        video_token_id=128003,64        **kwargs,65    ):66        self.image_token_id = image_token_id67        self.video_token_id = video_token_id68        if isinstance(vision_config, dict):69            vision_config = TimmWrapperConfig(**vision_config)70        elif isinstance(vision_config, TimmWrapperConfig):71            pass72        elif vision_config is None:73            vision_config = TimmWrapperConfig()74        self.vision_config = vision_config75        self.vision_use_cls_token = vision_use_cls_token76 77        if isinstance(text_config, dict):78            text_config["model_type"] = text_config.get("model_type", "llama")79            text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)80        elif text_config is None:81            text_config = CONFIG_MAPPING["llama"]()82 83        self.text_config = text_config84        self.projector_pooling_ratio = projector_pooling_ratio85        super().__init__(**kwargs)86 87 88__all__ = ["PerceptionLMConfig"]89