Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2021 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""AutoProcessor class."""16 17import importlib18import inspect19import json20import warnings21from collections import OrderedDict22 23# Build the list of all feature extractors24from ...configuration_utils import PretrainedConfig25from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code26from ...feature_extraction_utils import FeatureExtractionMixin27from ...image_processing_utils import ImageProcessingMixin28from ...processing_utils import ProcessorMixin29from ...tokenization_utils import TOKENIZER_CONFIG_FILE30from ...utils import FEATURE_EXTRACTOR_NAME, PROCESSOR_NAME, VIDEO_PROCESSOR_NAME, cached_file, logging31from ...video_processing_utils import BaseVideoProcessor32from .auto_factory import _LazyAutoMapping33from .configuration_auto import (34    CONFIG_MAPPING_NAMES,35    AutoConfig,36    model_type_to_module_name,37    replace_list_option_in_docstrings,38)39from .feature_extraction_auto import AutoFeatureExtractor40from .image_processing_auto import AutoImageProcessor41from .tokenization_auto import AutoTokenizer42 43 44logger = logging.get_logger(__name__)45 46PROCESSOR_MAPPING_NAMES = OrderedDict(47    [48        ("aimv2", "CLIPProcessor"),49        ("align", "AlignProcessor"),50        ("altclip", "AltCLIPProcessor"),51        ("aria", "AriaProcessor"),52        ("aya_vision", "AyaVisionProcessor"),53        ("bark", "BarkProcessor"),54        ("blip", "BlipProcessor"),55        ("blip-2", "Blip2Processor"),56        ("bridgetower", "BridgeTowerProcessor"),57        ("chameleon", "ChameleonProcessor"),58        ("chinese_clip", "ChineseCLIPProcessor"),59        ("clap", "ClapProcessor"),60        ("clip", "CLIPProcessor"),61        ("clipseg", "CLIPSegProcessor"),62        ("clvp", "ClvpProcessor"),63        ("cohere2_vision", "Cohere2VisionProcessor"),64        ("colpali", "ColPaliProcessor"),65        ("colqwen2", "ColQwen2Processor"),66        ("deepseek_vl", "DeepseekVLProcessor"),67        ("deepseek_vl_hybrid", "DeepseekVLHybridProcessor"),68        ("dia", "DiaProcessor"),69        ("edgetam", "Sam2Processor"),70        ("emu3", "Emu3Processor"),71        ("evolla", "EvollaProcessor"),72        ("flava", "FlavaProcessor"),73        ("florence2", "Florence2Processor"),74        ("fuyu", "FuyuProcessor"),75        ("gemma3", "Gemma3Processor"),76        ("gemma3n", "Gemma3nProcessor"),77        ("git", "GitProcessor"),78        ("glm4v", "Glm4vProcessor"),79        ("glm4v_moe", "Glm4vProcessor"),80        ("got_ocr2", "GotOcr2Processor"),81        ("granite_speech", "GraniteSpeechProcessor"),82        ("grounding-dino", "GroundingDinoProcessor"),83        ("groupvit", "CLIPProcessor"),84        ("hubert", "Wav2Vec2Processor"),85        ("idefics", "IdeficsProcessor"),86        ("idefics2", "Idefics2Processor"),87        ("idefics3", "Idefics3Processor"),88        ("instructblip", "InstructBlipProcessor"),89        ("instructblipvideo", "InstructBlipVideoProcessor"),90        ("internvl", "InternVLProcessor"),91        ("janus", "JanusProcessor"),92        ("kosmos-2", "Kosmos2Processor"),93        ("kosmos-2.5", "Kosmos2_5Processor"),94        ("kyutai_speech_to_text", "KyutaiSpeechToTextProcessor"),95        ("layoutlmv2", "LayoutLMv2Processor"),96        ("layoutlmv3", "LayoutLMv3Processor"),97        ("lfm2_vl", "Lfm2VlProcessor"),98        ("llama4", "Llama4Processor"),99        ("llava", "LlavaProcessor"),100        ("llava_next", "LlavaNextProcessor"),101        ("llava_next_video", "LlavaNextVideoProcessor"),102        ("llava_onevision", "LlavaOnevisionProcessor"),103        ("markuplm", "MarkupLMProcessor"),104        ("mctct", "MCTCTProcessor"),105        ("metaclip_2", "CLIPProcessor"),106        ("mgp-str", "MgpstrProcessor"),107        ("mistral3", "PixtralProcessor"),108        ("mllama", "MllamaProcessor"),109        ("mm-grounding-dino", "GroundingDinoProcessor"),110        ("moonshine", "Wav2Vec2Processor"),111        ("oneformer", "OneFormerProcessor"),112        ("ovis2", "Ovis2Processor"),113        ("owlv2", "Owlv2Processor"),114        ("owlvit", "OwlViTProcessor"),115        ("paligemma", "PaliGemmaProcessor"),116        ("perception_lm", "PerceptionLMProcessor"),117        ("phi4_multimodal", "Phi4MultimodalProcessor"),118        ("pix2struct", "Pix2StructProcessor"),119        ("pixtral", "PixtralProcessor"),120        ("pop2piano", "Pop2PianoProcessor"),121        ("qwen2_5_omni", "Qwen2_5OmniProcessor"),122        ("qwen2_5_vl", "Qwen2_5_VLProcessor"),123        ("qwen2_audio", "Qwen2AudioProcessor"),124        ("qwen2_vl", "Qwen2VLProcessor"),125        ("qwen3_omni_moe", "Qwen3OmniMoeProcessor"),126        ("qwen3_vl", "Qwen3VLProcessor"),127        ("qwen3_vl_moe", "Qwen3VLProcessor"),128        ("sam", "SamProcessor"),129        ("sam2", "Sam2Processor"),130        ("sam_hq", "SamHQProcessor"),131        ("seamless_m4t", "SeamlessM4TProcessor"),132        ("sew", "Wav2Vec2Processor"),133        ("sew-d", "Wav2Vec2Processor"),134        ("shieldgemma2", "ShieldGemma2Processor"),135        ("siglip", "SiglipProcessor"),136        ("siglip2", "Siglip2Processor"),137        ("smolvlm", "SmolVLMProcessor"),138        ("speech_to_text", "Speech2TextProcessor"),139        ("speech_to_text_2", "Speech2Text2Processor"),140        ("speecht5", "SpeechT5Processor"),141        ("trocr", "TrOCRProcessor"),142        ("tvlt", "TvltProcessor"),143        ("tvp", "TvpProcessor"),144        ("udop", "UdopProcessor"),145        ("unispeech", "Wav2Vec2Processor"),146        ("unispeech-sat", "Wav2Vec2Processor"),147        ("video_llava", "VideoLlavaProcessor"),148        ("vilt", "ViltProcessor"),149        ("vipllava", "LlavaProcessor"),150        ("vision-text-dual-encoder", "VisionTextDualEncoderProcessor"),151        ("voxtral", "VoxtralProcessor"),152        ("wav2vec2", "Wav2Vec2Processor"),153        ("wav2vec2-bert", "Wav2Vec2Processor"),154        ("wav2vec2-conformer", "Wav2Vec2Processor"),155        ("wavlm", "Wav2Vec2Processor"),156        ("whisper", "WhisperProcessor"),157        ("xclip", "XCLIPProcessor"),158    ]159)160 161PROCESSOR_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, PROCESSOR_MAPPING_NAMES)162 163 164def processor_class_from_name(class_name: str):165    for module_name, processors in PROCESSOR_MAPPING_NAMES.items():166        if class_name in processors:167            module_name = model_type_to_module_name(module_name)168 169            module = importlib.import_module(f".{module_name}", "transformers.models")170            try:171                return getattr(module, class_name)172            except AttributeError:173                continue174 175    for processor in PROCESSOR_MAPPING._extra_content.values():176        if getattr(processor, "__name__", None) == class_name:177            return processor178 179    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main180    # init and we return the proper dummy to get an appropriate error message.181    main_module = importlib.import_module("transformers")182    if hasattr(main_module, class_name):183        return getattr(main_module, class_name)184 185    return None186 187 188class AutoProcessor:189    r"""190    This is a generic processor class that will be instantiated as one of the processor classes of the library when191    created with the [`AutoProcessor.from_pretrained`] class method.192 193    This class cannot be instantiated directly using `__init__()` (throws an error).194    """195 196    def __init__(self):197        raise OSError(198            "AutoProcessor is designed to be instantiated "199            "using the `AutoProcessor.from_pretrained(pretrained_model_name_or_path)` method."200        )201 202    @classmethod203    @replace_list_option_in_docstrings(PROCESSOR_MAPPING_NAMES)204    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):205        r"""206        Instantiate one of the processor classes of the library from a pretrained model vocabulary.207 208        The processor class to instantiate is selected based on the `model_type` property of the config object (either209        passed as an argument or loaded from `pretrained_model_name_or_path` if possible):210 211        List options212 213        Params:214            pretrained_model_name_or_path (`str` or `os.PathLike`):215                This can be either:216 217                - a string, the *model id* of a pretrained feature_extractor hosted inside a model repo on218                  huggingface.co.219                - a path to a *directory* containing a processor files saved using the `save_pretrained()` method,220                  e.g., `./my_model_directory/`.221            cache_dir (`str` or `os.PathLike`, *optional*):222                Path to a directory in which a downloaded pretrained model feature extractor should be cached if the223                standard cache should not be used.224            force_download (`bool`, *optional*, defaults to `False`):225                Whether or not to force to (re-)download the feature extractor files and override the cached versions226                if they exist.227            resume_download:228                Deprecated and ignored. All downloads are now resumed by default when possible.229                Will be removed in v5 of Transformers.230            proxies (`dict[str, str]`, *optional*):231                A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',232                'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.233            token (`str` or *bool*, *optional*):234                The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated235                when running `hf auth login` (stored in `~/.huggingface`).236            revision (`str`, *optional*, defaults to `"main"`):237                The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a238                git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any239                identifier allowed by git.240            return_unused_kwargs (`bool`, *optional*, defaults to `False`):241                If `False`, then this function returns just the final feature extractor object. If `True`, then this242                functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a dictionary243                consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the part of244                `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored.245            trust_remote_code (`bool`, *optional*, defaults to `False`):246                Whether or not to allow for custom models defined on the Hub in their own modeling files. This option247                should only be set to `True` for repositories you trust and in which you have read the code, as it will248                execute code present on the Hub on your local machine.249            kwargs (`dict[str, Any]`, *optional*):250                The values in kwargs of any keys which are feature extractor attributes will be used to override the251                loaded values. Behavior concerning key/value pairs whose keys are *not* feature extractor attributes is252                controlled by the `return_unused_kwargs` keyword parameter.253 254        <Tip>255 256        Passing `token=True` is required when you want to use a private model.257 258        </Tip>259 260        Examples:261 262        ```python263        >>> from transformers import AutoProcessor264 265        >>> # Download processor from huggingface.co and cache.266        >>> processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h")267 268        >>> # If processor files are in a directory (e.g. processor was saved using *save_pretrained('./test/saved_model/')*)269        >>> # processor = AutoProcessor.from_pretrained("./test/saved_model/")270        ```"""271        use_auth_token = kwargs.pop("use_auth_token", None)272        if use_auth_token is not None:273            warnings.warn(274                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",275                FutureWarning,276            )277            if kwargs.get("token") is not None:278                raise ValueError(279                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."280                )281            kwargs["token"] = use_auth_token282 283        config = kwargs.pop("config", None)284        trust_remote_code = kwargs.pop("trust_remote_code", None)285        kwargs["_from_auto"] = True286 287        processor_class = None288        processor_auto_map = None289 290        # First, let's see if we have a processor or preprocessor config.291        # Filter the kwargs for `cached_file`.292        cached_file_kwargs = {key: kwargs[key] for key in inspect.signature(cached_file).parameters if key in kwargs}293        # We don't want to raise294        cached_file_kwargs.update(295            {296                "_raise_exceptions_for_gated_repo": False,297                "_raise_exceptions_for_missing_entries": False,298                "_raise_exceptions_for_connection_errors": False,299            }300        )301 302        # Let's start by checking whether the processor class is saved in a processor config303        processor_config_file = cached_file(pretrained_model_name_or_path, PROCESSOR_NAME, **cached_file_kwargs)304        if processor_config_file is not None:305            config_dict, _ = ProcessorMixin.get_processor_dict(pretrained_model_name_or_path, **kwargs)306            processor_class = config_dict.get("processor_class", None)307            if "AutoProcessor" in config_dict.get("auto_map", {}):308                processor_auto_map = config_dict["auto_map"]["AutoProcessor"]309 310        if processor_class is None:311            # If not found, let's check whether the processor class is saved in an image processor config312            preprocessor_config_file = cached_file(313                pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME, **cached_file_kwargs314            )315            if preprocessor_config_file is not None:316                config_dict, _ = ImageProcessingMixin.get_image_processor_dict(pretrained_model_name_or_path, **kwargs)317                processor_class = config_dict.get("processor_class", None)318                if "AutoProcessor" in config_dict.get("auto_map", {}):319                    processor_auto_map = config_dict["auto_map"]["AutoProcessor"]320 321            # Saved as video processor322            if preprocessor_config_file is None:323                preprocessor_config_file = cached_file(324                    pretrained_model_name_or_path, VIDEO_PROCESSOR_NAME, **cached_file_kwargs325                )326                if preprocessor_config_file is not None:327                    config_dict, _ = BaseVideoProcessor.get_video_processor_dict(328                        pretrained_model_name_or_path, **kwargs329                    )330                    processor_class = config_dict.get("processor_class", None)331                    if "AutoProcessor" in config_dict.get("auto_map", {}):332                        processor_auto_map = config_dict["auto_map"]["AutoProcessor"]333 334            # Saved as feature extractor335            if preprocessor_config_file is None:336                preprocessor_config_file = cached_file(337                    pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME, **cached_file_kwargs338                )339                if preprocessor_config_file is not None and processor_class is None:340                    config_dict, _ = FeatureExtractionMixin.get_feature_extractor_dict(341                        pretrained_model_name_or_path, **kwargs342                    )343                    processor_class = config_dict.get("processor_class", None)344                    if "AutoProcessor" in config_dict.get("auto_map", {}):345                        processor_auto_map = config_dict["auto_map"]["AutoProcessor"]346 347        if processor_class is None:348            # Next, let's check whether the processor class is saved in a tokenizer349            tokenizer_config_file = cached_file(350                pretrained_model_name_or_path, TOKENIZER_CONFIG_FILE, **cached_file_kwargs351            )352            if tokenizer_config_file is not None:353                with open(tokenizer_config_file, encoding="utf-8") as reader:354                    config_dict = json.load(reader)355 356                processor_class = config_dict.get("processor_class", None)357                if "AutoProcessor" in config_dict.get("auto_map", {}):358                    processor_auto_map = config_dict["auto_map"]["AutoProcessor"]359 360        if processor_class is None:361            # Otherwise, load config, if it can be loaded.362            if not isinstance(config, PretrainedConfig):363                config = AutoConfig.from_pretrained(364                    pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs365                )366 367            # And check if the config contains the processor class.368            processor_class = getattr(config, "processor_class", None)369            if hasattr(config, "auto_map") and "AutoProcessor" in config.auto_map:370                processor_auto_map = config.auto_map["AutoProcessor"]371 372        if processor_class is not None:373            processor_class = processor_class_from_name(processor_class)374 375        has_remote_code = processor_auto_map is not None376        has_local_code = processor_class is not None or type(config) in PROCESSOR_MAPPING377        if has_remote_code:378            if "--" in processor_auto_map:379                upstream_repo = processor_auto_map.split("--")[0]380            else:381                upstream_repo = None382            trust_remote_code = resolve_trust_remote_code(383                trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code, upstream_repo384            )385 386        if has_remote_code and trust_remote_code:387            processor_class = get_class_from_dynamic_module(388                processor_auto_map, pretrained_model_name_or_path, **kwargs389            )390            _ = kwargs.pop("code_revision", None)391            processor_class.register_for_auto_class()392            return processor_class.from_pretrained(393                pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs394            )395        elif processor_class is not None:396            return processor_class.from_pretrained(397                pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs398            )399        # Last try: we use the PROCESSOR_MAPPING.400        elif type(config) in PROCESSOR_MAPPING:401            return PROCESSOR_MAPPING[type(config)].from_pretrained(pretrained_model_name_or_path, **kwargs)402 403        # At this stage, there doesn't seem to be a `Processor` class available for this model, so let's try a404        # tokenizer.405        try:406            return AutoTokenizer.from_pretrained(407                pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs408            )409        except Exception:410            try:411                return AutoImageProcessor.from_pretrained(412                    pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs413                )414            except Exception:415                pass416 417            try:418                return AutoFeatureExtractor.from_pretrained(419                    pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs420                )421            except Exception:422                pass423 424        raise ValueError(425            f"Unrecognized processing class in {pretrained_model_name_or_path}. Can't instantiate a processor, a "426            "tokenizer, an image processor or a feature extractor for this model. Make sure the repository contains "427            "the files of at least one of those processing classes."428        )429 430    @staticmethod431    def register(config_class, processor_class, exist_ok=False):432        """433        Register a new processor for this class.434 435        Args:436            config_class ([`PretrainedConfig`]):437                The configuration corresponding to the model to register.438            processor_class ([`ProcessorMixin`]): The processor to register.439        """440        PROCESSOR_MAPPING.register(config_class, processor_class, exist_ok=exist_ok)441 442 443__all__ = ["PROCESSOR_MAPPING", "AutoProcessor"]444