Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2025 HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import inspect17import os18import textwrap19from pathlib import Path20from typing import Optional, Union, get_args21 22import regex as re23 24from .doc import (25 MODELS_TO_PIPELINE,26 PIPELINE_TASKS_TO_SAMPLE_DOCSTRINGS,27 PT_SAMPLE_DOCSTRINGS,28 _prepare_output_docstrings,29)30from .generic import ModelOutput31 32 33PATH_TO_TRANSFORMERS = Path("src").resolve() / "transformers"34 35 36AUTODOC_FILES = [37 "configuration_*.py",38 "modeling_*.py",39 "tokenization_*.py",40 "processing_*.py",41 "image_processing_*_fast.py",42 "image_processing_*.py",43 "feature_extractor_*.py",44]45 46PLACEHOLDER_TO_AUTO_MODULE = {47 "image_processor_class": ("image_processing_auto", "IMAGE_PROCESSOR_MAPPING_NAMES"),48 "video_processor_class": ("video_processing_auto", "VIDEO_PROCESSOR_MAPPING_NAMES"),49 "feature_extractor_class": ("feature_extraction_auto", "FEATURE_EXTRACTOR_MAPPING_NAMES"),50 "processor_class": ("processing_auto", "PROCESSOR_MAPPING_NAMES"),51 "config_class": ("configuration_auto", "CONFIG_MAPPING_NAMES"),52}53 54UNROLL_KWARGS_METHODS = {55 "preprocess",56}57 58UNROLL_KWARGS_CLASSES = {59 "ImageProcessorFast",60}61 62HARDCODED_CONFIG_FOR_MODELS = {63 "openai": "OpenAIGPTConfig",64 "x-clip": "XCLIPConfig",65 "kosmos2": "Kosmos2Config",66 "kosmos2-5": "Kosmos2_5Config",67 "donut": "DonutSwinConfig",68 "esmfold": "EsmConfig",69}70 71_re_checkpoint = re.compile(r"\[(.+?)\]\((https://huggingface\.co/.+?)\)")72 73 74class ImageProcessorArgs:75 images = {76 "description": """77 Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If78 passing in images with pixel values between 0 and 1, set `do_rescale=False`.79 """,80 "shape": None,81 }82 83 videos = {84 "description": """85 Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If86 passing in videos with pixel values between 0 and 1, set `do_rescale=False`.87 """,88 "shape": None,89 }90 91 do_resize = {92 "description": """93 Whether to resize the image.94 """,95 "shape": None,96 }97 98 size = {99 "description": """100 Describes the maximum input dimensions to the model.101 """,102 "shape": None,103 }104 105 default_to_square = {106 "description": """107 Whether to default to a square image when resizing, if size is an int.108 """,109 "shape": None,110 }111 112 resample = {113 "description": """114 Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only115 has an effect if `do_resize` is set to `True`.116 """,117 "shape": None,118 }119 120 do_center_crop = {121 "description": """122 Whether to center crop the image.123 """,124 "shape": None,125 }126 127 crop_size = {128 "description": """129 Size of the output image after applying `center_crop`.130 """,131 "shape": None,132 }133 134 do_pad = {135 "description": """136 Whether to pad the image. Padding is done either to the largest size in the batch137 or to a fixed square size per image. The exact padding strategy depends on the model.138 """,139 "shape": None,140 }141 142 pad_size = {143 "description": """144 The size in `{"height": int, "width" int}` to pad the images to. Must be larger than any image size145 provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest146 height and width in the batch. Applied only when `do_pad=True.`147 """,148 "shape": None,149 }150 151 do_rescale = {152 "description": """153 Whether to rescale the image.154 """,155 "shape": None,156 }157 158 rescale_factor = {159 "description": """160 Rescale factor to rescale the image by if `do_rescale` is set to `True`.161 """,162 "shape": None,163 }164 165 do_normalize = {166 "description": """167 Whether to normalize the image.168 """,169 "shape": None,170 }171 172 image_mean = {173 "description": """174 Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.175 """,176 "shape": None,177 }178 179 image_std = {180 "description": """181 Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to182 `True`.183 """,184 "shape": None,185 }186 187 do_convert_rgb = {188 "description": """189 Whether to convert the image to RGB.190 """,191 "shape": None,192 }193 194 return_tensors = {195 "description": """196 Returns stacked tensors if set to `pt, otherwise returns a list of tensors.197 """,198 "shape": None,199 }200 201 data_format = {202 "description": """203 Only `ChannelDimension.FIRST` is supported. Added for compatibility with slow processors.204 """,205 "shape": None,206 }207 208 input_data_format = {209 "description": """210 The channel dimension format for the input image. If unset, the channel dimension format is inferred211 from the input image. Can be one of:212 - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.213 - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.214 - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.215 """,216 "shape": None,217 }218 219 device = {220 "description": """221 The device to process the images on. If unset, the device is inferred from the input images.222 """,223 "shape": None,224 }225 226 disable_grouping = {227 "description": """228 Whether to disable grouping of images by size to process them individually and not in batches.229 If None, will be set to True if the images are on CPU, and False otherwise. This choice is based on230 empirical observations, as detailed here: https://github.com/huggingface/transformers/pull/38157231 """,232 "shape": None,233 }234 235 236class ModelArgs:237 labels = {238 "description": """239 Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,240 config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored241 (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.242 """,243 "shape": "of shape `(batch_size, sequence_length)`",244 }245 246 num_logits_to_keep = {247 "description": """248 Calculate logits for the last `num_logits_to_keep` tokens. If `0`, calculate logits for all249 `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that250 token can save memory, which becomes pretty significant for long sequences or large vocabulary size.251 """,252 "shape": None,253 }254 255 input_ids = {256 "description": """257 Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.258 259 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and260 [`PreTrainedTokenizer.__call__`] for details.261 262 [What are input IDs?](../glossary#input-ids)263 """,264 "shape": "of shape `(batch_size, sequence_length)`",265 }266 267 input_values = {268 "description": """269 Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file270 into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library271 (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).272 To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion273 into a tensor of type `torch.FloatTensor`. See [`{processor_class}.__call__`] for details.274 """,275 "shape": "of shape `(batch_size, sequence_length)`",276 }277 278 attention_mask = {279 "description": """280 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:281 282 - 1 for tokens that are **not masked**,283 - 0 for tokens that are **masked**.284 285 [What are attention masks?](../glossary#attention-mask)286 """,287 "shape": "of shape `(batch_size, sequence_length)`",288 }289 290 head_mask = {291 "description": """292 Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:293 294 - 1 indicates the head is **not masked**,295 - 0 indicates the head is **masked**.296 """,297 "shape": "of shape `(num_heads,)` or `(num_layers, num_heads)`",298 }299 300 cross_attn_head_mask = {301 "description": """302 Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`:303 304 - 1 indicates the head is **not masked**,305 - 0 indicates the head is **masked**.306 """,307 "shape": "of shape `(num_layers, num_heads)`",308 }309 310 decoder_attention_mask = {311 "description": """312 Mask to avoid performing attention on certain token indices. By default, a causal mask will be used, to313 make sure the model can only look at previous inputs in order to predict the future.314 """,315 "shape": "of shape `(batch_size, target_sequence_length)`",316 }317 318 decoder_head_mask = {319 "description": """320 Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`:321 322 - 1 indicates the head is **not masked**,323 - 0 indicates the head is **masked**.324 """,325 "shape": "of shape `(decoder_layers, decoder_attention_heads)`",326 }327 328 encoder_hidden_states = {329 "description": """330 Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention331 if the model is configured as a decoder.332 """,333 "shape": "of shape `(batch_size, sequence_length, hidden_size)`",334 }335 336 encoder_attention_mask = {337 "description": """338 Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in339 the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:340 341 - 1 for tokens that are **not masked**,342 - 0 for tokens that are **masked**.343 """,344 "shape": "of shape `(batch_size, sequence_length)`",345 }346 347 token_type_ids = {348 "description": """349 Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:350 351 - 0 corresponds to a *sentence A* token,352 - 1 corresponds to a *sentence B* token.353 354 [What are token type IDs?](../glossary#token-type-ids)355 """,356 "shape": "of shape `(batch_size, sequence_length)`",357 }358 359 position_ids = {360 "description": """361 Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`.362 363 [What are position IDs?](../glossary#position-ids)364 """,365 "shape": "of shape `(batch_size, sequence_length)`",366 }367 368 past_key_values = {369 "description": """370 Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention371 blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`372 returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.373 374 Only [`~cache_utils.Cache`] instance is allowed as input, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).375 If no `past_key_values` are passed, [`~cache_utils.DynamicCache`] will be initialized by default.376 377 The model will output the same cache format that is fed as input.378 379 If `past_key_values` are used, the user is expected to input only unprocessed `input_ids` (those that don't380 have their past key value states given to this model) of shape `(batch_size, unprocessed_length)` instead of all `input_ids`381 of shape `(batch_size, sequence_length)`.382 """,383 "shape": None,384 }385 386 inputs_embeds = {387 "description": """388 Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This389 is useful if you want more control over how to convert `input_ids` indices into associated vectors than the390 model's internal embedding lookup matrix.391 """,392 "shape": "of shape `(batch_size, sequence_length, hidden_size)`",393 }394 395 decoder_input_ids = {396 "description": """397 Indices of decoder input sequence tokens in the vocabulary.398 399 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and400 [`PreTrainedTokenizer.__call__`] for details.401 402 [What are decoder input IDs?](../glossary#decoder-input-ids)403 """,404 "shape": "of shape `(batch_size, target_sequence_length)`",405 }406 407 decoder_inputs_embeds = {408 "description": """409 Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded410 representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be411 input (see `past_key_values`). This is useful if you want more control over how to convert412 `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.413 414 If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value415 of `inputs_embeds`.416 """,417 "shape": "of shape `(batch_size, target_sequence_length, hidden_size)`",418 }419 420 use_cache = {421 "description": """422 If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see423 `past_key_values`).424 """,425 "shape": None,426 }427 428 output_attentions = {429 "description": """430 Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned431 tensors for more detail.432 """,433 "shape": None,434 }435 436 output_hidden_states = {437 "description": """438 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for439 more detail.440 """,441 "shape": None,442 }443 444 return_dict = {445 "description": """446 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.447 """,448 "shape": None,449 }450 451 cache_position = {452 "description": """453 Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,454 this tensor is not affected by padding. It is used to update the cache in the correct position and to infer455 the complete sequence length.456 """,457 "shape": "of shape `(sequence_length)`",458 }459 460 hidden_states = {461 "description": """ input to the layer of shape `(batch, seq_len, embed_dim)""",462 "shape": None,463 }464 465 interpolate_pos_encoding = {466 "description": """467 Whether to interpolate the pre-trained position encodings.468 """,469 "shape": None,470 }471 472 position_embeddings = {473 "description": """474 Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,475 with `head_dim` being the embedding dimension of each attention head.476 """,477 "shape": None,478 }479 480 config = {481 "description": """482 Model configuration class with all the parameters of the model. Initializing with a config file does not483 load the weights associated with the model, only the configuration. Check out the484 [`~PreTrainedModel.from_pretrained`] method to load the model weights.485 """,486 "shape": None,487 }488 489 start_positions = {490 "description": """491 Labels for position (index) of the start of the labelled span for computing the token classification loss.492 Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence493 are not taken into account for computing the loss.494 """,495 "shape": "of shape `(batch_size,)`",496 }497 498 end_positions = {499 "description": """500 Labels for position (index) of the end of the labelled span for computing the token classification loss.501 Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence502 are not taken into account for computing the loss.503 """,504 "shape": "of shape `(batch_size,)`",505 }506 507 encoder_outputs = {508 "description": """509 Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)510 `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of511 hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.512 """,513 "shape": None,514 }515 516 output_router_logits = {517 "description": """518 Whether or not to return the logits of all the routers. They are useful for computing the router loss, and519 should not be returned during inference.520 """,521 "shape": None,522 }523 524 logits_to_keep = {525 "description": """526 If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all527 `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that528 token can save memory, which becomes pretty significant for long sequences or large vocabulary size.529 If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension.530 This is useful when using packed tensor format (single dimension for batch and sequence length).531 """,532 "shape": None,533 }534 535 pixel_values = {536 "description": """537 The tensors corresponding to the input images. Pixel values can be obtained using538 [`{image_processor_class}`]. See [`{image_processor_class}.__call__`] for details ([`{processor_class}`] uses539 [`{image_processor_class}`] for processing images).540 """,541 "shape": "of shape `(batch_size, num_channels, image_size, image_size)`",542 }543 544 pixel_values_videos = {545 "description": """546 The tensors corresponding to the input video. Pixel values for videos can be obtained using547 [`{video_processor_class}`]. See [`{video_processor_class}.__call__`] for details ([`{processor_class}`] uses548 [`{video_processor_class}`] for processing videos).549 """,550 "shape": "of shape `(batch_size, num_frames, num_channels, frame_size, frame_size)`",551 }552 553 vision_feature_layer = {554 "description": """555 The index of the layer to select the vision feature. If multiple indices are provided,556 the vision feature of the corresponding indices will be concatenated to form the557 vision features.558 """,559 "shape": None,560 }561 562 vision_feature_select_strategy = {563 "description": """564 The feature selection strategy used to select the vision feature from the vision backbone.565 Can be one of `"default"` or `"full"`.566 """,567 "shape": None,568 }569 570 image_sizes = {571 "description": """572 The sizes of the images in the batch, being (height, width) for each image.573 """,574 "shape": "of shape `(batch_size, 2)`",575 }576 577 pixel_mask = {578 "description": """579 Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`:580 581 - 1 for pixels that are real (i.e. **not masked**),582 - 0 for pixels that are padding (i.e. **masked**).583 584 [What are attention masks?](../glossary#attention-mask)585 """,586 "shape": "of shape `(batch_size, height, width)`",587 }588 589 input_features = {590 "description": """591 The tensors corresponding to the input audio features. Audio features can be obtained using592 [`{feature_extractor_class}`]. See [`{feature_extractor_class}.__call__`] for details ([`{processor_class}`] uses593 [`{feature_extractor_class}`] for processing audios).594 """,595 "shape": "of shape `(batch_size, sequence_length, feature_dim)`",596 }597 598 599class ModelOutputArgs:600 last_hidden_state = {601 "description": """602 Sequence of hidden-states at the output of the last layer of the model.603 """,604 "shape": "of shape `(batch_size, sequence_length, hidden_size)`",605 }606 607 past_key_values = {608 "description": """609 It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).610 611 Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if612 `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`613 input) to speed up sequential decoding.614 """,615 "shape": None,616 "additional_info": "returned when `use_cache=True` is passed or when `config.use_cache=True`",617 }618 619 hidden_states = {620 "description": """621 Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +622 one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.623 624 Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.625 """,626 "shape": None,627 "additional_info": "returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`",628 }629 630 attentions = {631 "description": """632 Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,633 sequence_length)`.634 635 Attentions weights after the attention softmax, used to compute the weighted average in the self-attention636 heads.637 """,638 "shape": None,639 "additional_info": "returned when `output_attentions=True` is passed or when `config.output_attentions=True`",640 }641 642 pooler_output = {643 "description": """644 Last layer hidden-state after a pooling operation on the spatial dimensions.645 """,646 "shape": "of shape `(batch_size, hidden_size)`",647 }648 649 cross_attentions = {650 "description": """651 Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,652 sequence_length)`.653 654 Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the655 weighted average in the cross-attention heads.656 """,657 "shape": None,658 "additional_info": "returned when `output_attentions=True` is passed or when `config.output_attentions=True`",659 }660 661 decoder_hidden_states = {662 "description": """663 Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +664 one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.665 666 Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.667 """,668 "shape": None,669 "additional_info": "returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`",670 }671 672 decoder_attentions = {673 "description": """674 Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,675 sequence_length)`.676 677 Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the678 self-attention heads.679 """,680 "shape": None,681 "additional_info": "returned when `output_attentions=True` is passed or when `config.output_attentions=True`",682 }683 684 encoder_last_hidden_state = {685 "description": """686 Sequence of hidden-states at the output of the last layer of the encoder of the model.687 """,688 "shape": "of shape `(batch_size, sequence_length, hidden_size)`",689 }690 691 encoder_hidden_states = {692 "description": """693 Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +694 one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.695 696 Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.697 """,698 "shape": None,699 "additional_info": "returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`",700 }701 702 encoder_attentions = {703 "description": """704 Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,705 sequence_length)`.706 707 Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the708 self-attention heads.709 """,710 "shape": None,711 "additional_info": "returned when `output_attentions=True` is passed or when `config.output_attentions=True`",712 }713 714 router_logits = {715 "description": """716 Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, sequence_length, num_experts)`.717 718 Router logits of the model, useful to compute the auxiliary loss for Mixture of Experts models.719 """,720 "shape": None,721 "additional_info": "returned when `output_router_logits=True` is passed or when `config.add_router_probs=True`",722 }723 724 router_probs = {725 "description": """726 Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, sequence_length, num_experts)`.727 728 Raw router probabilities that are computed by MoE routers, these terms are used to compute the auxiliary729 loss and the z_loss for Mixture of Experts models.730 """,731 "shape": None,732 "additional_info": "returned when `output_router_probs=True` and `config.add_router_probs=True` is passed or when `config.output_router_probs=True`",733 }734 735 z_loss = {736 "description": """737 z_loss for the sparse modules.738 """,739 "shape": None,740 "additional_info": "returned when `labels` is provided",741 }742 743 aux_loss = {744 "description": """745 aux_loss for the sparse modules.746 """,747 "shape": None,748 "additional_info": "returned when `labels` is provided",749 }750 751 start_logits = {752 "description": """753 Span-start scores (before SoftMax).754 """,755 "shape": "of shape `(batch_size, sequence_length)`",756 }757 758 end_logits = {759 "description": """760 Span-end scores (before SoftMax).761 """,762 "shape": "of shape `(batch_size, sequence_length)`",763 }764 765 feature_maps = {766 "description": """767 Feature maps of the stages.768 """,769 "shape": "of shape `(batch_size, num_channels, height, width)`",770 }771 772 reconstruction = {773 "description": """774 Reconstructed / completed images.775 """,776 "shape": "of shape `(batch_size, num_channels, height, width)`",777 }778 779 spectrogram = {780 "description": """781 The predicted spectrogram.782 """,783 "shape": "of shape `(batch_size, sequence_length, num_bins)`",784 }785 786 predicted_depth = {787 "description": """788 Predicted depth for each pixel.789 """,790 "shape": "of shape `(batch_size, height, width)`",791 }792 793 sequences = {794 "description": """795 Sampled values from the chosen distribution.796 """,797 "shape": "of shape `(batch_size, num_samples, prediction_length)` or `(batch_size, num_samples, prediction_length, input_size)`",798 }799 800 params = {801 "description": """802 Parameters of the chosen distribution.803 """,804 "shape": "of shape `(batch_size, num_samples, num_params)`",805 }806 807 loc = {808 "description": """809 Shift values of each time series' context window which is used to give the model inputs of the same810 magnitude and then used to shift back to the original magnitude.811 """,812 "shape": "of shape `(batch_size,)` or `(batch_size, input_size)`",813 }814 815 scale = {816 "description": """817 Scaling values of each time series' context window which is used to give the model inputs of the same818 magnitude and then used to rescale back to the original magnitude.819 """,820 "shape": "of shape `(batch_size,)` or `(batch_size, input_size)`",821 }822 823 static_features = {824 "description": """825 Static features of each time series' in a batch which are copied to the covariates at inference time.826 """,827 "shape": "of shape `(batch_size, feature size)`",828 }829 830 embeddings = {831 "description": """832 Utterance embeddings used for vector similarity-based retrieval.833 """,834 "shape": "of shape `(batch_size, config.xvector_output_dim)`",835 }836 837 extract_features = {838 "description": """839 Sequence of extracted feature vectors of the last convolutional layer of the model.840 """,841 "shape": "of shape `(batch_size, sequence_length, conv_dim[-1])`",842 }843 844 projection_state = {845 "description": """846 Text embeddings before the projection layer, used to mimic the last hidden state of the teacher encoder.847 """,848 "shape": "of shape `(batch_size,config.project_dim)`",849 }850 851 image_hidden_states = {852 "description": """853 Image hidden states of the model produced by the vision encoder and after projecting the last hidden state.854 """,855 "shape": "of shape `(batch_size, num_images, sequence_length, hidden_size)`",856 }857 858 video_hidden_states = {859 "description": """860 Video hidden states of the model produced by the vision encoder and after projecting the last hidden state.861 """,862 "shape": "of shape `(batch_size * num_frames, num_images, sequence_length, hidden_size)`",863 }864 865 866class ClassDocstring:867 PreTrainedModel = r"""868 This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the869 library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads870 etc.)871 872 This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.873 Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage874 and behavior.875 """876 877 Model = r"""878 The bare {model_name} Model outputting raw hidden-states without any specific head on top.879 """880 881 ForPreTraining = r"""882 The {model_name} Model with a specified pretraining head on top.883 """884 885 Decoder = r"""886 The bare {model_name} Decoder outputting raw hidden-states without any specific head on top.887 """888 889 TextModel = r"""890 The bare {model_name} Text Model outputting raw hidden-states without any specific head on to.891 """892 893 ForSequenceClassification = r"""894 The {model_name} Model with a sequence classification/regression head on top e.g. for GLUE tasks.895 """896 897 ForQuestionAnswering = r"""898 The {model_name} transformer with a span classification head on top for extractive question-answering tasks like899 SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`).900 """901 902 ForMultipleChoice = r"""903 The {model_name} Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a904 softmax) e.g. for RocStories/SWAG tasks.905 """906 907 ForMaskedLM = r"""908 The {model_name} Model with a `language modeling` head on top."909 """910 911 ForTokenClassification = r"""912 The {model_name} transformer with a token classification head on top (a linear layer on top of the hidden-states913 output) e.g. for Named-Entity-Recognition (NER) tasks.914 """915 916 ForConditionalGeneration = r"""917 The {model_name} Model for token generation conditioned on other modalities (e.g. image-text-to-text generation).918 """919 920 ForCausalLM = r"""921 The {model_name} Model for causal language modeling.922 """923 924 ImageProcessorFast = r"""925 Constructs a fast {model_name} image processor.926 """927 928 Backbone = r"""929 The {model_name} backbone.930 """931 932 ForImageClassification = r"""933 The {model_name} Model with an image classification head on top e.g. for ImageNet.934 """935 ForSemanticSegmentation = r"""936 The {model_name} Model with a semantic segmentation head on top e.g. for ADE20K, CityScapes.937 """938 ForAudioClassification = r"""939 The {model_name} Model with an audio classification head on top (a linear layer on top of the pooled940 output).941 """942 943 ForAudioFrameClassification = r"""944 The {model_name} Model with a frame classification head on top for tasks like Speaker Diarization.945 """946 947 ForPrediction = r"""948 The {model_name} Model with a distribution head on top for time-series forecasting.949 """950 951 WithProjection = r"""952 The {model_name} Model with a projection layer on top (a linear layer on top of the pooled output).953 """954 955 956class ClassAttrs:957 # fmt: off958 base_model_prefix = r"""959 A string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model.960 """961 supports_gradient_checkpointing = r"""962 Whether the model supports gradient checkpointing or not. Gradient checkpointing is a memory-saving technique that trades compute for memory, by storing only a subset of activations (checkpoints) and recomputing the activations that are not stored during the backward pass.963 """964 _no_split_modules = r"""965 Layers of modules that should not be split across devices should be added to `_no_split_modules`. This can be useful for modules that contains skip connections or other operations that are not compatible with splitting the module across devices. Setting this attribute will enable the use of `device_map="auto"` in the `from_pretrained` method.966 """967 _skip_keys_device_placement = r"""968 A list of keys to ignore when moving inputs or outputs between devices when using the `accelerate` library.969 """970 _supports_flash_attn = r"""971 Whether the model's attention implementation supports FlashAttention.972 """973 _supports_sdpa = r"""974 Whether the model's attention implementation supports SDPA (Scaled Dot Product Attention).975 """976 _supports_flex_attn = r"""977 Whether the model's attention implementation supports FlexAttention.978 """979 _can_compile_fullgraph = r"""980 Whether the model can `torch.compile` fullgraph without graph breaks. Models will auto-compile if this flag is set to `True`981 in inference, if a compilable cache is used.982 """983 _supports_attention_backend = r"""984 Whether the model supports attention interface functions. This flag signal that the model can be used as an efficient backend in TGI and vLLM.985 """986 _tied_weights_keys = r"""987 A list of `state_dict` keys that are potentially tied to another key in the state_dict.988 """989 # fmt: on990 991 992ARGS_TO_IGNORE = {"self", "kwargs", "args", "deprecated_arguments"}993 994 995def get_indent_level(func):996 # Use this instead of `inspect.getsource(func)` as getsource can be very slow997 return (len(func.__qualname__.split(".")) - 1) * 4998 999 1000def equalize_indent(docstring, indent_level):1001 """1002 Adjust the indentation of a docstring to match the specified indent level.1003 """1004 # fully dedent the docstring1005 docstring = "\n".join([line.lstrip() for line in docstring.splitlines()])1006 return textwrap.indent(docstring, " " * indent_level)1007 1008 1009def set_min_indent(docstring, indent_level):1010 """1011 Adjust the indentation of a docstring to match the specified indent level.1012 """1013 return textwrap.indent(textwrap.dedent(docstring), " " * indent_level)1014 1015 1016def parse_shape(docstring):1017 shape_pattern = re.compile(r"(of shape\s*(?:`.*?`|\(.*?\)))")1018 match = shape_pattern.search(docstring)1019 if match:1020 return " " + match.group(1)1021 return None1022 1023 1024def parse_default(docstring):1025 default_pattern = re.compile(r"(defaults to \s*[^)]*)")1026 match = default_pattern.search(docstring)1027 if match:1028 return " " + match.group(1)1029 return None1030 1031 1032def parse_docstring(docstring, max_indent_level=0, return_intro=False):1033 """1034 Parse the docstring to extract the Args section and return it as a dictionary.1035 The docstring is expected to be in the format:1036 Args:1037 arg1 (type):1038 Description of arg1.1039 arg2 (type):1040 Description of arg2.1041 1042 # This function will also return the remaining part of the docstring after the Args section.1043 Returns:/Example:1044 ...1045 """1046 match = re.search(r"(?m)^([ \t]*)(?=Example|Return)", docstring)1047 if match:1048 remainder_docstring = docstring[match.start() :]1049 docstring = docstring[: match.start()]1050 else:1051 remainder_docstring = ""1052 args_pattern = re.compile(r"(?:Args:)(\n.*)?(\n)?$", re.DOTALL)1053 1054 args_match = args_pattern.search(docstring)1055 # still try to find args description in the docstring, if args are not preceded by "Args:"1056 docstring_intro = None1057 if args_match:1058 docstring_intro = docstring[: args_match.start()]1059 if docstring_intro.split("\n")[-1].strip() == '"""':1060 docstring_intro = "\n".join(docstring_intro.split("\n")[:-1])1061 if docstring_intro.split("\n")[0].strip() == 'r"""' or docstring_intro.split("\n")[0].strip() == '"""':1062 docstring_intro = "\n".join(docstring_intro.split("\n")[1:])1063 if docstring_intro.strip() == "":1064 docstring_intro = None1065 args_section = args_match.group(1).lstrip("\n") if args_match else docstring1066 if args_section.split("\n")[-1].strip() == '"""':1067 args_section = "\n".join(args_section.split("\n")[:-1])1068 if args_section.split("\n")[0].strip() == 'r"""' or args_section.split("\n")[0].strip() == '"""':1069 args_section = "\n".join(args_section.split("\n")[1:])1070 args_section = set_min_indent(args_section, 0)1071 params = {}1072 if args_section:1073 param_pattern = re.compile(1074 # |--- Group 1 ---|| Group 2 ||- Group 3 -||---------- Group 4 ----------|1075 rf"^\s{{0,{max_indent_level}}}(\w+)\s*\(\s*([^, \)]*)(\s*.*?)\s*\)\s*:\s*((?:(?!\n^\s{{0,{max_indent_level}}}\w+\s*\().)*)",1076 re.DOTALL | re.MULTILINE,1077 )1078 for match in param_pattern.finditer(args_section):1079 param_name = match.group(1)1080 param_type = match.group(2)1081 # param_type = match.group(2).replace("`", "")1082 additional_info = match.group(3)1083 optional = "optional" in additional_info1084 shape = parse_shape(additional_info)1085 default = parse_default(additional_info)1086 param_description = match.group(4).strip()1087 # set first line of param_description to 4 spaces:1088 param_description = re.sub(r"^", " " * 4, param_description, 1)1089 param_description = f"\n{param_description}"1090 params[param_name] = {1091 "type": param_type,1092 "description": param_description,1093 "optional": optional,1094 "shape": shape,1095 "default": default,1096 "additional_info": additional_info,1097 }1098 1099 if params and remainder_docstring:1100 remainder_docstring = "\n" + remainder_docstring1101 1102 remainder_docstring = set_min_indent(remainder_docstring, 0)1103 1104 if return_intro:1105 return params, remainder_docstring, docstring_intro1106 return params, remainder_docstring1107 1108 1109def contains_type(type_hint, target_type) -> tuple[bool, Optional[object]]:1110 """1111 Check if a "nested" type hint contains a specific target type,1112 return the first-level type containing the target_type if found.1113 """1114 args = get_args(type_hint)1115 if args == ():1116 try:1117 return issubclass(type_hint, target_type), type_hint1118 except Exception:1119 return issubclass(type(type_hint), target_type), type_hint1120 found_type_tuple = [contains_type(arg, target_type)[0] for arg in args]1121 found_type = any(found_type_tuple)1122 if found_type:1123 type_hint = args[found_type_tuple.index(True)]1124 return found_type, type_hint1125 1126 1127def get_model_name(obj):1128 """1129 Get the model name from the file path of the object.1130 """1131 path = inspect.getsourcefile(obj)1132 if path is None:1133 return None1134 if path.split(os.path.sep)[-3] != "models":1135 return None1136 file_name = path.split(os.path.sep)[-1]1137 for file_type in AUTODOC_FILES:1138 start = file_type.split("*")[0]1139 end = file_type.split("*")[-1] if "*" in file_type else ""1140 if file_name.startswith(start) and file_name.endswith(end):1141 model_name_lowercase = file_name[len(start) : -len(end)]1142 return model_name_lowercase1143 print(f"🚨 Something went wrong trying to find the model name in the path: {path}")1144 return "model"1145 1146 1147def get_placeholders_dict(placeholders: list, model_name: str) -> dict:1148 """1149 Get the dictionary of placeholders for the given model name.1150 """1151 # import here to avoid circular import1152 from transformers.models import auto as auto_module1153 1154 placeholders_dict = {}1155 for placeholder in placeholders:1156 # Infer placeholders from the model name and the auto modules1157 if placeholder in PLACEHOLDER_TO_AUTO_MODULE:1158 try:1159 place_holder_value = getattr(1160 getattr(auto_module, PLACEHOLDER_TO_AUTO_MODULE[placeholder][0]),1161 PLACEHOLDER_TO_AUTO_MODULE[placeholder][1],1162 ).get(model_name, None)1163 except ImportError:1164 # In case a library is not installed, we don't want to fail the docstring generation1165 place_holder_value = None1166 if place_holder_value is not None:1167 if isinstance(place_holder_value, (list, tuple)):1168 place_holder_value = place_holder_value[0]1169 placeholders_dict[placeholder] = place_holder_value if place_holder_value is not None else placeholder1170 else:1171 placeholders_dict[placeholder] = placeholder1172 1173 return placeholders_dict1174 1175 1176def format_args_docstring(docstring, model_name):1177 """1178 Replaces placeholders such as {image_processor_class} in the docstring with the actual values,1179 deducted from the model name and the auto modules.1180 """1181 # first check if there are any placeholders in the docstring, if not return it as is1182 placeholders = set(re.findall(r"{(.*?)}", docstring))1183 if not placeholders:1184 return docstring1185 1186 # get the placeholders dictionary for the given model name1187 placeholders_dict = get_placeholders_dict(placeholders, model_name)1188 # replace the placeholders in the docstring with the values from the placeholders_dict1189 for placeholder, value in placeholders_dict.items():1190 if placeholder is not None:1191 try:1192 docstring = docstring.replace(f"{{{placeholder}}}", value)1193 except Exception:1194 pass1195 return docstring1196 1197 1198def get_args_doc_from_source(args_classes: Union[object, list[object]]) -> dict:1199 if isinstance(args_classes, (list, tuple)):1200 args_classes_dict = {}