Aluode/PerceptionLabPortable

1#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨2#           This file was automatically generated from src/transformers/models/granite/modular_granite.py.3#               Do NOT edit this file manually as any edits will be overwritten by the generation of4#             the file from the modular. If any change should be done, please apply the change to the5#                          modular_granite.py file directly. One of our CI enforces this.6#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨7# coding=utf-88# Copyright 2024 IBM and the HuggingFace Inc. team. All rights reserved.9#10#11# Licensed under the Apache License, Version 2.0 (the "License");12# you may not use this file except in compliance with the License.13# You may obtain a copy of the License at14#15#     http://www.apache.org/licenses/LICENSE-2.016#17# Unless required by applicable law or agreed to in writing, software18# distributed under the License is distributed on an "AS IS" BASIS,19# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.20# See the License for the specific language governing permissions and21# limitations under the License.22from typing import Callable, Optional, Union23 24import torch25from torch import nn26 27from ...activations import ACT2FN28from ...cache_utils import Cache, DynamicCache29from ...generation import GenerationMixin30from ...integrations import use_kernel_forward_from_hub31from ...masking_utils import create_causal_mask32from ...modeling_layers import GradientCheckpointingLayer33from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast34from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update35from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel36from ...processing_utils import Unpack37from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging38from ...utils.deprecation import deprecate_kwarg39from ...utils.generic import check_model_inputs40from .configuration_granite import GraniteConfig41 42 43logger = logging.get_logger(__name__)44 45 46def rotate_half(x):47    """Rotates half the hidden dims of the input."""48    x1 = x[..., : x.shape[-1] // 2]49    x2 = x[..., x.shape[-1] // 2 :]50    return torch.cat((-x2, x1), dim=-1)51 52 53def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):54    """Applies Rotary Position Embedding to the query and key tensors.55 56    Args:57        q (`torch.Tensor`): The query tensor.58        k (`torch.Tensor`): The key tensor.59        cos (`torch.Tensor`): The cosine part of the rotary embedding.60        sin (`torch.Tensor`): The sine part of the rotary embedding.61        position_ids (`torch.Tensor`, *optional*):62            Deprecated and unused.63        unsqueeze_dim (`int`, *optional*, defaults to 1):64            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and65            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note66            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and67            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes68            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have69            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.70    Returns:71        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.72    """73    cos = cos.unsqueeze(unsqueeze_dim)74    sin = sin.unsqueeze(unsqueeze_dim)75    q_embed = (q * cos) + (rotate_half(q) * sin)76    k_embed = (k * cos) + (rotate_half(k) * sin)77    return q_embed, k_embed78 79 80def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:81    """82    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,83    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)84    """85    batch, num_key_value_heads, slen, head_dim = hidden_states.shape86    if n_rep == 1:87        return hidden_states88    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)89    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)90 91 92def eager_attention_forward(93    module: nn.Module,94    query: torch.Tensor,95    key: torch.Tensor,96    value: torch.Tensor,97    attention_mask: Optional[torch.Tensor],98    scaling: float,99    dropout: float = 0.0,100    **kwargs: Unpack[TransformersKwargs],101):102    key_states = repeat_kv(key, module.num_key_value_groups)103    value_states = repeat_kv(value, module.num_key_value_groups)104 105    attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling106    if attention_mask is not None:107        causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]108        attn_weights = attn_weights + causal_mask109 110    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)111    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)112    attn_output = torch.matmul(attn_weights, value_states)113    attn_output = attn_output.transpose(1, 2).contiguous()114 115    return attn_output, attn_weights116 117 118class GraniteAttention(nn.Module):119    """Multi-headed attention from 'Attention Is All You Need' paper"""120 121    def __init__(self, config: GraniteConfig, layer_idx: Optional[int] = None):122        super().__init__()123        self.config = config124        self.layer_idx = layer_idx125        self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)126        self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads127        self.scaling = config.attention_multiplier128        self.attention_dropout = config.attention_dropout129        self.is_causal = True130 131        self.q_proj = nn.Linear(132            config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias133        )134        self.k_proj = nn.Linear(135            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias136        )137        self.v_proj = nn.Linear(138            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias139        )140        self.o_proj = nn.Linear(141            config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias142        )143 144    @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")145    def forward(146        self,147        hidden_states: torch.Tensor,148        position_embeddings: tuple[torch.Tensor, torch.Tensor],149        attention_mask: Optional[torch.Tensor],150        past_key_values: Optional[Cache] = None,151        cache_position: Optional[torch.LongTensor] = None,152        **kwargs: Unpack[TransformersKwargs],153    ) -> tuple[torch.Tensor, torch.Tensor]:154        input_shape = hidden_states.shape[:-1]155        hidden_shape = (*input_shape, -1, self.head_dim)156 157        query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)158        key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)159        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)160 161        cos, sin = position_embeddings162        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)163 164        if past_key_values is not None:165            # sin and cos are specific to RoPE models; cache_position needed for the static cache166            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}167            key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs)168 169        attention_interface: Callable = eager_attention_forward170        if self.config._attn_implementation != "eager":171            attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]172 173        attn_output, attn_weights = attention_interface(174            self,175            query_states,176            key_states,177            value_states,178            attention_mask,179            dropout=0.0 if not self.training else self.attention_dropout,180            scaling=self.scaling,181            **kwargs,182        )183 184        attn_output = attn_output.reshape(*input_shape, -1).contiguous()185        attn_output = self.o_proj(attn_output)186        return attn_output, attn_weights187 188 189@use_kernel_forward_from_hub("RMSNorm")190class GraniteRMSNorm(nn.Module):191    def __init__(self, hidden_size, eps=1e-6):192        """193        GraniteRMSNorm is equivalent to T5LayerNorm194        """195        super().__init__()196        self.weight = nn.Parameter(torch.ones(hidden_size))197        self.variance_epsilon = eps198 199    def forward(self, hidden_states):200        input_dtype = hidden_states.dtype201        hidden_states = hidden_states.to(torch.float32)202        variance = hidden_states.pow(2).mean(-1, keepdim=True)203        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)204        return self.weight * hidden_states.to(input_dtype)205 206    def extra_repr(self):207        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"208 209 210class GraniteMLP(nn.Module):211    def __init__(self, config):212        super().__init__()213        self.config = config214        self.hidden_size = config.hidden_size215        self.intermediate_size = config.intermediate_size216        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)217        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)218        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.mlp_bias)219        self.act_fn = ACT2FN[config.hidden_act]220 221    def forward(self, x):222        down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))223        return down_proj224 225 226class GraniteDecoderLayer(GradientCheckpointingLayer):227    def __init__(self, config: GraniteConfig, layer_idx: int):228        super().__init__()229        self.hidden_size = config.hidden_size230        self.self_attn = GraniteAttention(config=config, layer_idx=layer_idx)231 232        self.mlp = GraniteMLP(config)233        self.input_layernorm = GraniteRMSNorm(config.hidden_size, eps=config.rms_norm_eps)234        self.post_attention_layernorm = GraniteRMSNorm(config.hidden_size, eps=config.rms_norm_eps)235        self.residual_multiplier = config.residual_multiplier236 237    @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")238    def forward(239        self,240        hidden_states: torch.Tensor,241        attention_mask: Optional[torch.Tensor] = None,242        position_ids: Optional[torch.LongTensor] = None,243        past_key_values: Optional[Cache] = None,244        output_attentions: Optional[bool] = False,245        use_cache: Optional[bool] = False,246        cache_position: Optional[torch.LongTensor] = None,247        position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC248        **kwargs,249    ) -> tuple[torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]]]:250        """251        Args:252            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`253            attention_mask (`torch.FloatTensor`, *optional*):254                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,255                query_sequence_length, key_sequence_length)` if default attention is used.256            output_attentions (`bool`, *optional*):257                Whether or not to return the attentions tensors of all attention layers. See `attentions` under258                returned tensors for more detail.259            use_cache (`bool`, *optional*):260                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding261                (see `past_key_values`).262            past_key_values (`Cache`, *optional*): cached past key and value projection states263            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):264                Indices depicting the position of the input sequence tokens in the sequence265            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):266                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,267                with `head_dim` being the embedding dimension of each attention head.268            kwargs (`dict`, *optional*):269                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code270                into the model271        """272        residual = hidden_states273 274        hidden_states = self.input_layernorm(hidden_states)275 276        # Self Attention277        hidden_states, self_attn_weights = self.self_attn(278            hidden_states=hidden_states,279            attention_mask=attention_mask,280            position_ids=position_ids,281            past_key_values=past_key_values,282            output_attentions=output_attentions,283            use_cache=use_cache,284            cache_position=cache_position,285            position_embeddings=position_embeddings,286            **kwargs,287        )288        hidden_states = residual + hidden_states * self.residual_multiplier289 290        # Fully Connected291        residual = hidden_states292        hidden_states = self.post_attention_layernorm(hidden_states)293        hidden_states = self.mlp(hidden_states)294        hidden_states = residual + hidden_states * self.residual_multiplier  # main diff with Llama295 296        outputs = (hidden_states,)297 298        if output_attentions:299            outputs += (self_attn_weights,)300 301        return outputs302 303 304@auto_docstring305class GranitePreTrainedModel(PreTrainedModel):306    config: GraniteConfig307    base_model_prefix = "model"308    supports_gradient_checkpointing = True309    _no_split_modules = ["GraniteDecoderLayer"]310    _skip_keys_device_placement = ["past_key_values"]311    _supports_flash_attn = True312    _supports_sdpa = True313    _supports_flex_attn = True314 315    _can_compile_fullgraph = True316    _supports_attention_backend = True317    _can_record_outputs = {318        "hidden_states": GraniteDecoderLayer,319        "attentions": GraniteAttention,320    }321 322 323class GraniteRotaryEmbedding(nn.Module):324    inv_freq: torch.Tensor  # fix linting for `register_buffer`325 326    def __init__(self, config: GraniteConfig, device=None):327        super().__init__()328        # BC: "rope_type" was originally "type"329        if hasattr(config, "rope_scaling") and isinstance(config.rope_scaling, dict):330            self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))331        else:332            self.rope_type = "default"333        self.max_seq_len_cached = config.max_position_embeddings334        self.original_max_seq_len = config.max_position_embeddings335 336        self.config = config337        self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]338 339        inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)340        self.register_buffer("inv_freq", inv_freq, persistent=False)341        self.original_inv_freq = self.inv_freq342 343    @torch.no_grad()344    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)345    def forward(self, x, position_ids):346        inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)347        position_ids_expanded = position_ids[:, None, :].float()348 349        device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"350        with torch.autocast(device_type=device_type, enabled=False):  # Force float32351            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)352            emb = torch.cat((freqs, freqs), dim=-1)353            cos = emb.cos() * self.attention_scaling354            sin = emb.sin() * self.attention_scaling355 356        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)357 358 359@auto_docstring360class GraniteModel(GranitePreTrainedModel):361    def __init__(self, config: GraniteConfig):362        super().__init__(config)363        self.padding_idx = config.pad_token_id364        self.vocab_size = config.vocab_size365 366        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)367        self.layers = nn.ModuleList(368            [GraniteDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]369        )370        self.norm = GraniteRMSNorm(config.hidden_size, eps=config.rms_norm_eps)371        self.rotary_emb = GraniteRotaryEmbedding(config=config)372        self.gradient_checkpointing = False373        self.embedding_multiplier = config.embedding_multiplier374 375        # Initialize weights and apply final processing376        self.post_init()377 378    @check_model_inputs()379    @auto_docstring380    def forward(381        self,382        input_ids: Optional[torch.LongTensor] = None,383        attention_mask: Optional[torch.Tensor] = None,384        position_ids: Optional[torch.LongTensor] = None,385        past_key_values: Optional[Cache] = None,386        inputs_embeds: Optional[torch.FloatTensor] = None,387        use_cache: Optional[bool] = None,388        output_attentions: Optional[bool] = None,389        output_hidden_states: Optional[bool] = None,390        cache_position: Optional[torch.LongTensor] = None,391        **kwargs: Unpack[TransformersKwargs],392    ) -> BaseModelOutputWithPast:393        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions394        output_hidden_states = (395            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states396        )397        use_cache = use_cache if use_cache is not None else self.config.use_cache398 399        if (input_ids is None) ^ (inputs_embeds is not None):400            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")401 402        if self.gradient_checkpointing and self.training and use_cache:403            logger.warning_once(404                "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`."405            )406            use_cache = False407 408        if inputs_embeds is None:409            inputs_embeds = self.embed_tokens(input_ids)410 411        inputs_embeds = inputs_embeds * self.embedding_multiplier  # main diff with Llama412 413        if use_cache and past_key_values is None:414            past_key_values = DynamicCache(config=self.config)415 416        if cache_position is None:417            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0418            cache_position = torch.arange(419                past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device420            )421 422        if position_ids is None:423            position_ids = cache_position.unsqueeze(0)424 425        causal_mask = create_causal_mask(426            config=self.config,427            input_embeds=inputs_embeds,428            attention_mask=attention_mask,429            cache_position=cache_position,430            past_key_values=past_key_values,431            position_ids=position_ids,432        )433 434        hidden_states = inputs_embeds435 436        # create position embeddings to be shared across the decoder layers437        position_embeddings = self.rotary_emb(hidden_states, position_ids)438 439        # decoder layers440        all_hidden_states = () if output_hidden_states else None441        all_self_attns = () if output_attentions else None442 443        for decoder_layer in self.layers[: self.config.num_hidden_layers]:444            if output_hidden_states:445                all_hidden_states += (hidden_states,)446 447            layer_outputs = decoder_layer(448                hidden_states,449                attention_mask=causal_mask,450                position_ids=position_ids,451                past_key_values=past_key_values,452                output_attentions=output_attentions,453                use_cache=use_cache,454                cache_position=cache_position,455                position_embeddings=position_embeddings,456                **kwargs,457            )458 459            hidden_states = layer_outputs[0]460 461            if output_attentions:462                all_self_attns += (layer_outputs[1],)463 464        hidden_states = self.norm(hidden_states)465 466        # add hidden states from the last decoder layer467        if output_hidden_states:468            all_hidden_states += (hidden_states,)469 470        return BaseModelOutputWithPast(471            last_hidden_state=hidden_states,472            past_key_values=past_key_values if use_cache else None,473            hidden_states=all_hidden_states,474            attentions=all_self_attns,475        )476 477 478@auto_docstring479class GraniteForCausalLM(GranitePreTrainedModel, GenerationMixin):480    _tied_weights_keys = ["lm_head.weight"]481    _tp_plan = {"lm_head": "colwise_rep"}482    _pp_plan = {"lm_head": (["hidden_states"], ["logits"])}483 484    def __init__(self, config):485        super().__init__(config)486        self.model = GraniteModel(config)487        self.vocab_size = config.vocab_size488        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)489 490        # Initialize weights and apply final processing491        self.post_init()492 493    @can_return_tuple494    @auto_docstring495    def forward(496        self,497        input_ids: Optional[torch.LongTensor] = None,498        attention_mask: Optional[torch.Tensor] = None,499        position_ids: Optional[torch.LongTensor] = None,500        past_key_values: Optional[Union[Cache, list[torch.FloatTensor]]] = None,501        inputs_embeds: Optional[torch.FloatTensor] = None,502        labels: Optional[torch.LongTensor] = None,503        use_cache: Optional[bool] = None,504        output_attentions: Optional[bool] = None,505        output_hidden_states: Optional[bool] = None,506        cache_position: Optional[torch.LongTensor] = None,507        logits_to_keep: Union[int, torch.Tensor] = 0,508        **kwargs: Unpack[TransformersKwargs],509    ) -> CausalLMOutputWithPast:510        r"""511        Example:512 513        ```python514        >>> from transformers import AutoTokenizer, GraniteForCausalLM515 516        >>> model = GraniteForCausalLM.from_pretrained("meta-granite/Granite-2-7b-hf")517        >>> tokenizer = AutoTokenizer.from_pretrained("meta-granite/Granite-2-7b-hf")518 519        >>> prompt = "Hey, are you conscious? Can you talk to me?"520        >>> inputs = tokenizer(prompt, return_tensors="pt")521 522        >>> # Generate523        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)524        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]525        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."526        ```"""527        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions528        output_hidden_states = (529            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states530        )531 532        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)533        outputs: BaseModelOutputWithPast = self.model(534            input_ids=input_ids,535            attention_mask=attention_mask,536            position_ids=position_ids,537            past_key_values=past_key_values,538            inputs_embeds=inputs_embeds,539            use_cache=use_cache,540            output_attentions=output_attentions,541            output_hidden_states=output_hidden_states,542            cache_position=cache_position,543            **kwargs,544        )545 546        hidden_states = outputs.last_hidden_state547        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss548        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep549        logits = self.lm_head(hidden_states[:, slice_indices, :])550        logits = logits / self.config.logits_scaling  # main diff with Llama551 552        loss = None553        if labels is not None:554            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)555 556        return CausalLMOutputWithPast(557            loss=loss,558            logits=logits,559            past_key_values=outputs.past_key_values,560            hidden_states=outputs.hidden_states,561            attentions=outputs.attentions,562        )563 564 565__all__ = ["GraniteForCausalLM", "GraniteModel", "GranitePreTrainedModel"]566