unsloth/Phi-4-mini-instruct
226.2k
1# coding=utf-82# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16"""PyTorch Phi-3 model."""17 18from typing import Callable, List, Optional, Tuple, Union19 20import torch21from torch import nn22 23from transformers.activations import ACT2FN24from transformers.cache_utils import Cache, DynamicCache, SlidingWindowCache, StaticCache25from transformers.generation import GenerationMixin26from transformers.modeling_attn_mask_utils import AttentionMaskConverter27from transformers.modeling_flash_attention_utils import FlashAttentionKwargs28from transformers.modeling_outputs import (29 BaseModelOutputWithPast,30 CausalLMOutputWithPast,31 SequenceClassifierOutputWithPast,32 TokenClassifierOutput,33)34from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS35from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel36from transformers.processing_utils import Unpack37from transformers.utils import (38 LossKwargs,39 add_code_sample_docstrings,40 add_start_docstrings,41 add_start_docstrings_to_model_forward,42 logging,43 replace_return_docstrings,44)45from transformers.utils.deprecation import deprecate_kwarg46from .configuration_phi3 import Phi3Config47 48 49logger = logging.get_logger(__name__)50 51_CHECKPOINT_FOR_DOC = "microsoft/Phi-3-mini-4k-instruct"52_CONFIG_FOR_DOC = "Phi3Config"53 54 55class Phi3MLP(nn.Module):56 def __init__(self, config):57 super().__init__()58 59 self.config = config60 self.gate_up_proj = nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False)61 self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)62 self.activation_fn = ACT2FN[config.hidden_act]63 64 def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:65 up_states = self.gate_up_proj(hidden_states)66 67 gate, up_states = up_states.chunk(2, dim=-1)68 up_states = up_states * self.activation_fn(gate)69 70 return self.down_proj(up_states)71 72 73def rotate_half(x):74 """Rotates half the hidden dims of the input."""75 x1 = x[..., : x.shape[-1] // 2]76 x2 = x[..., x.shape[-1] // 2 :]77 return torch.cat((-x2, x1), dim=-1)78 79 80def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:81 """82 This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,83 num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)84 """85 batch, num_key_value_heads, slen, head_dim = hidden_states.shape86 if n_rep == 1:87 return hidden_states88 hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)89 return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)90 91 92def eager_attention_forward(93 module: nn.Module,94 query: torch.Tensor,95 key: torch.Tensor,96 value: torch.Tensor,97 attention_mask: Optional[torch.Tensor],98 scaling: float,99 dropout: float = 0.0,100 **kwargs,101):102 key_states = repeat_kv(key, module.num_key_value_groups)103 value_states = repeat_kv(value, module.num_key_value_groups)104 105 attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling106 if attention_mask is not None:107 causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]108 attn_weights = attn_weights + causal_mask109 110 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)111 attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)112 attn_output = torch.matmul(attn_weights, value_states)113 attn_output = attn_output.transpose(1, 2).contiguous()114 115 return attn_output, attn_weights116 117 118def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):119 """Applies Rotary Position Embedding to the query and key tensors.120 121 Args:122 q (`torch.Tensor`): The query tensor.123 k (`torch.Tensor`): The key tensor.124 cos (`torch.Tensor`): The cosine part of the rotary embedding.125 sin (`torch.Tensor`): The sine part of the rotary embedding.126 position_ids (`torch.Tensor`, *optional*):127 Deprecated and unused.128 unsqueeze_dim (`int`, *optional*, defaults to 1):129 The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and130 sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note131 that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and132 k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes133 cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have134 the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.135 Returns:136 `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.137 """138 cos = cos.unsqueeze(unsqueeze_dim)139 sin = sin.unsqueeze(unsqueeze_dim)140 141 rotary_dim = cos.shape[-1]142 q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]143 k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]144 145 q_embed = torch.cat([(q_rot * cos) + (rotate_half(q_rot) * sin), q_pass], dim=-1)146 k_embed = torch.cat([(k_rot * cos) + (rotate_half(k_rot) * sin), k_pass], dim=-1)147 return q_embed, k_embed148 149 150class Phi3Attention(nn.Module):151 """Multi-headed attention from 'Attention Is All You Need' paper"""152 153 def __init__(self, config: Phi3Config, layer_idx: Optional[int] = None):154 super().__init__()155 self.config = config156 self.layer_idx = layer_idx157 self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)158 self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads159 self.num_key_value_heads = config.num_key_value_heads160 self.scaling = self.head_dim**-0.5161 self.attention_dropout = config.attention_dropout162 self.is_causal = True163 164 op_size = config.num_attention_heads * self.head_dim + 2 * (config.num_key_value_heads * self.head_dim)165 self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=False)166 self.qkv_proj = nn.Linear(config.hidden_size, op_size, bias=False)167 168 def forward(169 self,170 hidden_states: torch.Tensor,171 position_embeddings: Tuple[torch.Tensor, torch.Tensor],172 attention_mask: Optional[torch.Tensor],173 past_key_value: Optional[Cache] = None,174 cache_position: Optional[torch.LongTensor] = None,175 **kwargs: Unpack[FlashAttentionKwargs],176 ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:177 input_shape = hidden_states.shape[:-1]178 hidden_shape = (*input_shape, -1, self.head_dim)179 180 qkv = self.qkv_proj(hidden_states)181 query_pos = self.config.num_attention_heads * self.head_dim182 query_states = qkv[..., :query_pos]183 key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim]184 value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :]185 186 query_states = query_states.view(hidden_shape).transpose(1, 2)187 key_states = key_states.view(hidden_shape).transpose(1, 2)188 value_states = value_states.view(hidden_shape).transpose(1, 2)189 190 cos, sin = position_embeddings191 query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)192 193 if past_key_value is not None:194 # sin and cos are specific to RoPE models; cache_position needed for the static cache195 cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}196 key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)197 198 attention_interface: Callable = eager_attention_forward199 if self.config._attn_implementation != "eager":200 if self.config._attn_implementation == "sdpa" and kwargs.get("output_attentions", False):201 logger.warning_once(202 "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to "203 'eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'204 )205 else:206 attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]207 208 attn_output, attn_weights = attention_interface(209 self,210 query_states,211 key_states,212 value_states,213 attention_mask,214 dropout=0.0 if not self.training else self.attention_dropout,215 scaling=self.scaling,216 sliding_window=getattr(self.config, "sliding_window", None),217 **kwargs,218 )219 220 attn_output = attn_output.reshape(*input_shape, -1).contiguous()221 attn_output = self.o_proj(attn_output)222 return attn_output, attn_weights223 224 225class Phi3RMSNorm(nn.Module):226 def __init__(self, hidden_size, eps=1e-6):227 """228 Phi3RMSNorm is equivalent to T5LayerNorm229 """230 super().__init__()231 self.weight = nn.Parameter(torch.ones(hidden_size))232 self.variance_epsilon = eps233 234 def forward(self, hidden_states):235 input_dtype = hidden_states.dtype236 hidden_states = hidden_states.to(torch.float32)237 variance = hidden_states.pow(2).mean(-1, keepdim=True)238 hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)239 return self.weight * hidden_states.to(input_dtype)240 241 def extra_repr(self):242 return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"243 244 245class Phi3DecoderLayer(nn.Module):246 def __init__(self, config: Phi3Config, layer_idx: int):247 super().__init__()248 self.hidden_size = config.hidden_size249 self.self_attn = Phi3Attention(config=config, layer_idx=layer_idx)250 self.mlp = Phi3MLP(config)251 self.input_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)252 self.post_attention_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)253 self.config = config254 self.resid_attn_dropout = nn.Dropout(config.resid_pdrop)255 self.resid_mlp_dropout = nn.Dropout(config.resid_pdrop)256 257 def forward(258 self,259 hidden_states: torch.Tensor,260 attention_mask: Optional[torch.Tensor] = None,261 position_ids: Optional[torch.LongTensor] = None,262 past_key_value: Optional[Cache] = None,263 output_attentions: Optional[bool] = False,264 use_cache: Optional[bool] = False,265 cache_position: Optional[torch.LongTensor] = None,266 position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # necessary, but kept here for BC267 **kwargs: Unpack[FlashAttentionKwargs],268 ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:269 """270 Args:271 hidden_states (`torch.FloatTensor`):272 input to the layer of shape `(batch, seq_len, embed_dim)`273 attention_mask (`torch.FloatTensor`, *optional*): attention mask of size274 `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.275 position_ids (`torch.LongTensor` of shape `({0})`, *optional*):276 Indices of positions of each input sequence tokens in the position embeddings. Selected in the range277 `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)278 past_key_value (`Cache`, *optional*): cached past key and value projection states279 output_attentions (`bool`, *optional*):280 Whether or not to return the attentions tensors of all attention layers. See `attentions` under281 returned tensors for more detail.282 use_cache (`bool`, *optional*):283 If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding284 (see `past_key_values`).285 cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):286 Indices depicting the position of the input sequence tokens in the sequence287 kwargs (`dict`, *optional*):288 Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code289 into the model290 """291 residual = hidden_states292 293 hidden_states = self.input_layernorm(hidden_states)294 295 # Self Attention296 hidden_states, self_attn_weights = self.self_attn(297 hidden_states=hidden_states,298 attention_mask=attention_mask,299 position_ids=position_ids,300 past_key_value=past_key_value,301 output_attentions=output_attentions,302 use_cache=use_cache,303 cache_position=cache_position,304 position_embeddings=position_embeddings,305 **kwargs,306 )307 hidden_states = residual + self.resid_attn_dropout(hidden_states) # main diff with Llama308 309 residual = hidden_states310 hidden_states = self.post_attention_layernorm(hidden_states)311 hidden_states = self.mlp(hidden_states)312 hidden_states = residual + self.resid_mlp_dropout(hidden_states) # main diff with Llama313 314 outputs = (hidden_states,)315 if output_attentions:316 outputs += (self_attn_weights,)317 318 return outputs319 320 321class Phi3RotaryEmbedding(nn.Module):322 def __init__(self, config: Phi3Config, device=None):323 super().__init__()324 # BC: "rope_type" was originally "type"325 if hasattr(config, "rope_scaling") and config.rope_scaling is not None:326 self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))327 else:328 self.rope_type = "default"329 self.max_seq_len_cached = config.max_position_embeddings330 self.original_max_seq_len = config.max_position_embeddings331 332 self.config = config333 self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]334 335 inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)336 self.register_buffer("inv_freq", inv_freq, persistent=False)337 self.original_inv_freq = self.inv_freq338 339 def _dynamic_frequency_update(self, position_ids, device):340 """341 dynamic RoPE layers should recompute `inv_freq` in the following situations:342 1 - growing beyond the cached sequence length (allow scaling)343 2 - the current sequence length is in the original scale (avoid losing precision with small sequences)344 """345 seq_len = torch.max(position_ids) + 1346 if seq_len > self.max_seq_len_cached: # growth347 inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device, seq_len=seq_len)348 self.register_buffer("inv_freq", inv_freq, persistent=False) # TODO joao: may break with compilation349 self.max_seq_len_cached = seq_len350 351 if seq_len < self.original_max_seq_len and self.max_seq_len_cached > self.original_max_seq_len: # reset352 # This .to() is needed if the model has been moved to a device after being initialized (because353 # the buffer is automatically moved, but not the original copy)354 self.original_inv_freq = self.original_inv_freq.to(device)355 self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)356 self.max_seq_len_cached = self.original_max_seq_len357 358 @torch.no_grad()359 def forward(self, x, position_ids):360 if "dynamic" in self.rope_type:361 self._dynamic_frequency_update(position_ids, device=x.device)362 elif self.rope_type == "longrope":363 self._longrope_frequency_update(position_ids, device=x.device)364 365 # Core RoPE block366 inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)367 position_ids_expanded = position_ids[:, None, :].float()368 # Force float32 (see https://github.com/huggingface/transformers/pull/29285)369 device_type = x.device.type370 device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"371 with torch.autocast(device_type=device_type, enabled=False):372 freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)373 emb = torch.cat((freqs, freqs), dim=-1)374 cos = emb.cos()375 sin = emb.sin()376 377 # Advanced RoPE types (e.g. yarn) apply a post-processing scaling factor, equivalent to scaling attention378 cos = cos * self.attention_scaling379 sin = sin * self.attention_scaling380 381 return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)382 383 def _longrope_frequency_update(self, position_ids, device):384 """Longrope uses long factor if sequence is larger than original pretraining length, short otherwise."""385 seq_len = torch.max(position_ids) + 1386 if hasattr(self.config, "original_max_position_embeddings"):387 original_max_position_embeddings = self.config.original_max_position_embeddings388 else:389 original_max_position_embeddings = self.config.max_position_embeddings390 if seq_len > original_max_position_embeddings:391 if not hasattr(self, "long_inv_freq"):392 self.long_inv_freq, _ = self.rope_init_fn(393 self.config, device, seq_len=original_max_position_embeddings + 1394 )395 self.register_buffer("inv_freq", self.long_inv_freq, persistent=False)396 else:397 # This .to() is needed if the model has been moved to a device after being initialized (because398 # the buffer is automatically moved, but not the original copy)399 self.original_inv_freq = self.original_inv_freq.to(device)400 self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)401 402 403PHI3_START_DOCSTRING = r"""404 This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the405 library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads406 etc.)407 408 This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.409 Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage410 and behavior.411 412 Parameters:413 config ([`Phi3Config`]):414 Model configuration class with all the parameters of the model. Initializing with a config file does not415 load the weights associated with the model, only the configuration. Check out the416 [`~PreTrainedModel.from_pretrained`] method to load the model weights.417"""418 419 420@add_start_docstrings(421 "The bare Phi3 Model outputting raw hidden-states without any specific head on top.",422 PHI3_START_DOCSTRING,423)424class Phi3PreTrainedModel(PreTrainedModel):425 config_class = Phi3Config426 base_model_prefix = "model"427 supports_gradient_checkpointing = True428 _no_split_modules = ["Phi3DecoderLayer"]429 _skip_keys_device_placement = ["past_key_values"]430 _supports_flash_attn_2 = True431 _supports_sdpa = True432 _supports_flex_attn = True433 _supports_cache_class = True434 _supports_quantized_cache = True435 _supports_static_cache = True436 _supports_attention_backend = True437 _version = "0.0.5"438 439 def _init_weights(self, module):440 std = self.config.initializer_range441 if isinstance(module, nn.Linear):442 module.weight.data.normal_(mean=0.0, std=std)443 if module.bias is not None:444 module.bias.data.zero_()445 elif isinstance(module, nn.Embedding):446 module.weight.data.normal_(mean=0.0, std=std)447 if module.padding_idx is not None:448 module.weight.data[module.padding_idx].zero_()449 450 451PHI3_INPUTS_DOCSTRING = r"""452 Args:453 input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):454 Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide455 it.456 457 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and458 [`PreTrainedTokenizer.__call__`] for details.459 460 [What are input IDs?](../glossary#input-ids)461 attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):462 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:463 464 - 1 for tokens that are **not masked**,465 - 0 for tokens that are **masked**.466 467 [What are attention masks?](../glossary#attention-mask)468 469 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and470 [`PreTrainedTokenizer.__call__`] for details.471 472 If `past_key_values` is used, optionally only the last `input_ids` have to be input (see473 `past_key_values`).474 475 If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]476 and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more477 information on the default strategy.478 479 - 1 indicates the head is **not masked**,480 - 0 indicates the head is **masked**.481 position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):482 Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,483 config.n_positions - 1]`.484 485 [What are position IDs?](../glossary#position-ids)486 past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):487 Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention488 blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`489 returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.490 491 Two formats are allowed:492 - a [`~cache_utils.Cache`] instance, see our493 [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache);494 - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of495 shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy496 cache format.497 498 The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the499 legacy cache format will be returned.500 501 If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't502 have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`503 of shape `(batch_size, sequence_length)`.504 inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):505 Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This506 is useful if you want more control over how to convert `input_ids` indices into associated vectors than the507 model's internal embedding lookup matrix.508 use_cache (`bool`, *optional*):509 If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see510 `past_key_values`).511 output_attentions (`bool`, *optional*):512 Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned513 tensors for more detail.514 output_hidden_states (`bool`, *optional*):515 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for516 more detail.517 return_dict (`bool`, *optional*):518 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.519 cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):520 Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,521 this tensor is not affected by padding. It is used to update the cache in the correct position and to infer522 the complete sequence length.523"""524 525 526@add_start_docstrings(527 "The bare Phi3 Model outputting raw hidden-states without any specific head on top.",528 PHI3_START_DOCSTRING,529)530class Phi3Model(Phi3PreTrainedModel):531 """532 Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Phi3DecoderLayer`]533 534 Args:535 config: Phi3Config536 """537 538 def __init__(self, config: Phi3Config):539 super().__init__(config)540 self.padding_idx = config.pad_token_id541 self.vocab_size = config.vocab_size542 543 self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)544 self.layers = nn.ModuleList(545 [Phi3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]546 )547 self.norm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)548 self.rotary_emb = Phi3RotaryEmbedding(config=config)549 self.gradient_checkpointing = False550 551 # Initialize weights and apply final processing552 self.post_init()553 554 def get_input_embeddings(self):555 return self.embed_tokens556 557 def set_input_embeddings(self, value):558 self.embed_tokens = value559 560 @add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING)561 def forward(562 self,563 input_ids: torch.LongTensor = None,564 attention_mask: Optional[torch.Tensor] = None,565 position_ids: Optional[torch.LongTensor] = None,566 past_key_values: Optional[Cache] = None,567 inputs_embeds: Optional[torch.FloatTensor] = None,568 use_cache: Optional[bool] = None,569 output_attentions: Optional[bool] = None,570 output_hidden_states: Optional[bool] = None,571 return_dict: Optional[bool] = None,572 cache_position: Optional[torch.LongTensor] = None,573 **flash_attn_kwargs: Unpack[FlashAttentionKwargs],574 ) -> Union[Tuple, BaseModelOutputWithPast]:575 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions576 output_hidden_states = (577 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states578 )579 use_cache = use_cache if use_cache is not None else self.config.use_cache580 return_dict = return_dict if return_dict is not None else self.config.use_return_dict581 582 if (input_ids is None) ^ (inputs_embeds is not None):583 raise ValueError("You must specify exactly one of input_ids or inputs_embeds")584 585 if self.gradient_checkpointing and self.training and use_cache:586 logger.warning_once(587 "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`."588 )589 use_cache = False590 591 if inputs_embeds is None:592 inputs_embeds = self.embed_tokens(input_ids)593 594 if use_cache and past_key_values is None:595 past_key_values = DynamicCache()596 597 if cache_position is None:598 past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0599 cache_position = torch.arange(600 past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device601 )602 603 if position_ids is None:604 position_ids = cache_position.unsqueeze(0)605 606 causal_mask = self._update_causal_mask(607 attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions608 )609 610 hidden_states = inputs_embeds611 612 # create position embeddings to be shared across the decoder layers613 position_embeddings = self.rotary_emb(hidden_states, position_ids)614 615 # decoder layers616 all_hidden_states = () if output_hidden_states else None617 all_self_attns = () if output_attentions else None618 619 for decoder_layer in self.layers[: self.config.num_hidden_layers]:620 if output_hidden_states:621 all_hidden_states += (hidden_states,)622 623 if self.gradient_checkpointing and self.training:624 layer_outputs = self._gradient_checkpointing_func(625 decoder_layer.__call__,626 hidden_states,627 causal_mask,628 position_ids,629 past_key_values,630 output_attentions,631 use_cache,632 cache_position,633 position_embeddings,634 )635 else:636 layer_outputs = decoder_layer(637 hidden_states,638 attention_mask=causal_mask,639 position_ids=position_ids,640 past_key_value=past_key_values,641 output_attentions=output_attentions,642 use_cache=use_cache,643 cache_position=cache_position,644 position_embeddings=position_embeddings,645 **flash_attn_kwargs,646 )647 648 hidden_states = layer_outputs[0]649 650 if output_attentions:651 all_self_attns += (layer_outputs[1],)652 653 hidden_states = self.norm(hidden_states)654 655 # add hidden states from the last decoder layer656 if output_hidden_states:657 all_hidden_states += (hidden_states,)658 659 output = BaseModelOutputWithPast(660 last_hidden_state=hidden_states,661 past_key_values=past_key_values if use_cache else None,662 hidden_states=all_hidden_states,663 attentions=all_self_attns,664 )665 return output if return_dict else output.to_tuple()666 667 def _update_causal_mask(668 self,669 attention_mask: torch.Tensor,670 input_tensor: torch.Tensor,671 cache_position: torch.Tensor,672 past_key_values: Cache,673 output_attentions: bool,674 ):675 if self.config._attn_implementation == "flash_attention_2":676 if attention_mask is not None and past_key_values is not None:677 is_padding_right = attention_mask[:, -1].sum().item() != input_tensor.size()[0]678 if is_padding_right:679 raise ValueError(680 "You are attempting to perform batched generation with padding_side='right'"681 " this may lead to unexpected behaviour for Flash Attention version of Phi3. Make sure to "682 " call `tokenizer.padding_side = 'left'` before tokenizing the input. "683 )684 if attention_mask is not None and 0.0 in attention_mask:685 return attention_mask686 return None687 688 # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in689 # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail690 # to infer the attention mask.691 past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0692 using_static_cache = isinstance(past_key_values, StaticCache)693 using_sliding_window_cache = isinstance(past_key_values, SlidingWindowCache)694 695 # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward696 if (697 self.config._attn_implementation == "sdpa"698 and not (using_static_cache or using_sliding_window_cache)699 and not output_attentions700 ):701 if AttentionMaskConverter._ignore_causal_mask_sdpa(702 attention_mask,703 inputs_embeds=input_tensor,704 past_key_values_length=past_seen_tokens,705 sliding_window=self.config.sliding_window,706 is_training=self.training,707 ):708 return None709 710 dtype, device = input_tensor.dtype, input_tensor.device711 min_dtype = torch.finfo(dtype).min712 sequence_length = input_tensor.shape[1]713 # SlidingWindowCache or StaticCache714 if using_sliding_window_cache or using_static_cache:715 target_length = past_key_values.get_max_cache_shape()716 # DynamicCache or no cache717 else:718 target_length = (719 attention_mask.shape[-1]720 if isinstance(attention_mask, torch.Tensor)721 else past_seen_tokens + sequence_length + 1722 )723 724 # In case the provided `attention` mask is 2D, we generate a causal mask here (4D).725 causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position(726 attention_mask,727 sequence_length=sequence_length,728 target_length=target_length,729 dtype=dtype,730 device=device,731 cache_position=cache_position,732 batch_size=input_tensor.shape[0],733 config=self.config,734 past_key_values=past_key_values,735 )736 737 if (738 self.config._attn_implementation == "sdpa"739 and attention_mask is not None740 and attention_mask.device.type in ["cuda", "xpu"]741 and not output_attentions742 ):743 # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when744 # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path.745 # Details: https://github.com/pytorch/pytorch/issues/110213746 causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype)747 748 return causal_mask749 750 @staticmethod751 def _prepare_4d_causal_attention_mask_with_cache_position(752 attention_mask: torch.Tensor,753 sequence_length: int,754 target_length: int,755 dtype: torch.dtype,756 device: torch.device,757 cache_position: torch.Tensor,758 batch_size: int,759 config: Phi3Config,760 past_key_values: Cache,761 ):762 """763 Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape764 `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.765 766 Args:767 attention_mask (`torch.Tensor`):768 A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`.769 sequence_length (`int`):770 The sequence length being processed.771 target_length (`int`):772 The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet.773 dtype (`torch.dtype`):774 The dtype to use for the 4D attention mask.775 device (`torch.device`):776 The device to plcae the 4D attention mask on.777 cache_position (`torch.Tensor`):778 Indices depicting the position of the input sequence tokens in the sequence.779 batch_size (`torch.Tensor`):780 Batch size.781 config (`Phi3Config`):782 The model's configuration class783 past_key_values (`Cache`):784 The cache class that is being used currently to generate785 """786 if attention_mask is not None and attention_mask.dim() == 4:787 # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.788 causal_mask = attention_mask789 else:790 min_dtype = torch.finfo(dtype).min791 causal_mask = torch.full(792 (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device793 )794 diagonal_attend_mask = torch.arange(target_length, device=device) > cache_position.reshape(-1, 1)795 if config.sliding_window is not None:796 # if we have sliding window, we should not attend to tokens beyond sliding window length, so we mask them out also797 # the check is needed to verify is current checkpoint was trained with sliding window or not798 if not isinstance(past_key_values, SlidingWindowCache) or sequence_length > target_length:799 sliding_attend_mask = torch.arange(target_length, device=device) <= (800 cache_position.reshape(-1, 1) - config.sliding_window801 )802 diagonal_attend_mask.bitwise_or_(sliding_attend_mask)803 causal_mask *= diagonal_attend_mask804 causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)805 if attention_mask is not None:806 causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit807 if attention_mask.shape[-1] > target_length:808 attention_mask = attention_mask[:, :target_length]809 mask_length = attention_mask.shape[-1]810 padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :].to(811 causal_mask.device812 )813 padding_mask = padding_mask == 0814 causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill(815 padding_mask, min_dtype816 )817 return causal_mask818 819 820class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ...821 822 823class Phi3ForCausalLM(Phi3PreTrainedModel, GenerationMixin):824 _tied_weights_keys = ["lm_head.weight"]825 _tp_plan = {"lm_head": "colwise_rep"}826 _pp_plan = {"lm_head": (["hidden_states"], ["logits"])}827 828 def __init__(self, config):829 super().__init__(config)830 self.model = Phi3Model(config)831 self.vocab_size = config.vocab_size832 self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)833 834 # Initialize weights and apply final processing835 self.post_init()836 837 def get_input_embeddings(self):838 return self.model.embed_tokens839 840 def set_input_embeddings(self, value):841 self.model.embed_tokens = value842 843 def get_output_embeddings(self):844 return self.lm_head845 846 def set_output_embeddings(self, new_embeddings):847 self.lm_head = new_embeddings848 849 def set_decoder(self, decoder):850 self.model = decoder851 852 def get_decoder(self):853 return self.model854 855 @deprecate_kwarg("num_logits_to_keep", version="4.50", new_name="logits_to_keep")856 @add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING)857 @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)858 def forward(859 self,860 input_ids: torch.LongTensor = None,861 attention_mask: Optional[torch.Tensor] = None,862 position_ids: Optional[torch.LongTensor] = None,863 past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,864 inputs_embeds: Optional[torch.FloatTensor] = None,865 labels: Optional[torch.LongTensor] = None,866 use_cache: Optional[bool] = None,867 output_attentions: Optional[bool] = None,868 output_hidden_states: Optional[bool] = None,869 return_dict: Optional[bool] = None,870 cache_position: Optional[torch.LongTensor] = None,871 logits_to_keep: Union[int, torch.Tensor] = 0,872 **kwargs: Unpack[KwargsForCausalLM],873 ) -> Union[Tuple, CausalLMOutputWithPast]:874 r"""875 Args:876 labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):877 Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,878 config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored879 (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.880 881 logits_to_keep (`int` or `torch.Tensor`, *optional*):882 If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all883 `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that884 token can save memory, which becomes pretty significant for long sequences or large vocabulary size.885 If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension.886 This is useful when using packed tensor format (single dimension for batch and sequence length).887 888 Returns:889 890 Example:891 892 ```python893 >>> from transformers import AutoTokenizer, Phi3ForCausalLM894 895 >>> model = Phi3ForCausalLM.from_pretrained("meta-phi3/Phi3-2-7b-hf")896 >>> tokenizer = AutoTokenizer.from_pretrained("meta-phi3/Phi3-2-7b-hf")897 898 >>> prompt = "Hey, are you conscious? Can you talk to me?"899 >>> inputs = tokenizer(prompt, return_tensors="pt")900 901 >>> # Generate902 >>> generate_ids = model.generate(inputs.input_ids, max_length=30)903 >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]904 "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."905 ```"""906 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions907 output_hidden_states = (908 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states909 )910 return_dict = return_dict if return_dict is not None else self.config.use_return_dict911 912 # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)913 outputs = self.model(914 input_ids=input_ids,915 attention_mask=attention_mask,916 position_ids=position_ids,917 past_key_values=past_key_values,918 inputs_embeds=inputs_embeds,919 use_cache=use_cache,920 output_attentions=output_attentions,921 output_hidden_states=output_hidden_states,922 return_dict=return_dict,923 cache_position=cache_position,924 **kwargs,925 )926 927 hidden_states = outputs[0]928 # Only compute necessary logits, and do not upcast them to float if we are not computing the loss929 slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep930 logits = self.lm_head(hidden_states[:, slice_indices, :])931 932 loss = None933 if labels is not None:934 loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)935 936 if not return_dict:937 output = (logits,) + outputs[1:]938 return (loss,) + output if loss is not None else output939 940 return CausalLMOutputWithPast(941 loss=loss,942 logits=logits,943 past_key_values=outputs.past_key_values,944 hidden_states=outputs.hidden_states,945 attentions=outputs.attentions,946 )947 948 def prepare_inputs_for_generation(949 self,950 input_ids,951 past_key_values=None,952 attention_mask=None,953 inputs_embeds=None,954 cache_position=None,955 position_ids=None,956 use_cache=True,957 logits_to_keep=None,958 **kwargs,959 ):960 # Overwritten -- this model may need to switch between short and long rope, invalidating the cache in the961 # process962 963 # When the first time input length reached long and short factor switching point, enforce re-compute cache964 # It will cause downside of slower at this single token position, however, better than current failure.965 if (966 past_key_values967 and self.config.rope_scaling968 and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1969 ):970 past_length = cache_position[0]971 if past_length <= self.config.original_max_position_embeddings:972 past_key_values = None973 974 model_inputs = super().prepare_inputs_for_generation(975 input_ids=input_ids,976 past_key_values=past_key_values,977 attention_mask=attention_mask,978 inputs_embeds=inputs_embeds,979 cache_position=cache_position,980 position_ids=position_ids,981 use_cache=use_cache,982 logits_to_keep=logits_to_keep,983 **kwargs,984 )985 return model_inputs986 987 988@add_start_docstrings(989 """990 The Phi3 Model transformer with a sequence classification head on top (linear layer).991 992 [`Phi3ForSequenceClassification`] uses the last token in order to do the classification, as other causal models993 (e.g. GPT-2) do.994 995 Since it does classification on the last token, it requires to know the position of the last token. If a996 `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If997 no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the998 padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in999 each row of the batch).1000 """,1001 PHI3_START_DOCSTRING,1002)1003class Phi3ForSequenceClassification(Phi3PreTrainedModel):1004 def __init__(self, config):1005 super().__init__(config)1006 self.num_labels = config.num_labels1007 self.model = Phi3Model(config)1008 self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)1009 1010 # Initialize weights and apply final processing1011 self.post_init()1012 1013 def get_input_embeddings(self):1014 return self.model.embed_tokens1015 1016 def set_input_embeddings(self, value):1017 self.model.embed_tokens = value1018 1019 @add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING)1020 def forward(1021 self,1022 input_ids: Optional[torch.LongTensor] = None,1023 attention_mask: Optional[torch.Tensor] = None,1024 position_ids: Optional[torch.LongTensor] = None,1025 past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,1026 inputs_embeds: Optional[torch.FloatTensor] = None,1027 labels: Optional[torch.LongTensor] = None,1028 use_cache: Optional[bool] = None,1029 output_attentions: Optional[bool] = None,1030 output_hidden_states: Optional[bool] = None,1031 return_dict: Optional[bool] = None,1032 ) -> Union[Tuple, SequenceClassifierOutputWithPast]:1033 r"""1034 labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):1035 Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,1036 config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If1037 `config.num_labels > 1` a classification loss is computed (Cross-Entropy).1038 """1039 return_dict = return_dict if return_dict is not None else self.config.use_return_dict1040 1041 transformer_outputs = self.model(1042 input_ids,1043 attention_mask=attention_mask,1044 position_ids=position_ids,1045 past_key_values=past_key_values,1046 inputs_embeds=inputs_embeds,1047 use_cache=use_cache,1048 output_attentions=output_attentions,1049 output_hidden_states=output_hidden_states,1050 return_dict=return_dict,1051 )1052 hidden_states = transformer_outputs[0]1053 logits = self.score(hidden_states)1054 1055 if input_ids is not None:1056 batch_size = input_ids.shape[0]1057 else:1058 batch_size = inputs_embeds.shape[0]1059 1060 if self.config.pad_token_id is None and batch_size != 1:1061 raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")1062 if self.config.pad_token_id is None:1063 last_non_pad_token = -11064 elif input_ids is not None:1065 # To handle both left- and right- padding, we take the rightmost token that is not equal to pad_token_id1066 non_pad_mask = (input_ids != self.config.pad_token_id).to(logits.device, torch.int32)1067 token_indices = torch.arange(input_ids.shape[-1], device=logits.device)1068 last_non_pad_token = (token_indices * non_pad_mask).argmax(-1)1069 else:1070 last_non_pad_token = -11071 logger.warning_once(1072 f"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be "1073 "unexpected if using padding tokens in conjunction with `inputs_embeds.`"1074 )1075 1076 pooled_logits = logits[torch.arange(batch_size, device=logits.device), last_non_pad_token]1077 1078 loss = None1079 if labels is not None:1080 loss = self.loss_function(logits=logits, labels=labels, pooled_logits=pooled_logits, config=self.config)1081 1082 if not return_dict:1083 output = (pooled_logits,) + transformer_outputs[1:]1084 return ((loss,) + output) if loss is not None else output1085 1086 return SequenceClassifierOutputWithPast(1087 loss=loss,1088 logits=pooled_logits,1089 past_key_values=transformer_outputs.past_key_values,1090 hidden_states=transformer_outputs.hidden_states,1091 attentions=transformer_outputs.attentions,1092 )1093 1094 1095@add_start_docstrings(1096 """1097 The Phi3 Model transformer with a token classification head on top (a linear layer on top of the hidden-states1098 output) e.g. for Named-Entity-Recognition (NER) tasks.1099 """,1100 PHI3_START_DOCSTRING,1101)1102class Phi3ForTokenClassification(Phi3PreTrainedModel):1103 def __init__(self, config):1104 super().__init__(config)1105 self.num_labels = config.num_labels1106 self.model = Phi3Model(config)1107 if getattr(config, "classifier_dropout", None) is not None:1108 classifier_dropout = config.classifier_dropout1109 elif getattr(config, "hidden_dropout", None) is not None:1110 classifier_dropout = config.hidden_dropout1111 else:1112 classifier_dropout = 0.11113 self.dropout = nn.Dropout(classifier_dropout)1114 self.score = nn.Linear(config.hidden_size, config.num_labels)1115 1116 # Initialize weights and apply final processing1117 self.post_init()1118 1119 def get_input_embeddings(self):1120 return self.model.embed_tokens1121 1122 def set_input_embeddings(self, value):1123 self.model.embed_tokens = value1124 1125 @add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING)1126 @add_code_sample_docstrings(1127 checkpoint=_CHECKPOINT_FOR_DOC,1128 output_type=TokenClassifierOutput,1129 config_class=_CONFIG_FOR_DOC,1130 )1131 def forward(1132 self,1133 input_ids: Optional[torch.LongTensor] = None,1134 attention_mask: Optional[torch.Tensor] = None,1135 position_ids: Optional[torch.LongTensor] = None,1136 past_key_values: Optional[List[torch.FloatTensor]] = None,1137 inputs_embeds: Optional[torch.FloatTensor] = None,1138 labels: Optional[torch.LongTensor] = None,1139 use_cache: Optional[bool] = None,1140 output_attentions: Optional[bool] = None,1141 output_hidden_states: Optional[bool] = None,1142 return_dict: Optional[bool] = None,1143 ) -> Union[Tuple, TokenClassifierOutput]:1144 r"""1145 labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):1146 Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,1147 config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If1148 `config.num_labels > 1` a classification loss is computed (Cross-Entropy).1149 """1150 return_dict = return_dict if return_dict is not None else self.config.use_return_dict1151 1152 outputs = self.model(1153 input_ids,1154 attention_mask=attention_mask,1155 position_ids=position_ids,1156 past_key_values=past_key_values,1157 inputs_embeds=inputs_embeds,1158 use_cache=use_cache,1159 output_attentions=output_attentions,1160 output_hidden_states=output_hidden_states,1161 return_dict=return_dict,1162 )1163 sequence_output = outputs[0]1164 sequence_output = self.dropout(sequence_output)1165 logits = self.score(sequence_output)1166 1167 loss = None1168 if labels is not None:1169 loss = self.loss_function(logits, labels, self.config)1170 1171 if not return_dict:1172 output = (logits,) + outputs[2:]1173 return ((loss,) + output) if loss is not None else output1174 1175 return TokenClassifierOutput(1176 loss=loss,1177 logits=logits,1178 hidden_states=outputs.hidden_states,1179 attentions=outputs.attentions,1180 )1181