Mar2Ding/songcomposer_sft
16148
1# coding=utf-82# # Copyright (c) InternLM. All rights reserved.3#4# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX5# and OPT implementations in this library. It has been modified from its6# original forms to accommodate minor architectural differences compared7# to GPT-NeoX and OPT used by the Meta AI team that trained the model.8#9# Licensed under the Apache License, Version 2.0 (the "License");10# you may not use this file except in compliance with the License.11# You may obtain a copy of the License at12#13# http://www.apache.org/licenses/LICENSE-2.014#15# Unless required by applicable law or agreed to in writing, software16# distributed under the License is distributed on an "AS IS" BASIS,17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.18# See the License for the specific language governing permissions and19# limitations under the License.20""" PyTorch InternLM2 model."""21import math22import queue23import threading24import warnings25import copy26from typing import List, Optional, Tuple, Union27from torchvision import transforms28from torchvision.transforms.functional import InterpolationMode29from PIL import Image30 31import torch32import torch.utils.checkpoint33from einops import rearrange34from torch import nn35from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss36from transformers.activations import ACT2FN37from transformers.modeling_outputs import (38 BaseModelOutputWithPast,39 CausalLMOutputWithPast,40 SequenceClassifierOutputWithPast,41)42from transformers.modeling_utils import PreTrainedModel43from transformers.utils import (44 add_start_docstrings,45 add_start_docstrings_to_model_forward,46 logging,47 replace_return_docstrings,48)49from transformers import StoppingCriteria, StoppingCriteriaList50try:51 from transformers.generation.streamers import BaseStreamer52except: # noqa # pylint: disable=bare-except53 BaseStreamer = None54 55from .configuration_internlm import InternLMConfig as InternLM2Config56from .build_mlp import build_vision_tower, build_vision_projector, PLoRA57 58logger = logging.get_logger(__name__)59 60_CONFIG_FOR_DOC = "InternLM2Config"61 62 63 64class StoppingCriteriaSub(StoppingCriteria):65 def __init__(self, stops=[], encounters=1):66 super().__init__()67 self.stops = stops68 69 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor):70 for stop in self.stops:71 if torch.all((stop == input_ids[0][-len(stop):])).item():72 return True73 74 return False75 76def text_gen(inst, tokenizer, model, stopping_criteria, temp=1.0, rept=1.005, sample=True):77 d = f"{inst}"78 input_ids = tokenizer(d, return_tensors="pt")["input_ids"]79 eos_token_id = [tokenizer.eos_token_id, tokenizer.convert_tokens_to_ids(["[UNUSED_TOKEN_145]"])[0]]80 with torch.no_grad():81 generate = model.generate(input_ids.cuda(), 82 do_sample=sample,83 temperature=temp,84 repetition_penalty=rept, 85 max_new_tokens=1000, 86 top_p=0.8, 87 top_k=50, 88 eos_token_id=eos_token_id,89 stopping_criteria=stopping_criteria,)90 91 res = tokenizer.decode(generate[0].tolist(), skip_special_tokens=True)92 return (res)93 94# Copied from transformers.models.bart.modeling_bart._make_causal_mask95def _make_causal_mask(96 input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 097):98 """99 Make causal mask used for bi-directional self-attention.100 """101 bsz, tgt_len = input_ids_shape102 mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device)103 mask_cond = torch.arange(mask.size(-1), device=device)104 mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)105 mask = mask.to(dtype)106 107 if past_key_values_length > 0:108 mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)109 return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)110 111 112# Copied from transformers.models.bart.modeling_bart._expand_mask113def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):114 """115 Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.116 """117 bsz, src_len = mask.size()118 tgt_len = tgt_len if tgt_len is not None else src_len119 120 expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)121 122 inverted_mask = 1.0 - expanded_mask123 124 return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)125 126 127class InternLM2RMSNorm(nn.Module):128 def __init__(self, hidden_size, eps=1e-6):129 """130 InternLM2RMSNorm is equivalent to T5LayerNorm131 """132 super().__init__()133 self.weight = nn.Parameter(torch.ones(hidden_size))134 self.variance_epsilon = eps135 136 def forward(self, hidden_states):137 input_dtype = hidden_states.dtype138 hidden_states = hidden_states.to(torch.float32)139 variance = hidden_states.pow(2).mean(-1, keepdim=True)140 hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)141 return self.weight * hidden_states.to(input_dtype)142 143 144class InternLM2RotaryEmbedding(nn.Module):145 def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):146 super().__init__()147 148 self.dim = dim149 self.max_position_embeddings = max_position_embeddings150 self.base = base151 inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))152 self.register_buffer("inv_freq", inv_freq, persistent=False)153 154 # Build here to make `torch.jit.trace` work.155 self._set_cos_sin_cache(156 seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype()157 )158 159 def _set_cos_sin_cache(self, seq_len, device, dtype):160 self.max_seq_len_cached = seq_len161 t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)162 163 freqs = torch.einsum("i,j->ij", t, self.inv_freq)164 # Different from paper, but it uses a different permutation in order to obtain the same calculation165 emb = torch.cat((freqs, freqs), dim=-1)166 self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)167 self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)168 169 def forward(self, x, seq_len=None):170 # x: [bs, num_attention_heads, seq_len, head_size]171 if seq_len > self.max_seq_len_cached:172 self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)173 174 return (175 self.cos_cached[:seq_len].to(dtype=x.dtype),176 self.sin_cached[:seq_len].to(dtype=x.dtype),177 )178 179 180class InternLM2LinearScalingRotaryEmbedding(InternLM2RotaryEmbedding):181 """InternLM2RotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""182 183 def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):184 self.scaling_factor = scaling_factor185 super().__init__(dim, max_position_embeddings, base, device)186 187 def _set_cos_sin_cache(self, seq_len, device, dtype):188 self.max_seq_len_cached = seq_len189 t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)190 t = t / self.scaling_factor191 192 freqs = torch.einsum("i,j->ij", t, self.inv_freq)193 # Different from paper, but it uses a different permutation in order to obtain the same calculation194 emb = torch.cat((freqs, freqs), dim=-1)195 self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)196 self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)197 198 199class InternLM2DynamicNTKScalingRotaryEmbedding(InternLM2RotaryEmbedding):200 """InternLM2RotaryEmbedding extended with Dynamic NTK scaling.201 Credits to the Reddit users /u/bloc97 and /u/emozilla.202 """203 204 def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):205 self.scaling_factor = scaling_factor206 super().__init__(dim, max_position_embeddings, base, device)207 208 def _set_cos_sin_cache(self, seq_len, device, dtype):209 self.max_seq_len_cached = seq_len210 211 if seq_len > self.max_position_embeddings:212 base = self.base * (213 (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1)214 ) ** (self.dim / (self.dim - 2))215 inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))216 self.register_buffer("inv_freq", inv_freq, persistent=False)217 218 t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)219 220 freqs = torch.einsum("i,j->ij", t, self.inv_freq)221 # Different from paper, but it uses a different permutation in order to obtain the same calculation222 emb = torch.cat((freqs, freqs), dim=-1)223 self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)224 self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)225 226 227def rotate_half(x):228 """Rotates half the hidden dims of the input."""229 x1 = x[..., : x.shape[-1] // 2]230 x2 = x[..., x.shape[-1] // 2 :]231 return torch.cat((-x2, x1), dim=-1)232 233 234def apply_rotary_pos_emb(q, k, cos, sin, position_ids):235 # The first two dimensions of cos and sin are always 1, so we can `squeeze` them.236 cos = cos.squeeze(1).squeeze(0) # [seq_len, dim]237 sin = sin.squeeze(1).squeeze(0) # [seq_len, dim]238 cos = cos.unsqueeze(0).unsqueeze(0).expand(len(position_ids), -1, -1, -1)239 sin = sin.unsqueeze(0).unsqueeze(0).expand(len(position_ids), -1, -1, -1)240 if q.size(2) == 1:241 q_embed = (q * cos[:, :, -1:, :]) + (rotate_half(q) * sin[:, :, -1:, :])242 else:243 q_embed = (q * cos) + (rotate_half(q) * sin)244 245 if k.size(2) == 1:246 k_embed = (k * cos[:, :, -1:, :]) + (rotate_half(k) * sin[:, :, -1:, :])247 else:248 k_embed = (k * cos) + (rotate_half(k) * sin)249 250 return q_embed, k_embed251 252 253class InternLM2MLP(nn.Module):254 def __init__(self, config):255 super().__init__()256 self.config = config257 self.hidden_size = config.hidden_size258 self.intermediate_size = config.intermediate_size259 #self.w1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)260 #self.w3 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)261 #self.w2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)262 263 self.w1 = PLoRA(self.hidden_size, self.intermediate_size, bias=False,264 lora_r=256, lora_alpha=256, lora_len=256)265 self.w3 = PLoRA(self.hidden_size, self.intermediate_size, bias=False,266 lora_r=256, lora_alpha=256, lora_len=256)267 self.w2 = PLoRA(self.intermediate_size, self.hidden_size, bias=False,268 lora_r=256, lora_alpha=256, lora_len=256)269 270 self.act_fn = ACT2FN[config.hidden_act]271 272 def forward(self, x, im_mask):273 down_proj = self.w2(self.act_fn(self.w1(x, im_mask)) * self.w3(x, im_mask), im_mask)274 275 return down_proj276 277 278def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:279 """280 This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,281 num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)282 """283 batch, num_key_value_heads, slen, head_dim = hidden_states.shape284 if n_rep == 1:285 return hidden_states286 hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)287 return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)288 289 290class InternLM2Attention(nn.Module):291 """Multi-headed attention from 'Attention Is All You Need' paper"""292 293 def __init__(self, config: InternLM2Config):294 super().__init__()295 self.config = config296 self.hidden_size = config.hidden_size297 self.num_heads = config.num_attention_heads298 self.head_dim = self.hidden_size // self.num_heads299 self.num_key_value_heads = config.num_key_value_heads300 self.num_key_value_groups = self.num_heads // self.num_key_value_heads301 self.max_position_embeddings = config.max_position_embeddings302 self.is_causal = True303 304 if (self.head_dim * self.num_heads) != self.hidden_size:305 raise ValueError(306 f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"307 f" and `num_heads`: {self.num_heads})."308 )309 310 #self.wqkv = nn.Linear(311 self.wqkv = PLoRA(312 self.hidden_size,313 (self.num_heads + 2 * self.num_key_value_heads) * self.head_dim,314 bias=config.bias,315 lora_r=256, lora_alpha=256, lora_len=256316 )317 318 #self.wo = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.bias)319 self.wo = PLoRA(self.num_heads * self.head_dim, self.hidden_size, bias=config.bias,320 lora_r=256, lora_alpha=256, lora_len=256)321 self._init_rope()322 323 def _init_rope(self):324 if self.config.rope_scaling is None:325 self.rotary_emb = InternLM2RotaryEmbedding(326 self.head_dim,327 max_position_embeddings=self.max_position_embeddings,328 base=self.config.rope_theta,329 )330 else:331 scaling_type = self.config.rope_scaling["type"]332 scaling_factor = self.config.rope_scaling["factor"]333 if scaling_type == "dynamic":334 self.rotary_emb = InternLM2DynamicNTKScalingRotaryEmbedding(335 self.head_dim,336 max_position_embeddings=self.max_position_embeddings,337 base=self.config.rope_theta,338 scaling_factor=scaling_factor339 )340 else:341 raise ValueError("Currently we only support rotary embedding's type being 'dynamic'.")342 return self.rotary_emb343 344 def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):345 return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()346 347 def forward(348 self,349 hidden_states: torch.Tensor,350 attention_mask: Optional[torch.Tensor] = None,351 position_ids: Optional[torch.LongTensor] = None,352 past_key_value: Optional[Tuple[torch.Tensor]] = None,353 output_attentions: bool = False,354 use_cache: bool = False,355 im_mask: Optional[Tuple[torch.Tensor]] = None,356 **kwargs,357 ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:358 if "padding_mask" in kwargs:359 warnings.warn(360 "Passing `padding_mask` is deprecated and will be removed in v4.37. "361 "Please make sure use `attention_mask` instead.`"362 )363 364 bsz, q_len, _ = hidden_states.size()365 366 qkv_states = self.wqkv(hidden_states, im_mask)367 368 qkv_states = rearrange(369 qkv_states,370 "b q (h gs d) -> b q h gs d",371 gs=2 + self.num_key_value_groups,372 d=self.head_dim,373 )374 375 query_states = qkv_states[..., : self.num_key_value_groups, :]376 query_states = rearrange(query_states, "b q h gs d -> b q (h gs) d")377 key_states = qkv_states[..., -2, :]378 value_states = qkv_states[..., -1, :]379 380 query_states = query_states.transpose(1, 2)381 key_states = key_states.transpose(1, 2)382 value_states = value_states.transpose(1, 2)383 384 kv_seq_len = key_states.shape[-2]385 if past_key_value is not None:386 kv_seq_len += past_key_value[0].shape[-2]387 cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)388 query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)389 390 if past_key_value is not None:391 # reuse k, v, self_attention392 key_states = torch.cat([past_key_value[0], key_states], dim=2)393 value_states = torch.cat([past_key_value[1], value_states], dim=2)394 395 past_key_value = (key_states, value_states) if use_cache else None396 397 key_states = repeat_kv(key_states, self.num_key_value_groups)398 value_states = repeat_kv(value_states, self.num_key_value_groups)399 400 attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)401 402 if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):403 raise ValueError(404 f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"405 f" {attn_weights.size()}"406 )407 408 if attention_mask is not None:409 if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):410 raise ValueError(411 f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"412 )413 attn_weights = attn_weights + attention_mask414 415 # upcast attention to fp32416 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)417 attn_output = torch.matmul(attn_weights, value_states)418 419 if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):420 raise ValueError(421 f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"422 f" {attn_output.size()}"423 )424 425 attn_output = attn_output.transpose(1, 2).contiguous()426 attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)427 428 attn_output = self.wo(attn_output, im_mask)429 430 if not output_attentions:431 attn_weights = None432 433 return attn_output, attn_weights, past_key_value434 435 436class InternLM2FlashAttention2(InternLM2Attention):437 """438 InternLM2 flash attention module. This module inherits from `InternLM2Attention` as the weights of the module stays439 untouched. The only required change would be on the forward pass where it needs to correctly call the public API of440 flash attention and deal with padding tokens in case the input contains any of them.441 """442 443 def forward(444 self,445 hidden_states: torch.Tensor,446 attention_mask: Optional[torch.LongTensor] = None,447 position_ids: Optional[torch.LongTensor] = None,448 past_key_value: Optional[Tuple[torch.Tensor]] = None,449 output_attentions: bool = False,450 use_cache: bool = False,451 im_mask: Optional[Tuple[torch.Tensor]] = None,452 **kwargs,453 ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:454 # InternLM2FlashAttention2 attention does not support output_attentions455 if "padding_mask" in kwargs:456 warnings.warn(457 "Passing `padding_mask` is deprecated and will be removed in v4.37. "458 "Please make sure use `attention_mask` instead.`"459 )460 461 # overwrite attention_mask with padding_mask462 attention_mask = kwargs.pop("padding_mask")463 464 output_attentions = False465 466 bsz, q_len, _ = hidden_states.size()467 468 qkv_states = self.wqkv(hidden_states, im_mask)469 470 qkv_states = rearrange(471 qkv_states,472 "b q (h gs d) -> b q h gs d",473 gs=self.num_heads + 2 * self.num_key_value_heads,474 d=self.head_dim,475 q=q_len,476 )477 478 query_states = qkv_states[..., : self.num_key_value_groups, :]479 query_states = rearrange(query_states, "b q h gs d -> b q (h gs) d")480 key_states = qkv_states[..., -2, :]481 value_states = qkv_states[..., -1, :]482 483 kv_seq_len = key_states.shape[-2]484 if past_key_value is not None:485 kv_seq_len += past_key_value[0].shape[-2]486 487 cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)488 489 query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)490 491 if past_key_value is not None:492 # reuse k, v, self_attention493 key_states = torch.cat([past_key_value[0], key_states], dim=2)494 value_states = torch.cat([past_key_value[1], value_states], dim=2)495 496 past_key_value = (key_states, value_states) if use_cache else None497 498 query_states = query_states.transpose(1, 2)499 key_states = key_states.transpose(1, 2)500 value_states = value_states.transpose(1, 2)501 502 dropout_rate = 0.0 if not self.training else self.attention_dropout503 504 # In PEFT, usually we cast the layer norms in float32 for training stability reasons505 # therefore the input hidden states gets silently casted in float32. Hence, we need506 # cast them back in the correct dtype just to be sure everything works as expected.507 # This might slowdown training & inference so it is recommended to not cast the LayerNorms508 # in fp32. (InternLM2RMSNorm handles it correctly)509 510 input_dtype = query_states.dtype511 if input_dtype == torch.float32:512 # Handle the case where the model is quantized513 if hasattr(self.config, "_pre_quantization_dtype"):514 target_dtype = self.config._pre_quantization_dtype515 else:516 target_dtype = self.q_proj.weight.dtype517 518 logger.warning_once(519 f"The input hidden states seems to be silently casted in float32, this might be related to"520 f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back "521 f"the input in {target_dtype}."522 )523 524 query_states = query_states.to(target_dtype)525 key_states = key_states.to(target_dtype)526 value_states = value_states.to(target_dtype)527 528 attn_output = self._flash_attention_forward(529 query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate530 )531 532 attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()533 attn_output = self.wo(attn_output, im_mask)534 535 if not output_attentions:536 attn_weights = None537 538 return attn_output, attn_weights, past_key_value539 540 541class InternLM2DecoderLayer(nn.Module):542 def __init__(self, config: InternLM2Config):543 super().__init__()544 self.hidden_size = config.hidden_size545 self.attention = (546 InternLM2Attention(config=config)547 if not getattr(config, "_flash_attn_2_enabled", False)548 else InternLM2FlashAttention2(config=config)549 )550 self.feed_forward = InternLM2MLP(config)551 self.attention_norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)552 self.ffn_norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)553 554 def forward(555 self,556 hidden_states: torch.Tensor,557 attention_mask: Optional[torch.Tensor] = None,558 position_ids: Optional[torch.LongTensor] = None,559 past_key_value: Optional[Tuple[torch.Tensor]] = None,560 output_attentions: Optional[bool] = False,561 use_cache: Optional[bool] = False,562 im_mask: Optional[Tuple[torch.Tensor]] = None,563 **kwargs,564 ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:565 """566 Args:567 hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`568 attention_mask (`torch.FloatTensor`, *optional*):569 attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,570 query_sequence_length, key_sequence_length)` if default attention is used.571 output_attentions (`bool`, *optional*):572 Whether or not to return the attentions tensors of all attention layers. See `attentions` under573 returned tensors for more detail.574 use_cache (`bool`, *optional*):575 If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding576 (see `past_key_values`).577 past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states578 """579 if "padding_mask" in kwargs:580 warnings.warn(581 "Passing `padding_mask` is deprecated and will be removed in v4.37. "582 "Please make sure use `attention_mask` instead.`"583 )584 585 residual = hidden_states586 587 hidden_states = self.attention_norm(hidden_states)588 589 # Self Attention590 hidden_states, self_attn_weights, present_key_value = self.attention(591 hidden_states=hidden_states,592 attention_mask=attention_mask,593 position_ids=position_ids,594 past_key_value=past_key_value,595 output_attentions=output_attentions,596 use_cache=use_cache,597 im_mask=im_mask,598 **kwargs,599 )600 hidden_states = residual + hidden_states601 602 # Fully Connected603 residual = hidden_states604 hidden_states = self.ffn_norm(hidden_states)605 hidden_states = self.feed_forward(hidden_states, im_mask)606 hidden_states = residual + hidden_states607 608 outputs = (hidden_states,)609 610 if output_attentions:611 outputs += (self_attn_weights,)612 613 if use_cache:614 outputs += (present_key_value,)615 616 return outputs617 618 619InternLM2_START_DOCSTRING = r"""620 This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the621 library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads622 etc.)623 624 This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.625 Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage626 and behavior.627 628 Parameters:629 config ([`InternLM2Config`]):630 Model configuration class with all the parameters of the model. Initializing with a config file does not631 load the weights associated with the model, only the configuration. Check out the632 [`~PreTrainedModel.from_pretrained`] method to load the model weights.633"""634 635 636@add_start_docstrings(637 "The bare InternLM2 Model outputting raw hidden-states without any specific head on top.",638 InternLM2_START_DOCSTRING,639)640class InternLM2PreTrainedModel(PreTrainedModel):641 config_class = InternLM2Config642 base_model_prefix = "model"643 supports_gradient_checkpointing = True644 _no_split_modules = ["InternLM2DecoderLayer"]645 _skip_keys_device_placement = "past_key_values"646 _supports_flash_attn_2 = True647 648 def _init_weights(self, module):649 std = self.config.initializer_range650 if isinstance(module, nn.Linear):651 module.weight.data.normal_(mean=0.0, std=std)652 if module.bias is not None:653 module.bias.data.zero_()654 elif isinstance(module, nn.Embedding):655 module.weight.data.normal_(mean=0.0, std=std)656 if module.padding_idx is not None:657 module.weight.data[module.padding_idx].zero_()658 659 660InternLM2_INPUTS_DOCSTRING = r"""661 Args:662 input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):663 Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide664 it.665 666 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and667 [`PreTrainedTokenizer.__call__`] for details.668 669 [What are input IDs?](../glossary#input-ids)670 attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):671 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:672 673 - 1 for tokens that are **not masked**,674 - 0 for tokens that are **masked**.675 676 [What are attention masks?](../glossary#attention-mask)677 678 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and679 [`PreTrainedTokenizer.__call__`] for details.680 681 If `past_key_values` is used, optionally only the last `input_ids` have to be input (see682 `past_key_values`).683 684 If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]685 and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more686 information on the default strategy.687 688 - 1 indicates the head is **not masked**,689 - 0 indicates the head is **masked**.690 position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):691 Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,692 config.n_positions - 1]`.693 694 [What are position IDs?](../glossary#position-ids)695 past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or696 when `config.use_cache=True`):697 Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape698 `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape699 `(batch_size, num_heads, decoder_sequence_length, embed_size_per_head)`.700 701 Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention702 blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.703 704 If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't705 have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`706 of shape `(batch_size, sequence_length)`.707 inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):708 Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This709 is useful if you want more control over how to convert `input_ids` indices into associated vectors than the710 model's internal embedding lookup matrix.711 use_cache (`bool`, *optional*):712 If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see713 `past_key_values`).714 output_attentions (`bool`, *optional*):715 Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned716 tensors for more detail.717 output_hidden_states (`bool`, *optional*):718 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for719 more detail.720 return_dict (`bool`, *optional*):721 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.722"""723 724 725@add_start_docstrings(726 "The bare InternLM2 Model outputting raw hidden-states without any specific head on top.",727 InternLM2_START_DOCSTRING,728)729class InternLM2Model(InternLM2PreTrainedModel):730 """731 Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`InternLM2DecoderLayer`]732 733 Args:734 config: InternLM2Config735 """736 737 _auto_class = "AutoModel"738 739 def __init__(self, config: InternLM2Config):740 super().__init__(config)741 self.padding_idx = config.pad_token_id742 self.vocab_size = config.vocab_size743 744 self.tok_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)745 self.layers = nn.ModuleList([InternLM2DecoderLayer(config) for _ in range(config.num_hidden_layers)])746 self.norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)747 748 self.gradient_checkpointing = False749 # Initialize weights and apply final processing750 self.post_init()751 752 def get_input_embeddings(self):753 return self.tok_embeddings754 755 def set_input_embeddings(self, value):756 self.tok_embeddings = value757 758 # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask759 def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):760 # create causal mask761 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]762 combined_attention_mask = None763 if input_shape[-1] > 1:764 combined_attention_mask = _make_causal_mask(765 input_shape,766 inputs_embeds.dtype,767 device=inputs_embeds.device,768 past_key_values_length=past_key_values_length,769 )770 771 if attention_mask is not None:772 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]773 expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(774 inputs_embeds.device775 )776 combined_attention_mask = (777 expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask778 )779 780 return combined_attention_mask781 782 @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING)783 def forward(784 self,785 input_ids: torch.LongTensor = None,786 attention_mask: Optional[torch.Tensor] = None,787 position_ids: Optional[torch.LongTensor] = None,788 past_key_values: Optional[List[torch.FloatTensor]] = None,789 inputs_embeds: Optional[torch.FloatTensor] = None,790 use_cache: Optional[bool] = None,791 output_attentions: Optional[bool] = None,792 output_hidden_states: Optional[bool] = None,793 return_dict: Optional[bool] = None,794 **kwargs795 ) -> Union[Tuple, BaseModelOutputWithPast]:796 797 im_mask = kwargs.get('im_mask', None)798 799 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions800 output_hidden_states = (801 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states802 )803 use_cache = use_cache if use_cache is not None else self.config.use_cache804 805 return_dict = return_dict if return_dict is not None else self.config.use_return_dict806 807 # retrieve input_ids and inputs_embeds808 if input_ids is not None and inputs_embeds is not None:809 raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")810 elif input_ids is not None:811 batch_size, seq_length = input_ids.shape[:2]812 elif inputs_embeds is not None:813 batch_size, seq_length = inputs_embeds.shape[:2]814 else:815 raise ValueError("You have to specify either input_ids or inputs_embeds")816 817 seq_length_with_past = seq_length818 past_key_values_length = 0819 if past_key_values is not None:820 past_key_values_length = past_key_values[0][0].shape[2]821 seq_length_with_past = seq_length_with_past + past_key_values_length822 823 if position_ids is None:824 device = input_ids.device if input_ids is not None else inputs_embeds.device825 position_ids = torch.arange(826 past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device827 )828 position_ids = position_ids.unsqueeze(0)829 830 if inputs_embeds is None:831 inputs_embeds = self.tok_embeddings(input_ids)832 im_mask = torch.zeros(inputs_embeds.shape[:2]).to(inputs_embeds.device).bool()833 # embed positions834 if attention_mask is None:835 attention_mask = torch.ones(836 (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device837 )838 attention_mask = self._prepare_decoder_attention_mask(839 attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length840 )841 842 # embed positions843 hidden_states = inputs_embeds844 845 if self.gradient_checkpointing and self.training:846 if use_cache:847 logger.warning_once(848 "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."849 )850 use_cache = False851 852 # decoder layers853 all_hidden_states = () if output_hidden_states else None854 all_self_attns = () if output_attentions else None855 next_decoder_cache = () if use_cache else None856 857 for idx, decoder_layer in enumerate(self.layers):858 if output_hidden_states:859 all_hidden_states += (hidden_states,)860 861 past_key_value = past_key_values[idx] if past_key_values is not None else None862 863 if self.gradient_checkpointing and self.training:864 865 def create_custom_forward(module):866 def custom_forward(*inputs):867 # None for past_key_value868 return module(*inputs, output_attentions, None, im_mask)869 870 return custom_forward871 872 layer_outputs = torch.utils.checkpoint.checkpoint(873 create_custom_forward(decoder_layer),874 hidden_states,875 attention_mask,876 position_ids,877 None,878 )879 else:880 layer_outputs = decoder_layer(881 hidden_states,882 attention_mask=attention_mask,883 position_ids=position_ids,884 past_key_value=past_key_value,885 output_attentions=output_attentions,886 use_cache=use_cache,887 im_mask=im_mask,888 )889 890 hidden_states = layer_outputs[0]891 892 if use_cache:893 next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)894 895 if output_attentions:896 all_self_attns += (layer_outputs[1],)897 898 hidden_states = self.norm(hidden_states)899 900 # add hidden states from the last decoder layer901 if output_hidden_states:902 all_hidden_states += (hidden_states,)903 904 next_cache = next_decoder_cache if use_cache else None905 if not return_dict:906 return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)907 return BaseModelOutputWithPast(908 last_hidden_state=hidden_states,909 past_key_values=next_cache,910 hidden_states=all_hidden_states,911 attentions=all_self_attns,912 )913 914 915class InternLM2ForCausalLM(InternLM2PreTrainedModel):916 _auto_class = "AutoModelForCausalLM"917 918 _tied_weights_keys = ["output.weight"]919 920 def __init__(self, config):921 super().__init__(config)922 self.model = InternLM2Model(config)923 self.vocab_size = config.vocab_size924 self.output = nn.Linear(config.hidden_size, config.vocab_size, bias=False)925 self.debug_flag = 1926 self.tokenizer = None927 928 self.max_length = config.max_length929 print (f'Set max length to {self.max_length}')930 self.debug_flag = 1931 # Initialize weights and apply final processing932 self.post_init()933 934 # self.vit = build_vision_tower()935 # self.vision_proj = build_vision_projector()936 # self.im_size = 224937 # self.vis_processor = transforms.Compose([938 # transforms.Resize((224, 224),939 # interpolation=InterpolationMode.BICUBIC),940 # transforms.ToTensor(),941 # transforms.Normalize((0.48145466, 0.4578275, 0.40821073),942 # (0.26862954, 0.26130258, 0.27577711)),943 # ])944 945 def _set_gradient_checkpointing(self, module, value=False):946 if isinstance(module, InternLM2Model):947 module.gradient_checkpointing = value948 # if value:949 # self.vit.vision_tower.vision_model.encoder.gradient_checkpointing = value950 951 def get_input_embeddings(self):952 return self.model.tok_embeddings953 954 def set_input_embeddings(self, value):955 self.model.tok_embeddings = value956 957 def get_output_embeddings(self):958 return self.output959 960 def set_output_embeddings(self, new_embeddings):961 self.output = new_embeddings962 963 def set_decoder(self, decoder):964 self.model = decoder965 966 def get_decoder(self):967 return self.model968 def encode_text(self, t, add_special_tokens=False):969 t = t.replace('<|User|>:', '[UNUSED_TOKEN_146]user\n')970 t = t.replace('<|Bot|>:', '[UNUSED_TOKEN_146]assistant\n')971 t = t.replace('<TOKENS_UNUSED_0>', '[UNUSED_TOKEN_145]')972 t = t.replace('<TOKENS_UNUSED_1>', '[UNUSED_TOKEN_145]')973 t = t.replace('[UNUSED_TOKEN_0]', '[UNUSED_TOKEN_145]')974 t = t.replace('[UNUSED_TOKEN_1]', '[UNUSED_TOKEN_145]')975 976 text = t977 token = self.tokenizer(text,978 return_tensors='pt',979 add_special_tokens=add_special_tokens).input_ids.to(self.device)980 embs = self.model.tok_embeddings(token)981 return embs982 983 # def encode_img(self, image):984 # if image is None:985 # return None986 # if isinstance(image, str):987 # image = Image.open(image).convert("RGB")988 # image = self.vis_processor(image).unsqueeze(0).to(self.device)989 # else:990 # assert isinstance(image, torch.Tensor)991 992 # img_embeds, atts_img, img_target = self.img2emb(image)993 # return img_embeds994 995 996 997 # def img2emb(self, image):998 # img_embeds = self.vision_proj(999 # self.vit(image.to(self.device)))1000 # atts_img = torch.ones(img_embeds.size()[:-1], dtype=torch.long).to(img_embeds.device)1001 1002 # img_target = torch.ones(img_embeds.size()[:2], dtype=torch.long).to(img_embeds.device) * -1001003 1004 # return img_embeds, atts_img, img_target1005 1006 def prompt_wrap(self, img_embeds, prompt):1007 batch_size = img_embeds.shape[0]1008 p_before, p_after = prompt.split('<ImageHere>')1009 p_before_tokens = self.tokenizer(1010 p_before, return_tensors="pt", add_special_tokens=True).to(img_embeds.device)1011 1012 p_before_embeds = self.model.tok_embeddings(p_before_tokens.input_ids).expand(batch_size, -1, -1)1013 wrapped_img_embeds = torch.cat([p_before_embeds, img_embeds], dim=1)1014 1015 wrapped_atts_img = torch.ones(wrapped_img_embeds.size()[:-1], dtype=torch.long).to(img_embeds.device)1016 1017 wrapped_target = torch.ones(batch_size, wrapped_img_embeds.shape[1], dtype=torch.long).to(img_embeds.device) * -1001018 1019 1020 return wrapped_img_embeds, wrapped_atts_img, wrapped_target1021 1022 def text2emb(self, text, add_special=False):1023 # import pdb; pdb.set_trace()1024 new_text = []1025 for t in text:1026 t = t.replace('<|User|>:', '[UNUSED_TOKEN_146]user\n')1027 t = t.replace('<|Bot|>:', '[UNUSED_TOKEN_146]assistant\n')1028 t = t.replace('<TOKENS_UNUSED_0>', '[UNUSED_TOKEN_145]')1029 t = t.replace('<TOKENS_UNUSED_1>', '[UNUSED_TOKEN_145]')1030 new_text.append(t)1031 text = new_text1032 to_regress_tokens = self.tokenizer(1033 text,1034 return_tensors="pt",1035 padding="longest",1036 truncation=True,1037 max_length=self.max_length,1038 add_special_tokens=add_special1039 ).to(self.device)1040 1041 # targets = self.mask_human_targets(to_regress_tokens.input_ids)1042 # targets = targets.to(self.device)1043 targets = to_regress_tokens.input_ids.masked_fill(1044 to_regress_tokens.input_ids == self.tokenizer.pad_token_id, -1001045 ).to(self.device)1046 1047 1048 return to_regress_tokens, targets1049 1050 def mask_human_targets(self, input_ids, pure=False):1051 target_batch = []1052 for bs in range(input_ids.shape[0]):1053 cur_idx = 01054 ids = input_ids[bs]1055 targets = copy.deepcopy(ids)1056 end_count = 01057 last_eoa = 01058 for i, temp_id in enumerate(ids):1059 if temp_id == 92542:1060 if end_count % 2 == 0:1061 targets[last_eoa: i+6] = -1001062 else:1063 last_eoa = i + 11064 end_count += 11065 elif temp_id == 2: ### eos and following pad1066 targets[i+1:] = -100 #### loss on eos, but not on pad 1067 break1068 if temp_id != 2 and end_count % 2 == 0: ### trunction, end at last question1069 targets[last_eoa+1:] = -100 #### mask all after the last answer1070 1071 target_batch.append(targets.unsqueeze(0))1072 if self.debug_flag and 0:1073 print ('#### Warining! System meta is not support now')1074 targets_vis = targets.clone()1075 targets_vis[targets_vis==-100] = 923991076 targets_vis_tokens = ''.join(self.tokenizer.convert_ids_to_tokens(targets_vis)).replace('[UNUSED_TOKEN_2]', " ")1077 print(''.join(self.tokenizer.convert_ids_to_tokens(ids)))1078 print('-----------')1079 print([targets_vis_tokens])1080 print('-----------------------------')1081 1082 target_batch = torch.cat(target_batch, dim=0)1083 return target_batch1084 1085 @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING)1086 @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)1087 def forward(1088 self,1089 input_ids: torch.LongTensor = None,1090 attention_mask: Optional[torch.Tensor] = None,1091 position_ids: Optional[torch.LongTensor] = None,1092 past_key_values: Optional[List[torch.FloatTensor]] = None,1093 inputs_embeds: Optional[torch.FloatTensor] = None,1094 labels: Optional[torch.LongTensor] = None,1095 use_cache: Optional[bool] = None,1096 output_attentions: Optional[bool] = None,1097 output_hidden_states: Optional[bool] = None,1098 return_dict: Optional[bool] = None,1099 **kwargs1100 ) -> Union[Tuple, CausalLMOutputWithPast]:1101 r"""1102 Args:1103 labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):1104 Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,1105 config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored1106 (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.1107 1108 Returns:1109 1110 Example:1111 1112 ```python1113 >>> from transformers import AutoTokenizer, InternLM2ForCausalLM1114 1115 >>> model = InternLM2ForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)1116 >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)1117 1118 >>> prompt = "Hey, are you conscious? Can you talk to me?"1119 >>> inputs = tokenizer(prompt, return_tensors="pt")1120 1121 >>> # Generate1122 >>> generate_ids = model.generate(inputs.input_ids, max_length=30)1123 >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]1124 "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."1125 ```"""1126 samples = kwargs.get('samples', None)1127 if samples:1128 if self.debug_flag:1129 self.debug_flag += 11130 if self.debug_flag > 5:1131 self.debug_flag = 01132 1133 has_img = 'image' in samples.keys()1134 # import pdb; pdb.set_trace()1135 ### encode text1136 # sp_token = samples["sp_token"]1137 1138 text = samples['text_input']1139 text = ['<|User|>:' + t for t in text]1140 to_regress_tokens, targets = self.text2emb(text, add_special = True)1141 1142 to_regress_embeds = self.model.tok_embeddings(to_regress_tokens.input_ids)1143 attention_mask = to_regress_tokens.attention_mask1144 1145 if has_img:1146 ### encode image1147 image = samples["image"][0]1148 bs = to_regress_embeds.shape[0]1149 assert image.shape[0] == bs1150 ### combine text and image1151 if samples['data_type'][0] != 'nlp':1152 img_embeds, atts_img, img_target = self.img2emb(image)1153 to_regress_embeds = torch.cat([to_regress_embeds[:,:1], img_embeds, to_regress_embeds[:,1:]], dim=1)1154 attention_mask = torch.cat([attention_mask[:,:1], atts_img, attention_mask[:,1:]], dim=1)1155 targets = torch.cat([targets[:,:1], img_target, targets[:,1:]], dim=1)1156 1157 im_len = img_embeds.shape[1]1158 im_mask = torch.zeros(to_regress_embeds.shape[:2]).cuda()1159 im_mask[:,1:1+im_len] = 11160 temp_max_length = self.max_length1161 1162 else:1163 img_embeds, atts_img, img_target = self.img2emb(torch.zeros(1,3,self.im_size,self.im_size).to(image.device).to(image.dtype))1164 to_regress_embeds += img_embeds.sum() * 01165 im_mask = torch.zeros(to_regress_embeds.shape[:2]).cuda()1166 temp_max_length = self.max_length1167 1168 temp_max_length = self.max_length1169 inputs_embeds = to_regress_embeds[:, :temp_max_length]1170 attention_mask = attention_mask[:, :temp_max_length]1171 targets = targets[:, :temp_max_length]1172 # im_mask = im_mask[:, :temp_max_length].bool()1173 labels = targets1174 if self.debug_flag:1175 print (targets.shape, inputs_embeds.shape, attention_mask.shape)1176 le = len(samples['text_input'])1177 data_type = samples['data_type'][0]1178 print (f'DataType: {data_type}. Has Image: {has_img}. Current max length: {self.max_length}, BatchSize is {le}')1179 if has_img:1180 print (img_embeds.shape)1181 1182 else:1183 self.debug_flag = 01184 im_mask = kwargs.get('im_mask', None)1185 if im_mask is None and inputs_embeds is not None:1186 im_mask = torch.zeros(inputs_embeds.shape[:2]).to(inputs_embeds.device)1187 im_mask[:,1:1+256] = 11188 im_mask = im_mask.bool()1189 1190 1191 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions1192 output_hidden_states = (1193 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states1194 )1195 return_dict = return_dict if return_dict is not None else self.config.use_return_dict1196 1197 # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)1198 outputs = self.model(1199 input_ids=input_ids,1200 attention_mask=attention_mask,