ByteDance/Sa2VA-1B
31825
1# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved.2#3# This code is based on transformers/src/transformers/models/llama/modeling_llama.py4#5# Licensed under the Apache License, Version 2.0 (the "License");6# you may not use this file except in compliance with the License.7# You may obtain a copy of the License at8#9# http://www.apache.org/licenses/LICENSE-2.010#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.14# See the License for the specific language governing permissions and15# limitations under the License.16""" PyTorch InternLM2 model."""17import math18import queue19import threading20import warnings21from typing import List, Optional, Tuple, Union22 23import torch24import torch.nn.functional as F25import torch.utils.checkpoint26from einops import rearrange27from torch import nn28from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss29from transformers.activations import ACT2FN30from transformers.modeling_outputs import (BaseModelOutputWithPast,31 CausalLMOutputWithPast,32 SequenceClassifierOutputWithPast)33from transformers.modeling_utils import PreTrainedModel34from transformers.utils import (add_start_docstrings,35 add_start_docstrings_to_model_forward, logging,36 replace_return_docstrings)37 38try:39 from transformers.generation.streamers import BaseStreamer40except: # noqa # pylint: disable=bare-except41 BaseStreamer = None42 43from .configuration_internlm2 import InternLM2Config44 45logger = logging.get_logger(__name__)46 47_CONFIG_FOR_DOC = 'InternLM2Config'48 49flash_attn_func, flash_attn_varlen_func = None, None50pad_input, index_first_axis, unpad_input = None, None, None51try:52 from flash_attn import flash_attn_func as _flash_attn_func53 from flash_attn import flash_attn_varlen_func as _flash_attn_varlen_func54 from flash_attn.bert_padding import index_first_axis as _index_first_axis55 from flash_attn.bert_padding import pad_input as _pad_input56 from flash_attn.bert_padding import unpad_input as _unpad_input57 58 flash_attn_func, flash_attn_varlen_func = _flash_attn_func, _flash_attn_varlen_func59 pad_input, index_first_axis, unpad_input = _pad_input, _index_first_axis, _unpad_input60 has_flash_attn = True61except:62 has_flash_attn = False63 64 65def _import_flash_attn():66 global flash_attn_func, flash_attn_varlen_func67 global pad_input, index_first_axis, unpad_input68 try:69 from flash_attn import flash_attn_func as _flash_attn_func70 from flash_attn import \71 flash_attn_varlen_func as _flash_attn_varlen_func72 from flash_attn.bert_padding import \73 index_first_axis as _index_first_axis74 from flash_attn.bert_padding import pad_input as _pad_input75 from flash_attn.bert_padding import unpad_input as _unpad_input76 flash_attn_func, flash_attn_varlen_func = _flash_attn_func, _flash_attn_varlen_func77 pad_input, index_first_axis, unpad_input = _pad_input, _index_first_axis, _unpad_input78 except ImportError:79 raise ImportError('flash_attn is not installed.')80 81 82# Copied from transformers.models.llama.modeling_llama._get_unpad_data83def _get_unpad_data(attention_mask):84 seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)85 indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()86 max_seqlen_in_batch = seqlens_in_batch.max().item()87 cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))88 return (89 indices,90 cu_seqlens,91 max_seqlen_in_batch,92 )93 94 95# Copied from transformers.models.bart.modeling_bart._make_causal_mask96def _make_causal_mask(97 input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 098):99 """100 Make causal mask used for bi-directional self-attention.101 """102 bsz, tgt_len = input_ids_shape103 mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device)104 mask_cond = torch.arange(mask.size(-1), device=device)105 mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)106 mask = mask.to(dtype)107 108 if past_key_values_length > 0:109 mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)110 return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)111 112 113# Copied from transformers.models.bart.modeling_bart._expand_mask114def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):115 """116 Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.117 """118 bsz, src_len = mask.size()119 tgt_len = tgt_len if tgt_len is not None else src_len120 121 expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)122 123 inverted_mask = 1.0 - expanded_mask124 125 return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)126 127 128# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->InternLM2129class InternLM2RMSNorm(nn.Module):130 def __init__(self, hidden_size, eps=1e-6):131 """132 InternLM2RMSNorm is equivalent to T5LayerNorm133 """134 super().__init__()135 self.weight = nn.Parameter(torch.ones(hidden_size))136 self.variance_epsilon = eps137 138 def forward(self, hidden_states):139 input_dtype = hidden_states.dtype140 hidden_states = hidden_states.to(torch.float32)141 variance = hidden_states.pow(2).mean(-1, keepdim=True)142 hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)143 return self.weight * hidden_states.to(input_dtype)144 145 146try:147 from functools import partial148 149 from apex.normalization import FusedRMSNorm150 InternLM2RMSNorm = partial(FusedRMSNorm, eps=1e-6) # noqa151 print('Discovered apex.normalization.FusedRMSNorm - will use it instead of InternLM2RMSNorm')152except ImportError:153 # using the normal LlamaRMSNorm154 pass155except Exception:156 print('discovered apex but it failed to load, falling back to InternLM2RMSNorm')157 pass158 159 160# Copied from transformers.model.llama.modeling_llama.LlamaRotaryEmbedding with Llama->InternLM2161class InternLM2RotaryEmbedding(nn.Module):162 def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):163 super().__init__()164 165 self.dim = dim166 self.max_position_embeddings = max_position_embeddings167 self.base = base168 inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))169 self.register_buffer('inv_freq', inv_freq, persistent=False)170 171 # Build here to make `torch.jit.trace` work.172 self._set_cos_sin_cache(173 seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype()174 )175 176 def _set_cos_sin_cache(self, seq_len, device, dtype):177 self.max_seq_len_cached = seq_len178 t = torch.arange(self.max_seq_len_cached, device=device).to(dtype=self.inv_freq.dtype)179 180 freqs = torch.einsum('i,j->ij', t, self.inv_freq)181 # Different from paper, but it uses a different permutation in order to obtain the same calculation182 emb = torch.cat((freqs, freqs), dim=-1)183 self.register_buffer('cos_cached', emb.cos().to(dtype), persistent=False)184 self.register_buffer('sin_cached', emb.sin().to(dtype), persistent=False)185 186 def forward(self, x, seq_len=None):187 # x: [bs, num_attention_heads, seq_len, head_size]188 if seq_len > self.max_seq_len_cached:189 self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=torch.float32)190 191 return (192 self.cos_cached[:seq_len].to(dtype=x.dtype),193 self.sin_cached[:seq_len].to(dtype=x.dtype),194 )195 196 197# Copied from transformers.model.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding with Llama->InternLM2198class InternLM2LinearScalingRotaryEmbedding(InternLM2RotaryEmbedding):199 """InternLM2RotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""200 201 def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):202 self.scaling_factor = scaling_factor203 super().__init__(dim, max_position_embeddings, base, device)204 205 def _set_cos_sin_cache(self, seq_len, device, dtype):206 self.max_seq_len_cached = seq_len207 t = torch.arange(self.max_seq_len_cached, device=device).to(dtype=self.inv_freq.dtype)208 t = t / self.scaling_factor209 210 freqs = torch.einsum('i,j->ij', t, self.inv_freq)211 # Different from paper, but it uses a different permutation in order to obtain the same calculation212 emb = torch.cat((freqs, freqs), dim=-1)213 self.register_buffer('cos_cached', emb.cos().to(dtype), persistent=False)214 self.register_buffer('sin_cached', emb.sin().to(dtype), persistent=False)215 216 217# Copied from transformers.model.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding with Llama->InternLM2218class InternLM2DynamicNTKScalingRotaryEmbedding(InternLM2RotaryEmbedding):219 """InternLM2RotaryEmbedding extended with Dynamic NTK scaling.220 Credits to the Reddit users /u/bloc97 and /u/emozilla.221 """222 223 def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):224 self.scaling_factor = scaling_factor225 super().__init__(dim, max_position_embeddings, base, device)226 227 def _set_cos_sin_cache(self, seq_len, device, dtype):228 self.max_seq_len_cached = seq_len229 230 if seq_len > self.max_position_embeddings:231 base = self.base * (232 (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1)233 ) ** (self.dim / (self.dim - 2))234 inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))235 self.register_buffer('inv_freq', inv_freq, persistent=False)236 237 t = torch.arange(self.max_seq_len_cached, device=device).to(dtype=self.inv_freq.dtype)238 239 freqs = torch.einsum('i,j->ij', t, self.inv_freq)240 # Different from paper, but it uses a different permutation in order to obtain the same calculation241 emb = torch.cat((freqs, freqs), dim=-1)242 self.register_buffer('cos_cached', emb.cos().to(dtype), persistent=False)243 self.register_buffer('sin_cached', emb.sin().to(dtype), persistent=False)244 245 246# Copied from transformers.model.llama.modeling_llama.rotate_half247def rotate_half(x):248 """Rotates half the hidden dims of the input."""249 x1 = x[..., : x.shape[-1] // 2]250 x2 = x[..., x.shape[-1] // 2:]251 return torch.cat((-x2, x1), dim=-1)252 253 254# Copied from transformers.model.llama.modeling_llama.apply_rotary_pos_emb255def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):256 """Applies Rotary Position Embedding to the query and key tensors."""257 cos = cos[position_ids].unsqueeze(unsqueeze_dim)258 sin = sin[position_ids].unsqueeze(unsqueeze_dim)259 q_embed = (q * cos) + (rotate_half(q) * sin)260 k_embed = (k * cos) + (rotate_half(k) * sin)261 return q_embed, k_embed262 263 264class InternLM2MLP(nn.Module):265 def __init__(self, config):266 super().__init__()267 self.config = config268 self.hidden_size = config.hidden_size269 self.intermediate_size = config.intermediate_size270 self.w1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)271 self.w3 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)272 self.w2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)273 self.act_fn = ACT2FN[config.hidden_act]274 275 def forward(self, x):276 down_proj = self.w2(self.act_fn(self.w1(x)) * self.w3(x))277 278 return down_proj279 280 281# Copied from transformers.model.llama.modeling_llama.repeat_kv282def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:283 """284 This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,285 num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)286 """287 batch, num_key_value_heads, slen, head_dim = hidden_states.shape288 if n_rep == 1:289 return hidden_states290 hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)291 return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)292 293 294# Modified from transformers.model.llama.modeling_llama.LlamaAttention295class InternLM2Attention(nn.Module):296 """Multi-headed attention from 'Attention Is All You Need' paper"""297 298 def __init__(self, config: InternLM2Config):299 super().__init__()300 self.config = config301 self.hidden_size = config.hidden_size302 self.num_heads = config.num_attention_heads303 self.head_dim = self.hidden_size // self.num_heads304 self.num_key_value_heads = config.num_key_value_heads305 self.num_key_value_groups = self.num_heads // self.num_key_value_heads306 self.max_position_embeddings = config.max_position_embeddings307 self.is_causal = True308 309 if (self.head_dim * self.num_heads) != self.hidden_size:310 raise ValueError(311 f'hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}'312 f' and `num_heads`: {self.num_heads}).'313 )314 315 self.wqkv = nn.Linear(316 self.hidden_size,317 (self.num_heads + 2 * self.num_key_value_heads) * self.head_dim,318 bias=config.bias,319 )320 321 self.wo = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.bias)322 self._init_rope()323 324 def _init_rope(self):325 if self.config.rope_scaling is None:326 self.rotary_emb = InternLM2RotaryEmbedding(327 self.head_dim,328 max_position_embeddings=self.max_position_embeddings,329 base=self.config.rope_theta,330 )331 else:332 scaling_type = self.config.rope_scaling['type']333 scaling_factor = self.config.rope_scaling['factor']334 if scaling_type == 'dynamic':335 self.rotary_emb = InternLM2DynamicNTKScalingRotaryEmbedding(336 self.head_dim,337 max_position_embeddings=self.max_position_embeddings,338 base=self.config.rope_theta,339 scaling_factor=scaling_factor,340 )341 elif scaling_type == 'linear':342 self.rotary_emb = InternLM2LinearScalingRotaryEmbedding(343 self.head_dim,344 max_position_embeddings=self.max_position_embeddings,345 base=self.config.rope_theta,346 scaling_factor=scaling_factor,347 )348 else:349 raise ValueError("Currently we only support rotary embedding's type being 'dynamic' or 'linear'.")350 return self.rotary_emb351 352 def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):353 return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()354 355 def forward(356 self,357 hidden_states: torch.Tensor,358 attention_mask: Optional[torch.Tensor] = None,359 position_ids: Optional[torch.LongTensor] = None,360 past_key_value: Optional[Tuple[torch.Tensor]] = None,361 output_attentions: bool = False,362 use_cache: bool = False,363 **kwargs,364 ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:365 if 'padding_mask' in kwargs:366 warnings.warn(367 'Passing `padding_mask` is deprecated and will be removed in v4.37. '368 'Please make sure use `attention_mask` instead.`'369 )370 371 bsz, q_len, _ = hidden_states.size()372 373 qkv_states = self.wqkv(hidden_states)374 375 qkv_states = rearrange(376 qkv_states,377 'b q (h gs d) -> b q h gs d',378 gs=2 + self.num_key_value_groups,379 d=self.head_dim,380 )381 382 query_states = qkv_states[..., : self.num_key_value_groups, :]383 query_states = rearrange(query_states, 'b q h gs d -> b q (h gs) d')384 key_states = qkv_states[..., -2, :]385 value_states = qkv_states[..., -1, :]386 387 query_states = query_states.transpose(1, 2)388 key_states = key_states.transpose(1, 2)389 value_states = value_states.transpose(1, 2)390 391 kv_seq_len = key_states.shape[-2]392 if past_key_value is not None:393 kv_seq_len += past_key_value[0].shape[-2]394 cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)395 query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)396 397 if past_key_value is not None:398 # reuse k, v, self_attention399 key_states = torch.cat([past_key_value[0], key_states], dim=2)400 value_states = torch.cat([past_key_value[1], value_states], dim=2)401 402 past_key_value = (key_states, value_states) if use_cache else None403 404 key_states = repeat_kv(key_states, self.num_key_value_groups)405 value_states = repeat_kv(value_states, self.num_key_value_groups)406 407 attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)408 409 if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):410 raise ValueError(411 f'Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is'412 f' {attn_weights.size()}'413 )414 415 if attention_mask is not None:416 if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):417 raise ValueError(418 f'Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}'419 )420 attn_weights = attn_weights + attention_mask421 422 # upcast attention to fp32423 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)424 attn_output = torch.matmul(attn_weights, value_states)425 426 if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):427 raise ValueError(428 f'`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is'429 f' {attn_output.size()}'430 )431 432 attn_output = attn_output.transpose(1, 2).contiguous()433 attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)434 435 attn_output = self.wo(attn_output)436 437 if not output_attentions:438 attn_weights = None439 440 return attn_output, attn_weights, past_key_value441 442 443# Modified from transformers.model.llama.modeling_llama.InternLM2FlashAttention2444class InternLM2FlashAttention2(InternLM2Attention):445 """446 InternLM2 flash attention module. This module inherits from `InternLM2Attention` as the weights of the module stays447 untouched. The only required change would be on the forward pass where it needs to correctly call the public API of448 flash attention and deal with padding tokens in case the input contains any of them.449 """450 451 def forward(452 self,453 hidden_states: torch.Tensor,454 attention_mask: Optional[torch.LongTensor] = None,455 position_ids: Optional[torch.LongTensor] = None,456 past_key_value: Optional[Tuple[torch.Tensor]] = None,457 output_attentions: bool = False,458 use_cache: bool = False,459 **kwargs,460 ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:461 # InternLM2FlashAttention2 attention does not support output_attentions462 if 'padding_mask' in kwargs:463 warnings.warn(464 'Passing `padding_mask` is deprecated and will be removed in v4.37. '465 'Please make sure use `attention_mask` instead.`'466 )467 468 # overwrite attention_mask with padding_mask469 attention_mask = kwargs.pop('padding_mask')470 471 output_attentions = False472 473 bsz, q_len, _ = hidden_states.size()474 475 qkv_states = self.wqkv(hidden_states)476 477 qkv_states = rearrange(478 qkv_states,479 'b q (h gs d) -> b q h gs d',480 gs=2 + self.num_key_value_groups,481 d=self.head_dim,482 )483 484 query_states = qkv_states[..., : self.num_key_value_groups, :]485 query_states = rearrange(query_states, 'b q h gs d -> b q (h gs) d')486 key_states = qkv_states[..., -2, :]487 value_states = qkv_states[..., -1, :]488 489 query_states = query_states.transpose(1, 2)490 key_states = key_states.transpose(1, 2)491 value_states = value_states.transpose(1, 2)492 493 kv_seq_len = key_states.shape[-2]494 if past_key_value is not None:495 kv_seq_len += past_key_value[0].shape[-2]496 497 cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)498 499 query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)500 501 if past_key_value is not None:502 # reuse k, v, self_attention503 key_states = torch.cat([past_key_value[0], key_states], dim=2)504 value_states = torch.cat([past_key_value[1], value_states], dim=2)505 506 past_key_value = (key_states, value_states) if use_cache else None507 508 query_states = query_states.transpose(1, 2)509 key_states = key_states.transpose(1, 2)510 value_states = value_states.transpose(1, 2)511 512 attn_output = self._flash_attention_forward(513 query_states, key_states, value_states, attention_mask, q_len514 )515 attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()516 attn_output = self.wo(attn_output)517 518 if not output_attentions:519 attn_weights = None520 521 return attn_output, attn_weights, past_key_value522 523 def _flash_attention_forward(524 self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None525 ):526 """527 Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token528 first unpad the input, then computes the attention scores and pad the final attention scores.529 530 Args:531 query_states (`torch.Tensor`):532 Input query states to be passed to Flash Attention API533 key_states (`torch.Tensor`):534 Input key states to be passed to Flash Attention API535 value_states (`torch.Tensor`):536 Input value states to be passed to Flash Attention API537 attention_mask (`torch.Tensor`):538 The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the539 position of padding tokens and 1 for the position of non-padding tokens.540 dropout (`int`, *optional*):541 Attention dropout542 softmax_scale (`float`, *optional*):543 The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)544 """545 # Contains at least one padding token in the sequence546 causal = self.is_causal and query_length != 1547 if attention_mask is not None:548 batch_size = query_states.shape[0]549 query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._unpad_input(550 query_states, key_states, value_states, attention_mask, query_length551 )552 553 cu_seqlens_q, cu_seqlens_k = cu_seq_lens554 max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens555 556 attn_output_unpad = flash_attn_varlen_func(557 query_states,558 key_states,559 value_states,560 cu_seqlens_q=cu_seqlens_q,561 cu_seqlens_k=cu_seqlens_k,562 max_seqlen_q=max_seqlen_in_batch_q,563 max_seqlen_k=max_seqlen_in_batch_k,564 dropout_p=dropout,565 softmax_scale=softmax_scale,566 causal=causal,567 )568 569 attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)570 else:571 attn_output = flash_attn_func(572 query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal573 )574 575 return attn_output576 577 def _unpad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):578 indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)579 batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape580 581 key_layer = index_first_axis(582 key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k583 )584 value_layer = index_first_axis(585 value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k586 )587 588 if query_length == kv_seq_len:589 query_layer = index_first_axis(590 query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k591 )592 cu_seqlens_q = cu_seqlens_k593 max_seqlen_in_batch_q = max_seqlen_in_batch_k594 indices_q = indices_k595 elif query_length == 1:596 max_seqlen_in_batch_q = 1597 cu_seqlens_q = torch.arange(598 batch_size + 1, dtype=torch.int32, device=query_layer.device599 ) # There is a memcpy here, that is very bad.600 indices_q = cu_seqlens_q[:-1]601 query_layer = query_layer.squeeze(1)602 else:603 # The -q_len: slice assumes left padding.604 attention_mask = attention_mask[:, -query_length:]605 query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)606 607 return (608 query_layer,609 key_layer,610 value_layer,611 indices_q.to(torch.int64),612 (cu_seqlens_q, cu_seqlens_k),613 (max_seqlen_in_batch_q, max_seqlen_in_batch_k),614 )615 616 617INTERNLM2_ATTENTION_CLASSES = {618 'eager': InternLM2Attention,619 'flash_attention_2': InternLM2FlashAttention2,620}621 622 623# Modified from transformers.model.llama.modeling_llama.LlamaDecoderLayer624class InternLM2DecoderLayer(nn.Module):625 def __init__(self, config: InternLM2Config):626 super().__init__()627 self.hidden_size = config.hidden_size628 629 self.attention = INTERNLM2_ATTENTION_CLASSES[config.attn_implementation](config=config)630 631 self.feed_forward = InternLM2MLP(config)632 self.attention_norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)633 self.ffn_norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)634 635 def forward(636 self,637 hidden_states: torch.Tensor,638 attention_mask: Optional[torch.Tensor] = None,639 position_ids: Optional[torch.LongTensor] = None,640 past_key_value: Optional[Tuple[torch.Tensor]] = None,641 output_attentions: Optional[bool] = False,642 use_cache: Optional[bool] = False,643 **kwargs,644 ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:645 """646 Args:647 hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`648 attention_mask (`torch.FloatTensor`, *optional*):649 attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,650 query_sequence_length, key_sequence_length)` if default attention is used.651 output_attentions (`bool`, *optional*):652 Whether or not to return the attentions tensors of all attention layers. See `attentions` under653 returned tensors for more detail.654 use_cache (`bool`, *optional*):655 If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding656 (see `past_key_values`).657 past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states658 """659 if 'padding_mask' in kwargs:660 warnings.warn(661 'Passing `padding_mask` is deprecated and will be removed in v4.37. '662 'Please make sure use `attention_mask` instead.`'663 )664 665 residual = hidden_states666 667 hidden_states = self.attention_norm(hidden_states)668 669 # Self Attention670 hidden_states, self_attn_weights, present_key_value = self.attention(671 hidden_states=hidden_states,672 attention_mask=attention_mask,673 position_ids=position_ids,674 past_key_value=past_key_value,675 output_attentions=output_attentions,676 use_cache=use_cache,677 **kwargs,678 )679 hidden_states = residual + hidden_states680 681 # Fully Connected682 residual = hidden_states683 hidden_states = self.ffn_norm(hidden_states)684 hidden_states = self.feed_forward(hidden_states)685 hidden_states = residual + hidden_states686 687 outputs = (hidden_states,)688 689 if output_attentions:690 outputs += (self_attn_weights,)691 692 if use_cache:693 outputs += (present_key_value,)694 695 return outputs696 697 698InternLM2_START_DOCSTRING = r"""699 This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the700 library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads701 etc.)702 703 This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.704 Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage705 and behavior.706 707 Parameters:708 config ([`InternLM2Config`]):709 Model configuration class with all the parameters of the model. Initializing with a config file does not710 load the weights associated with the model, only the configuration. Check out the711 [`~PreTrainedModel.from_pretrained`] method to load the model weights.712"""713 714 715# Copied from transformers.models.llama.modeling_llama.LlamaPreTrainedModel with Llama->InternLM2716@add_start_docstrings(717 'The bare InternLM2 Model outputting raw hidden-states without any specific head on top.',718 InternLM2_START_DOCSTRING,719)720class InternLM2PreTrainedModel(PreTrainedModel):721 config_class = InternLM2Config722 base_model_prefix = 'model'723 supports_gradient_checkpointing = True724 _no_split_modules = ['InternLM2DecoderLayer']725 _skip_keys_device_placement = 'past_key_values'726 _supports_flash_attn_2 = True727 728 def _init_weights(self, module):729 std = self.config.initializer_range730 if isinstance(module, nn.Linear):731 module.weight.data.normal_(mean=0.0, std=std)732 if module.bias is not None:733 module.bias.data.zero_()734 elif isinstance(module, nn.Embedding):735 module.weight.data.normal_(mean=0.0, std=std)736 if module.padding_idx is not None:737 module.weight.data[module.padding_idx].zero_()738 739 740InternLM2_INPUTS_DOCSTRING = r"""741 Args:742 input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):743 Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide744 it.745 746 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and747 [`PreTrainedTokenizer.__call__`] for details.748 749 [What are input IDs?](../glossary#input-ids)750 attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):751 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:752 753 - 1 for tokens that are **not masked**,754 - 0 for tokens that are **masked**.755 756 [What are attention masks?](../glossary#attention-mask)757 758 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and759 [`PreTrainedTokenizer.__call__`] for details.760 761 If `past_key_values` is used, optionally only the last `input_ids` have to be input (see762 `past_key_values`).763 764 If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]765 and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more766 information on the default strategy.767 768 - 1 indicates the head is **not masked**,769 - 0 indicates the head is **masked**.770 position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):771 Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,772 config.n_positions - 1]`.773 774 [What are position IDs?](../glossary#position-ids)775 past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or776 when `config.use_cache=True`):777 Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape778 `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape779 `(batch_size, num_heads, decoder_sequence_length, embed_size_per_head)`.780 781 Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention782 blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.783 784 If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't785 have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`786 of shape `(batch_size, sequence_length)`.787 inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):788 Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This789 is useful if you want more control over how to convert `input_ids` indices into associated vectors than the790 model's internal embedding lookup matrix.791 use_cache (`bool`, *optional*):792 If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see793 `past_key_values`).794 output_attentions (`bool`, *optional*):795 Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned796 tensors for more detail.797 output_hidden_states (`bool`, *optional*):798 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for799 more detail.800 return_dict (`bool`, *optional*):801 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.802"""803 804 805# Modified from transformers.model.llama.modeling_llama.LlamaModel806@add_start_docstrings(807 'The bare InternLM2 Model outputting raw hidden-states without any specific head on top.',808 InternLM2_START_DOCSTRING,809)810class InternLM2Model(InternLM2PreTrainedModel):811 """812 Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`InternLM2DecoderLayer`]813 814 Args:815 config: InternLM2Config816 """817 818 _auto_class = 'AutoModel'819 820 def __init__(self, config: InternLM2Config):821 super().__init__(config)822 self.padding_idx = config.pad_token_id823 self.vocab_size = config.vocab_size824 self.config = config825 if not has_flash_attn:826 self.config.attn_implementation = 'eager'827 print('Warning: Flash attention is not available, using eager attention instead.')828 829 self.tok_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)830 831 self.layers = nn.ModuleList([InternLM2DecoderLayer(config) for _ in range(config.num_hidden_layers)])832 self.norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)833 834 self.gradient_checkpointing = False835 # Initialize weights and apply final processing836 self.post_init()837 838 def get_input_embeddings(self):839 return self.tok_embeddings840 841 def set_input_embeddings(self, value):842 self.tok_embeddings = value843 844 def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):845 # create causal mask846 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]847 combined_attention_mask = None848 if input_shape[-1] > 1:849 combined_attention_mask = _make_causal_mask(850 input_shape,851 inputs_embeds.dtype,852 device=inputs_embeds.device,853 past_key_values_length=past_key_values_length,854 )855 856 if attention_mask is not None:857 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]858 expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(859 inputs_embeds.device860 )861 combined_attention_mask = (862 expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask863 )864 865 return combined_attention_mask866 867 @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING)868 def forward(869 self,870 input_ids: torch.LongTensor = None,871 attention_mask: Optional[torch.Tensor] = None,872 position_ids: Optional[torch.LongTensor] = None,873 past_key_values: Optional[List[torch.FloatTensor]] = None,874 inputs_embeds: Optional[torch.FloatTensor] = None,875 use_cache: Optional[bool] = None,876 output_attentions: Optional[bool] = None,877 output_hidden_states: Optional[bool] = None,878 return_dict: Optional[bool] = None,879 ) -> Union[Tuple, BaseModelOutputWithPast]:880 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions881 output_hidden_states = (882 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states883 )884 use_cache = use_cache if use_cache is not None else self.config.use_cache885 886 return_dict = return_dict if return_dict is not None else self.config.use_return_dict887 888 if self.config.attn_implementation == 'flash_attention_2':889 _import_flash_attn()890 891 # retrieve input_ids and inputs_embeds892 if input_ids is not None and inputs_embeds is not None:893 raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time')894 elif input_ids is not None:895 batch_size, seq_length = input_ids.shape[:2]896 elif inputs_embeds is not None:897 batch_size, seq_length = inputs_embeds.shape[:2]898 else:899 raise ValueError('You have to specify either input_ids or inputs_embeds')900 901 seq_length_with_past = seq_length902 past_key_values_length = 0903 if past_key_values is not None:904 past_key_values_length = past_key_values[0][0].shape[2]905 seq_length_with_past = seq_length_with_past + past_key_values_length906 907 if position_ids is None:908 device = input_ids.device if input_ids is not None else inputs_embeds.device909 position_ids = torch.arange(910 past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device911 )912 position_ids = position_ids.unsqueeze(0)913 914 if inputs_embeds is None:915 inputs_embeds = self.tok_embeddings(input_ids)916 917 if self.config.attn_implementation == 'flash_attention_2':918 # 2d mask is passed through the layers919 attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None920 else:921 if attention_mask is None:922 attention_mask = torch.ones(923 (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device924 )925 attention_mask = self._prepare_decoder_attention_mask(926 attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length927 )928 929 # embed positions930 hidden_states = inputs_embeds931 932 if self.gradient_checkpointing and self.training:933 if use_cache:934 logger.warning_once(935 '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...'936 )937 use_cache = False938 939 # decoder layers940 all_hidden_states = () if output_hidden_states else None941 all_self_attns = () if output_attentions else None942 next_decoder_cache = () if use_cache else None943 944 for idx, decoder_layer in enumerate(self.layers):945 if output_hidden_states:946 all_hidden_states += (hidden_states,)947 948 past_key_value = past_key_values[idx] if past_key_values is not None else None949 950 if self.gradient_checkpointing and self.training:951 952 def create_custom_forward(module):953 def custom_forward(*inputs):954 # None for past_key_value955 return module(*inputs, output_attentions, None)956 957 return custom_forward958 959 layer_outputs = torch.utils.checkpoint.checkpoint(960 create_custom_forward(decoder_layer),961 hidden_states,962 attention_mask,963 position_ids,964 None,965 )966 else:967 layer_outputs = decoder_layer(968 hidden_states,969 attention_mask=attention_mask,970 position_ids=position_ids,971 past_key_value=past_key_value,972 output_attentions=output_attentions,973 use_cache=use_cache,974 )975 976 hidden_states = layer_outputs[0]977 978 if use_cache:979 next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)980 981 if output_attentions:982 all_self_attns += (layer_outputs[1],)983 984 hidden_states = self.norm(hidden_states)985 986 # add hidden states from the last decoder layer987 if output_hidden_states:988 all_hidden_states += (hidden_states,)989 990 next_cache = next_decoder_cache if use_cache else None991 if not return_dict:992 return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)993 return BaseModelOutputWithPast(994 last_hidden_state=hidden_states,995 past_key_values=next_cache,996 hidden_states=all_hidden_states,997 attentions=all_self_attns,998 )999 1000 1001# Modified from transformers.model.llama.modeling_llama.LlamaForCausalLM1002class InternLM2ForCausalLM(InternLM2PreTrainedModel):1003 _auto_class = 'AutoModelForCausalLM'1004 1005 _tied_weights_keys = ['output.weight']1006 1007 def __init__(self, config):1008 super().__init__(config)1009 self.model = InternLM2Model(config)1010 self.vocab_size = config.vocab_size1011 self.output = nn.Linear(config.hidden_size, config.vocab_size, bias=False)1012 1013 # Initialize weights and apply final processing1014 self.post_init()1015 1016 def get_input_embeddings(self):1017 return self.model.tok_embeddings1018 1019 def set_input_embeddings(self, value):1020 self.model.tok_embeddings = value1021 1022 def get_output_embeddings(self):1023 return self.output1024 1025 def set_output_embeddings(self, new_embeddings):1026 self.output = new_embeddings1027 1028 def set_decoder(self, decoder):1029 self.model = decoder1030 1031 def get_decoder(self):1032 return self.model1033 1034 @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING)1035 @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)1036 def forward(1037 self,1038 input_ids: torch.LongTensor = None,1039 attention_mask: Optional[torch.Tensor] = None,1040 position_ids: Optional[torch.LongTensor] = None,1041 past_key_values: Optional[List[torch.FloatTensor]] = None,1042 inputs_embeds: Optional[torch.FloatTensor] = None,1043 labels: Optional[torch.LongTensor] = None,1044 use_cache: Optional[bool] = None,1045 output_attentions: Optional[bool] = None,1046 output_hidden_states: Optional[bool] = None,1047 return_dict: Optional[bool] = None,1048 ) -> Union[Tuple, CausalLMOutputWithPast]:1049 r"""1050 Args:1051 labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):1052 Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,1053 config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored1054 (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.1055 1056 Returns:1057 1058 Example:1059 1060 ```python1061 >>> from transformers import AutoTokenizer, InternLM2ForCausalLM1062 1063 >>> model = InternLM2ForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)1064 >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)1065 1066 >>> prompt = "Hey, are you conscious? Can you talk to me?"1067 >>> inputs = tokenizer(prompt, return_tensors="pt")1068 1069 >>> # Generate1070 >>> generate_ids = model.generate(inputs.input_ids, max_length=30)1071 >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]1072 "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."1073 ```"""1074 1075 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions1076 output_hidden_states = (1077 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states1078 )1079 return_dict = return_dict if return_dict is not None else self.config.use_return_dict1080 1081 # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)1082 outputs = self.model(1083 input_ids=input_ids,1084 attention_mask=attention_mask,1085 position_ids=position_ids,1086 past_key_values=past_key_values,1087 inputs_embeds=inputs_embeds,1088 use_cache=use_cache,1089 output_attentions=output_attentions,1090 output_hidden_states=output_hidden_states,1091 return_dict=return_dict,1092 )1093 1094 hidden_states = outputs[0]1095 logits = self.output(hidden_states)1096 logits = logits.float()1097 1098 loss = None1099 if labels is not None:1100 # Shift so that tokens < n predict n1101 shift_logits = logits[..., :-1, :].contiguous()1102 shift_labels = labels[..., 1:].contiguous()1103 # Flatten the tokens1104 loss_fct = CrossEntropyLoss()1105 shift_logits = shift_logits.view(-1, self.config.vocab_size)1106 shift_labels = shift_labels.view(-1)1107 # Enable model parallelism1108 shift_labels = shift_labels.to(shift_logits.device)1109 loss = loss_fct(shift_logits, shift_labels)1110 1111 if not return_dict:1112 output = (logits,) + outputs[1:]1113 return (loss,) + output if loss is not None else output1114 1115 device = input_ids.device if input_ids is not None else inputs_embeds.device1116 output = CausalLMOutputWithPast(1117 loss=loss,1118 logits=logits,1119 past_key_values=outputs.past_key_values,1120 hidden_states=outputs.hidden_states,1121 attentions=outputs.attentions,1122 )1123 output['logits'] = output['logits'].to(device)1124 return output1125 1126 def prepare_inputs_for_generation(1127 self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs1128 ):1129 if past_key_values is not None:1130 past_length = past_key_values[0][0].shape[2]1131 1132 # Some generation methods already pass only the last input ID1133 if input_ids.shape[1] > past_length:1134 remove_prefix_length = past_length1135 else:1136 # Default to old behavior: keep only final ID1137 remove_prefix_length = input_ids.shape[1] - 11138 1139 input_ids = input_ids[:, remove_prefix_length:]1140 1141 position_ids = kwargs.get('position_ids', None)1142 if attention_mask is not None and position_ids is None:1143 # create position_ids on the fly for batch generation1144 position_ids = attention_mask.long().cumsum(-1) - 11145 position_ids.masked_fill_(attention_mask == 0, 1)1146 if past_key_values:1147 position_ids = position_ids[:, -input_ids.shape[1]:]1148 1149 # if `inputs_embeds` are passed, we only want to use them in the 1st generation step1150 if inputs_embeds is not None and past_key_values is None:1151 model_inputs = {'inputs_embeds': inputs_embeds}1152 else:1153 model_inputs = {'input_ids': input_ids}1154 1155 model_inputs.update(1156 {1157 'position_ids': position_ids,1158 'past_key_values': past_key_values,1159 'use_cache': kwargs.get('use_cache'),1160 'attention_mask': attention_mask,1161 }1162 )1163 return model_inputs1164 1165 @staticmethod1166 def _reorder_cache(past_key_values, beam_idx):1167 reordered_past = ()1168 for layer_past in past_key_values:1169 reordered_past += (1170 tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),1171 )1172 return reordered_past1173 1174 def build_inputs(self, tokenizer, query: str, history: List[Tuple[str, str]] = [], meta_instruction=''):1175 if tokenizer.add_bos_token:1176 prompt = ''1177 else:1178 prompt = tokenizer.bos_token1179 if meta_instruction:1180 prompt += f"""<|im_start|>system\n{meta_instruction}<|im_end|>\n"""1181 for record in history:1182 prompt += f"""<|im_start|>user\n{record[0]}<|im_end|>\n<|im_start|>assistant\n{record[1]}<|im_end|>\n"""1183 prompt += f"""<|im_start|>user\n{query}<|im_end|>\n<|im_start|>assistant\n"""1184 return tokenizer([prompt], return_tensors='pt')1185 1186 @torch.no_grad()1187 def chat(1188 self,1189 tokenizer,1190 query: str,1191 history: List[Tuple[str, str]] = [],1192 streamer: Optional[BaseStreamer] = None,1193 max_new_tokens: int = 1024,1194 do_sample: bool = True,1195 temperature: float = 0.8,1196 top_p: float = 0.8,1197 meta_instruction: str = 'You are an AI assistant whose name is InternLM (书生·浦语).\n'1198 '- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.\n'1199 '- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文.',1200 **kwargs,