brycewang2018/EmoLLM-mother
212
1# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved.2#3# This code is based on transformers/src/transformers/models/llama/modeling_llama.py4#5# Licensed under the Apache License, Version 2.0 (the "License");6# you may not use this file except in compliance with the License.7# You may obtain a copy of the License at8#9# http://www.apache.org/licenses/LICENSE-2.010#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.14# See the License for the specific language governing permissions and15# limitations under the License.16""" PyTorch InternLM2 model."""17import math18import queue19import threading20import warnings21from typing import List, Optional, Tuple, Union22 23import torch24import torch.nn.functional as F25import torch.utils.checkpoint26from einops import rearrange27from torch import nn28from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss29from transformers.activations import ACT2FN30from transformers.modeling_outputs import (31 BaseModelOutputWithPast,32 CausalLMOutputWithPast,33 SequenceClassifierOutputWithPast,34)35from transformers.modeling_utils import PreTrainedModel36from transformers.utils import (37 add_start_docstrings,38 add_start_docstrings_to_model_forward,39 logging,40 replace_return_docstrings,41)42 43try:44 from transformers.generation.streamers import BaseStreamer45except: # noqa # pylint: disable=bare-except46 BaseStreamer = None47 48from .configuration_internlm2 import InternLM2Config49 50logger = logging.get_logger(__name__)51 52_CONFIG_FOR_DOC = "InternLM2Config"53 54flash_attn_func, flash_attn_varlen_func = None, None55pad_input, index_first_axis, unpad_input = None, None, None56def _import_flash_attn():57 global flash_attn_func, flash_attn_varlen_func58 global pad_input, index_first_axis, unpad_input59 try:60 from flash_attn import flash_attn_func as _flash_attn_func, flash_attn_varlen_func as _flash_attn_varlen_func61 from flash_attn.bert_padding import pad_input as _pad_input, index_first_axis as _index_first_axis, unpad_input as _unpad_input62 flash_attn_func, flash_attn_varlen_func = _flash_attn_func, _flash_attn_varlen_func63 pad_input, index_first_axis, unpad_input = _pad_input, _index_first_axis, _unpad_input64 except ImportError:65 raise ImportError("flash_attn is not installed.")66 67# Copied from transformers.models.llama.modeling_llama._get_unpad_data68def _get_unpad_data(attention_mask):69 seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)70 indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()71 max_seqlen_in_batch = seqlens_in_batch.max().item()72 cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))73 return (74 indices,75 cu_seqlens,76 max_seqlen_in_batch,77 )78 79 80# Copied from transformers.models.bart.modeling_bart._make_causal_mask81def _make_causal_mask(82 input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 083):84 """85 Make causal mask used for bi-directional self-attention.86 """87 bsz, tgt_len = input_ids_shape88 mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device)89 mask_cond = torch.arange(mask.size(-1), device=device)90 mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)91 mask = mask.to(dtype)92 93 if past_key_values_length > 0:94 mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)95 return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)96 97 98# Copied from transformers.models.bart.modeling_bart._expand_mask99def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):100 """101 Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.102 """103 bsz, src_len = mask.size()104 tgt_len = tgt_len if tgt_len is not None else src_len105 106 expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)107 108 inverted_mask = 1.0 - expanded_mask109 110 return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)111 112 113# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->InternLM2114class InternLM2RMSNorm(nn.Module):115 def __init__(self, hidden_size, eps=1e-6):116 """117 InternLM2RMSNorm is equivalent to T5LayerNorm118 """119 super().__init__()120 self.weight = nn.Parameter(torch.ones(hidden_size))121 self.variance_epsilon = eps122 123 def forward(self, hidden_states):124 input_dtype = hidden_states.dtype125 hidden_states = hidden_states.to(torch.float32)126 variance = hidden_states.pow(2).mean(-1, keepdim=True)127 hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)128 return self.weight * hidden_states.to(input_dtype)129 130 131# Copied from transformers.model.llama.modeling_llama.LlamaRotaryEmbedding with Llama->InternLM2132class InternLM2RotaryEmbedding(nn.Module):133 def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):134 super().__init__()135 136 self.dim = dim137 self.max_position_embeddings = max_position_embeddings138 self.base = base139 inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))140 self.register_buffer("inv_freq", inv_freq, persistent=False)141 142 # Build here to make `torch.jit.trace` work.143 self._set_cos_sin_cache(144 seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype()145 )146 147 def _set_cos_sin_cache(self, seq_len, device, dtype):148 self.max_seq_len_cached = seq_len149 t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)150 151 freqs = torch.einsum("i,j->ij", t, self.inv_freq)152 # Different from paper, but it uses a different permutation in order to obtain the same calculation153 emb = torch.cat((freqs, freqs), dim=-1)154 self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)155 self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)156 157 def forward(self, x, seq_len=None):158 # x: [bs, num_attention_heads, seq_len, head_size]159 if seq_len > self.max_seq_len_cached:160 self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=torch.float32)161 162 return (163 self.cos_cached[:seq_len].to(dtype=x.dtype),164 self.sin_cached[:seq_len].to(dtype=x.dtype),165 )166 167 168# Copied from transformers.model.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding with Llama->InternLM2169class InternLM2LinearScalingRotaryEmbedding(InternLM2RotaryEmbedding):170 """InternLM2RotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""171 172 def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):173 self.scaling_factor = scaling_factor174 super().__init__(dim, max_position_embeddings, base, device)175 176 def _set_cos_sin_cache(self, seq_len, device, dtype):177 self.max_seq_len_cached = seq_len178 t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)179 t = t / self.scaling_factor180 181 freqs = torch.einsum("i,j->ij", t, self.inv_freq)182 # Different from paper, but it uses a different permutation in order to obtain the same calculation183 emb = torch.cat((freqs, freqs), dim=-1)184 self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)185 self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)186 187 188# Copied from transformers.model.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding with Llama->InternLM2189class InternLM2DynamicNTKScalingRotaryEmbedding(InternLM2RotaryEmbedding):190 """InternLM2RotaryEmbedding extended with Dynamic NTK scaling.191 Credits to the Reddit users /u/bloc97 and /u/emozilla.192 """193 194 def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):195 self.scaling_factor = scaling_factor196 super().__init__(dim, max_position_embeddings, base, device)197 198 def _set_cos_sin_cache(self, seq_len, device, dtype):199 self.max_seq_len_cached = seq_len200 201 if seq_len > self.max_position_embeddings:202 base = self.base * (203 (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1)204 ) ** (self.dim / (self.dim - 2))205 inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))206 self.register_buffer("inv_freq", inv_freq, persistent=False)207 208 t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)209 210 freqs = torch.einsum("i,j->ij", t, self.inv_freq)211 # Different from paper, but it uses a different permutation in order to obtain the same calculation212 emb = torch.cat((freqs, freqs), dim=-1)213 self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)214 self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)215 216 217# Copied from transformers.model.llama.modeling_llama.rotate_half218def rotate_half(x):219 """Rotates half the hidden dims of the input."""220 x1 = x[..., : x.shape[-1] // 2]221 x2 = x[..., x.shape[-1] // 2 :]222 return torch.cat((-x2, x1), dim=-1)223 224 225# Copied from transformers.model.llama.modeling_llama.apply_rotary_pos_emb226def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):227 """Applies Rotary Position Embedding to the query and key tensors."""228 cos = cos[position_ids].unsqueeze(unsqueeze_dim)229 sin = sin[position_ids].unsqueeze(unsqueeze_dim)230 q_embed = (q * cos) + (rotate_half(q) * sin)231 k_embed = (k * cos) + (rotate_half(k) * sin)232 return q_embed, k_embed233 234 235class InternLM2MLP(nn.Module):236 def __init__(self, config):237 super().__init__()238 self.config = config239 self.hidden_size = config.hidden_size240 self.intermediate_size = config.intermediate_size241 self.w1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)242 self.w3 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)243 self.w2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)244 self.act_fn = ACT2FN[config.hidden_act]245 246 def forward(self, x):247 down_proj = self.w2(self.act_fn(self.w1(x)) * self.w3(x))248 249 return down_proj250 251 252# Copied from transformers.model.llama.modeling_llama.repeat_kv253def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:254 """255 This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,256 num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)257 """258 batch, num_key_value_heads, slen, head_dim = hidden_states.shape259 if n_rep == 1:260 return hidden_states261 hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)262 return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)263 264 265# Modified from transformers.model.llama.modeling_llama.LlamaAttention266class InternLM2Attention(nn.Module):267 """Multi-headed attention from 'Attention Is All You Need' paper"""268 269 def __init__(self, config: InternLM2Config):270 super().__init__()271 self.config = config272 self.hidden_size = config.hidden_size273 self.num_heads = config.num_attention_heads274 self.head_dim = self.hidden_size // self.num_heads275 self.num_key_value_heads = config.num_key_value_heads276 self.num_key_value_groups = self.num_heads // self.num_key_value_heads277 self.max_position_embeddings = config.max_position_embeddings278 self.is_causal = True279 280 if (self.head_dim * self.num_heads) != self.hidden_size:281 raise ValueError(282 f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"283 f" and `num_heads`: {self.num_heads})."284 )285 286 self.wqkv = nn.Linear(287 self.hidden_size,288 (self.num_heads + 2 * self.num_key_value_heads) * self.head_dim,289 bias=config.bias,290 )291 292 self.wo = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.bias)293 self._init_rope()294 295 def _init_rope(self):296 if self.config.rope_scaling is None:297 self.rotary_emb = InternLM2RotaryEmbedding(298 self.head_dim,299 max_position_embeddings=self.max_position_embeddings,300 base=self.config.rope_theta,301 )302 else:303 scaling_type = self.config.rope_scaling["type"]304 scaling_factor = self.config.rope_scaling["factor"]305 if scaling_type == "dynamic":306 self.rotary_emb = InternLM2DynamicNTKScalingRotaryEmbedding(307 self.head_dim,308 max_position_embeddings=self.max_position_embeddings,309 base=self.config.rope_theta,310 scaling_factor=scaling_factor,311 )312 elif scaling_type == "linear":313 self.rotary_emb = InternLM2LinearScalingRotaryEmbedding(314 self.head_dim,315 max_position_embeddings=self.max_position_embeddings,316 base=self.config.rope_theta,317 scaling_factor=scaling_factor,318 )319 else:320 raise ValueError("Currently we only support rotary embedding's type being 'dynamic' or 'linear'.")321 return self.rotary_emb322 323 def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):324 return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()325 326 def forward(327 self,328 hidden_states: torch.Tensor,329 attention_mask: Optional[torch.Tensor] = None,330 position_ids: Optional[torch.LongTensor] = None,331 past_key_value: Optional[Tuple[torch.Tensor]] = None,332 output_attentions: bool = False,333 use_cache: bool = False,334 **kwargs,335 ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:336 if "padding_mask" in kwargs:337 warnings.warn(338 "Passing `padding_mask` is deprecated and will be removed in v4.37. "339 "Please make sure use `attention_mask` instead.`"340 )341 342 bsz, q_len, _ = hidden_states.size()343 344 qkv_states = self.wqkv(hidden_states)345 346 qkv_states = rearrange(347 qkv_states,348 "b q (h gs d) -> b q h gs d",349 gs=2 + self.num_key_value_groups,350 d=self.head_dim,351 )352 353 query_states = qkv_states[..., : self.num_key_value_groups, :]354 query_states = rearrange(query_states, "b q h gs d -> b q (h gs) d")355 key_states = qkv_states[..., -2, :]356 value_states = qkv_states[..., -1, :]357 358 query_states = query_states.transpose(1, 2)359 key_states = key_states.transpose(1, 2)360 value_states = value_states.transpose(1, 2)361 362 kv_seq_len = key_states.shape[-2]363 if past_key_value is not None:364 kv_seq_len += past_key_value[0].shape[-2]365 cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)366 query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)367 368 if past_key_value is not None:369 # reuse k, v, self_attention370 key_states = torch.cat([past_key_value[0], key_states], dim=2)371 value_states = torch.cat([past_key_value[1], value_states], dim=2)372 373 past_key_value = (key_states, value_states) if use_cache else None374 375 key_states = repeat_kv(key_states, self.num_key_value_groups)376 value_states = repeat_kv(value_states, self.num_key_value_groups)377 378 attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)379 380 if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):381 raise ValueError(382 f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"383 f" {attn_weights.size()}"384 )385 386 if attention_mask is not None:387 if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):388 raise ValueError(389 f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"390 )391 attn_weights = attn_weights + attention_mask392 393 # upcast attention to fp32394 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)395 attn_output = torch.matmul(attn_weights, value_states)396 397 if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):398 raise ValueError(399 f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"400 f" {attn_output.size()}"401 )402 403 attn_output = attn_output.transpose(1, 2).contiguous()404 attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)405 406 attn_output = self.wo(attn_output)407 408 if not output_attentions:409 attn_weights = None410 411 return attn_output, attn_weights, past_key_value412 413 414# Modified from transformers.model.llama.modeling_llama.InternLM2FlashAttention2415class InternLM2FlashAttention2(InternLM2Attention):416 """417 InternLM2 flash attention module. This module inherits from `InternLM2Attention` as the weights of the module stays418 untouched. The only required change would be on the forward pass where it needs to correctly call the public API of419 flash attention and deal with padding tokens in case the input contains any of them.420 """421 422 def forward(423 self,424 hidden_states: torch.Tensor,425 attention_mask: Optional[torch.LongTensor] = None,426 position_ids: Optional[torch.LongTensor] = None,427 past_key_value: Optional[Tuple[torch.Tensor]] = None,428 output_attentions: bool = False,429 use_cache: bool = False,430 **kwargs,431 ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:432 # InternLM2FlashAttention2 attention does not support output_attentions433 if "padding_mask" in kwargs:434 warnings.warn(435 "Passing `padding_mask` is deprecated and will be removed in v4.37. "436 "Please make sure use `attention_mask` instead.`"437 )438 439 # overwrite attention_mask with padding_mask440 attention_mask = kwargs.pop("padding_mask")441 442 output_attentions = False443 444 bsz, q_len, _ = hidden_states.size()445 446 qkv_states = self.wqkv(hidden_states)447 448 qkv_states = rearrange(449 qkv_states,450 "b q (h gs d) -> b q h gs d",451 gs=2 + self.num_key_value_groups,452 d=self.head_dim,453 )454 455 query_states = qkv_states[..., : self.num_key_value_groups, :]456 query_states = rearrange(query_states, "b q h gs d -> b q (h gs) d")457 key_states = qkv_states[..., -2, :]458 value_states = qkv_states[..., -1, :]459 460 query_states = query_states.transpose(1, 2)461 key_states = key_states.transpose(1, 2)462 value_states = value_states.transpose(1, 2)463 464 kv_seq_len = key_states.shape[-2]465 if past_key_value is not None:466 kv_seq_len += past_key_value[0].shape[-2]467 468 cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)469 470 query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)471 472 if past_key_value is not None:473 # reuse k, v, self_attention474 key_states = torch.cat([past_key_value[0], key_states], dim=2)475 value_states = torch.cat([past_key_value[1], value_states], dim=2)476 477 past_key_value = (key_states, value_states) if use_cache else None478 479 query_states = query_states.transpose(1, 2)480 key_states = key_states.transpose(1, 2)481 value_states = value_states.transpose(1, 2)482 483 attn_output = self._flash_attention_forward(484 query_states, key_states, value_states, attention_mask, q_len485 )486 attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()487 attn_output = self.wo(attn_output)488 489 if not output_attentions:490 attn_weights = None491 492 return attn_output, attn_weights, past_key_value493 494 def _flash_attention_forward(495 self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None496 ):497 """498 Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token499 first unpad the input, then computes the attention scores and pad the final attention scores.500 501 Args:502 query_states (`torch.Tensor`):503 Input query states to be passed to Flash Attention API504 key_states (`torch.Tensor`):505 Input key states to be passed to Flash Attention API506 value_states (`torch.Tensor`):507 Input value states to be passed to Flash Attention API508 attention_mask (`torch.Tensor`):509 The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the510 position of padding tokens and 1 for the position of non-padding tokens.511 dropout (`int`, *optional*):512 Attention dropout513 softmax_scale (`float`, *optional*):514 The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)515 """516 # Contains at least one padding token in the sequence517 causal = self.is_causal and query_length != 1518 if attention_mask is not None:519 batch_size = query_states.shape[0]520 query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._unpad_input(521 query_states, key_states, value_states, attention_mask, query_length522 )523 524 cu_seqlens_q, cu_seqlens_k = cu_seq_lens525 max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens526 527 attn_output_unpad = flash_attn_varlen_func(528 query_states,529 key_states,530 value_states,531 cu_seqlens_q=cu_seqlens_q,532 cu_seqlens_k=cu_seqlens_k,533 max_seqlen_q=max_seqlen_in_batch_q,534 max_seqlen_k=max_seqlen_in_batch_k,535 dropout_p=dropout,536 softmax_scale=softmax_scale,537 causal=causal,538 )539 540 attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)541 else:542 attn_output = flash_attn_func(543 query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal544 )545 546 return attn_output547 548 def _unpad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):549 indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)550 batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape551 552 key_layer = index_first_axis(553 key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k554 )555 value_layer = index_first_axis(556 value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k557 )558 559 if query_length == kv_seq_len:560 query_layer = index_first_axis(561 query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k562 )563 cu_seqlens_q = cu_seqlens_k564 max_seqlen_in_batch_q = max_seqlen_in_batch_k565 indices_q = indices_k566 elif query_length == 1:567 max_seqlen_in_batch_q = 1568 cu_seqlens_q = torch.arange(569 batch_size + 1, dtype=torch.int32, device=query_layer.device570 ) # There is a memcpy here, that is very bad.571 indices_q = cu_seqlens_q[:-1]572 query_layer = query_layer.squeeze(1)573 else:574 # The -q_len: slice assumes left padding.575 attention_mask = attention_mask[:, -query_length:]576 query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)577 578 return (579 query_layer,580 key_layer,581 value_layer,582 indices_q.to(torch.int64),583 (cu_seqlens_q, cu_seqlens_k),584 (max_seqlen_in_batch_q, max_seqlen_in_batch_k),585 )586 587INTERNLM2_ATTENTION_CLASSES = {588 "eager": InternLM2Attention,589 "flash_attention_2": InternLM2FlashAttention2,590}591 592# Modified from transformers.model.llama.modeling_llama.LlamaDecoderLayer593class InternLM2DecoderLayer(nn.Module):594 def __init__(self, config: InternLM2Config):595 super().__init__()596 self.hidden_size = config.hidden_size597 598 self.attention = INTERNLM2_ATTENTION_CLASSES[config.attn_implementation](config=config)599 600 self.feed_forward = InternLM2MLP(config)601 self.attention_norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)602 self.ffn_norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)603 604 def forward(605 self,606 hidden_states: torch.Tensor,607 attention_mask: Optional[torch.Tensor] = None,608 position_ids: Optional[torch.LongTensor] = None,609 past_key_value: Optional[Tuple[torch.Tensor]] = None,610 output_attentions: Optional[bool] = False,611 use_cache: Optional[bool] = False,612 **kwargs,613 ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:614 """615 Args:616 hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`617 attention_mask (`torch.FloatTensor`, *optional*):618 attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,619 query_sequence_length, key_sequence_length)` if default attention is used.620 output_attentions (`bool`, *optional*):621 Whether or not to return the attentions tensors of all attention layers. See `attentions` under622 returned tensors for more detail.623 use_cache (`bool`, *optional*):624 If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding625 (see `past_key_values`).626 past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states627 """628 if "padding_mask" in kwargs:629 warnings.warn(630 "Passing `padding_mask` is deprecated and will be removed in v4.37. "631 "Please make sure use `attention_mask` instead.`"632 )633 634 residual = hidden_states635 636 hidden_states = self.attention_norm(hidden_states)637 638 # Self Attention639 hidden_states, self_attn_weights, present_key_value = self.attention(640 hidden_states=hidden_states,641 attention_mask=attention_mask,642 position_ids=position_ids,643 past_key_value=past_key_value,644 output_attentions=output_attentions,645 use_cache=use_cache,646 **kwargs,647 )648 hidden_states = residual + hidden_states649 650 # Fully Connected651 residual = hidden_states652 hidden_states = self.ffn_norm(hidden_states)653 hidden_states = self.feed_forward(hidden_states)654 hidden_states = residual + hidden_states655 656 outputs = (hidden_states,)657 658 if output_attentions:659 outputs += (self_attn_weights,)660 661 if use_cache:662 outputs += (present_key_value,)663 664 return outputs665 666 667InternLM2_START_DOCSTRING = r"""668 This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the669 library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads670 etc.)671 672 This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.673 Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage674 and behavior.675 676 Parameters:677 config ([`InternLM2Config`]):678 Model configuration class with all the parameters of the model. Initializing with a config file does not679 load the weights associated with the model, only the configuration. Check out the680 [`~PreTrainedModel.from_pretrained`] method to load the model weights.681"""682 683 684# Copied from transformers.models.llama.modeling_llama.LlamaPreTrainedModel with Llama->InternLM2685@add_start_docstrings(686 "The bare InternLM2 Model outputting raw hidden-states without any specific head on top.",687 InternLM2_START_DOCSTRING,688)689class InternLM2PreTrainedModel(PreTrainedModel):690 config_class = InternLM2Config691 base_model_prefix = "model"692 supports_gradient_checkpointing = True693 _no_split_modules = ["InternLM2DecoderLayer"]694 _skip_keys_device_placement = "past_key_values"695 696 def _init_weights(self, module):697 std = self.config.initializer_range698 if isinstance(module, nn.Linear):699 module.weight.data.normal_(mean=0.0, std=std)700 if module.bias is not None:701 module.bias.data.zero_()702 elif isinstance(module, nn.Embedding):703 module.weight.data.normal_(mean=0.0, std=std)704 if module.padding_idx is not None:705 module.weight.data[module.padding_idx].zero_()706 707 708InternLM2_INPUTS_DOCSTRING = r"""709 Args:710 input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):711 Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide712 it.713 714 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and715 [`PreTrainedTokenizer.__call__`] for details.716 717 [What are input IDs?](../glossary#input-ids)718 attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):719 Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:720 721 - 1 for tokens that are **not masked**,722 - 0 for tokens that are **masked**.723 724 [What are attention masks?](../glossary#attention-mask)725 726 Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and727 [`PreTrainedTokenizer.__call__`] for details.728 729 If `past_key_values` is used, optionally only the last `input_ids` have to be input (see730 `past_key_values`).731 732 If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]733 and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more734 information on the default strategy.735 736 - 1 indicates the head is **not masked**,737 - 0 indicates the head is **masked**.738 position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):739 Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,740 config.n_positions - 1]`.741 742 [What are position IDs?](../glossary#position-ids)743 past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or744 when `config.use_cache=True`):745 Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape746 `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape747 `(batch_size, num_heads, decoder_sequence_length, embed_size_per_head)`.748 749 Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention750 blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.751 752 If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't753 have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`754 of shape `(batch_size, sequence_length)`.755 inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):756 Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This757 is useful if you want more control over how to convert `input_ids` indices into associated vectors than the758 model's internal embedding lookup matrix.759 use_cache (`bool`, *optional*):760 If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see761 `past_key_values`).762 output_attentions (`bool`, *optional*):763 Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned764 tensors for more detail.765 output_hidden_states (`bool`, *optional*):766 Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for767 more detail.768 return_dict (`bool`, *optional*):769 Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.770"""771 772 773# Modified from transformers.model.llama.modeling_llama.LlamaModel774@add_start_docstrings(775 "The bare InternLM2 Model outputting raw hidden-states without any specific head on top.",776 InternLM2_START_DOCSTRING,777)778class InternLM2Model(InternLM2PreTrainedModel):779 """780 Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`InternLM2DecoderLayer`]781 782 Args:783 config: InternLM2Config784 """785 786 _auto_class = "AutoModel"787 788 def __init__(self, config: InternLM2Config):789 super().__init__(config)790 self.padding_idx = config.pad_token_id791 self.vocab_size = config.vocab_size792 self.config = config793 794 self.tok_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)795 796 self.layers = nn.ModuleList([InternLM2DecoderLayer(config) for _ in range(config.num_hidden_layers)])797 self.norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)798 799 self.gradient_checkpointing = False800 # Initialize weights and apply final processing801 self.post_init()802 803 def get_input_embeddings(self):804 return self.tok_embeddings805 806 def set_input_embeddings(self, value):807 self.tok_embeddings = value808 809 def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):810 # create causal mask811 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]812 combined_attention_mask = None813 if input_shape[-1] > 1:814 combined_attention_mask = _make_causal_mask(815 input_shape,816 inputs_embeds.dtype,817 device=inputs_embeds.device,818 past_key_values_length=past_key_values_length,819 )820 821 if attention_mask is not None:822 # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]823 expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(824 inputs_embeds.device825 )826 combined_attention_mask = (827 expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask828 )829 830 return combined_attention_mask831 832 @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING)833 def forward(834 self,835 input_ids: torch.LongTensor = None,836 attention_mask: Optional[torch.Tensor] = None,837 position_ids: Optional[torch.LongTensor] = None,838 past_key_values: Optional[List[torch.FloatTensor]] = None,839 inputs_embeds: Optional[torch.FloatTensor] = None,840 use_cache: Optional[bool] = None,841 output_attentions: Optional[bool] = None,842 output_hidden_states: Optional[bool] = None,843 return_dict: Optional[bool] = None,844 ) -> Union[Tuple, BaseModelOutputWithPast]:845 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions846 output_hidden_states = (847 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states848 )849 use_cache = use_cache if use_cache is not None else self.config.use_cache850 851 return_dict = return_dict if return_dict is not None else self.config.use_return_dict852 853 if self.config.attn_implementation == "flash_attention_2":854 _import_flash_attn()855 856 # retrieve input_ids and inputs_embeds857 if input_ids is not None and inputs_embeds is not None:858 raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")859 elif input_ids is not None:860 batch_size, seq_length = input_ids.shape[:2]861 elif inputs_embeds is not None:862 batch_size, seq_length = inputs_embeds.shape[:2]863 else:864 raise ValueError("You have to specify either input_ids or inputs_embeds")865 866 seq_length_with_past = seq_length867 past_key_values_length = 0868 if past_key_values is not None:869 past_key_values_length = past_key_values[0][0].shape[2]870 seq_length_with_past = seq_length_with_past + past_key_values_length871 872 if position_ids is None:873 device = input_ids.device if input_ids is not None else inputs_embeds.device874 position_ids = torch.arange(875 past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device876 )877 position_ids = position_ids.unsqueeze(0)878 879 if inputs_embeds is None:880 inputs_embeds = self.tok_embeddings(input_ids)881 882 if self.config.attn_implementation == "flash_attention_2":883 # 2d mask is passed through the layers884 attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None885 else:886 if attention_mask is None:887 attention_mask = torch.ones(888 (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device889 )890 attention_mask = self._prepare_decoder_attention_mask(891 attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length892 )893 894 # embed positions895 hidden_states = inputs_embeds896 897 if self.gradient_checkpointing and self.training:898 if use_cache:899 logger.warning_once(900 "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."901 )902 use_cache = False903 904 # decoder layers905 all_hidden_states = () if output_hidden_states else None906 all_self_attns = () if output_attentions else None907 next_decoder_cache = () if use_cache else None908 909 for idx, decoder_layer in enumerate(self.layers):910 if output_hidden_states:911 all_hidden_states += (hidden_states,)912 913 past_key_value = past_key_values[idx] if past_key_values is not None else None914 915 if self.gradient_checkpointing and self.training:916 917 def create_custom_forward(module):918 def custom_forward(*inputs):919 # None for past_key_value920 return module(*inputs, output_attentions, None)921 922 return custom_forward923 924 layer_outputs = torch.utils.checkpoint.checkpoint(925 create_custom_forward(decoder_layer),926 hidden_states,927 attention_mask,928 position_ids,929 None,930 )931 else:932 layer_outputs = decoder_layer(933 hidden_states,934 attention_mask=attention_mask,935 position_ids=position_ids,936 past_key_value=past_key_value,937 output_attentions=output_attentions,938 use_cache=use_cache,939 )940 941 hidden_states = layer_outputs[0]942 943 if use_cache:944 next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)945 946 if output_attentions:947 all_self_attns += (layer_outputs[1],)948 949 hidden_states = self.norm(hidden_states)950 951 # add hidden states from the last decoder layer952 if output_hidden_states:953 all_hidden_states += (hidden_states,)954 955 next_cache = next_decoder_cache if use_cache else None956 if not return_dict:957 return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)958 return BaseModelOutputWithPast(959 last_hidden_state=hidden_states,960 past_key_values=next_cache,961 hidden_states=all_hidden_states,962 attentions=all_self_attns,963 )964 965 966# Modified from transformers.model.llama.modeling_llama.LlamaForCausalLM967class InternLM2ForCausalLM(InternLM2PreTrainedModel):968 _auto_class = "AutoModelForCausalLM"969 970 _tied_weights_keys = ["output.weight"]971 972 def __init__(self, config):973 super().__init__(config)974 self.model = InternLM2Model(config)975 self.vocab_size = config.vocab_size976 self.output = nn.Linear(config.hidden_size, config.vocab_size, bias=False)977 978 # Initialize weights and apply final processing979 self.post_init()980 981 def get_input_embeddings(self):982 return self.model.tok_embeddings983 984 def set_input_embeddings(self, value):985 self.model.tok_embeddings = value986 987 def get_output_embeddings(self):988 return self.output989 990 def set_output_embeddings(self, new_embeddings):991 self.output = new_embeddings992 993 def set_decoder(self, decoder):994 self.model = decoder995 996 def get_decoder(self):997 return self.model998 999 @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING)1000 @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)1001 def forward(1002 self,1003 input_ids: torch.LongTensor = None,1004 attention_mask: Optional[torch.Tensor] = None,1005 position_ids: Optional[torch.LongTensor] = None,1006 past_key_values: Optional[List[torch.FloatTensor]] = None,1007 inputs_embeds: Optional[torch.FloatTensor] = None,1008 labels: Optional[torch.LongTensor] = None,1009 use_cache: Optional[bool] = None,1010 output_attentions: Optional[bool] = None,1011 output_hidden_states: Optional[bool] = None,1012 return_dict: Optional[bool] = None,1013 ) -> Union[Tuple, CausalLMOutputWithPast]:1014 r"""1015 Args:1016 labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):1017 Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,1018 config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored1019 (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.1020 1021 Returns:1022 1023 Example:1024 1025 ```python1026 >>> from transformers import AutoTokenizer, InternLM2ForCausalLM1027 1028 >>> model = InternLM2ForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)1029 >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)1030 1031 >>> prompt = "Hey, are you conscious? Can you talk to me?"1032 >>> inputs = tokenizer(prompt, return_tensors="pt")1033 1034 >>> # Generate1035 >>> generate_ids = model.generate(inputs.input_ids, max_length=30)1036 >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]1037 "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."1038 ```"""1039 1040 output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions1041 output_hidden_states = (1042 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states1043 )1044 return_dict = return_dict if return_dict is not None else self.config.use_return_dict1045 1046 # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)1047 outputs = self.model(1048 input_ids=input_ids,1049 attention_mask=attention_mask,1050 position_ids=position_ids,1051 past_key_values=past_key_values,1052 inputs_embeds=inputs_embeds,1053 use_cache=use_cache,1054 output_attentions=output_attentions,1055 output_hidden_states=output_hidden_states,1056 return_dict=return_dict,1057 )1058 1059 hidden_states = outputs[0]1060 logits = self.output(hidden_states)1061 logits = logits.float()1062 1063 loss = None1064 if labels is not None:1065 # Shift so that tokens < n predict n1066 shift_logits = logits[..., :-1, :].contiguous()1067 shift_labels = labels[..., 1:].contiguous()1068 # Flatten the tokens1069 loss_fct = CrossEntropyLoss()1070 shift_logits = shift_logits.view(-1, self.config.vocab_size)1071 shift_labels = shift_labels.view(-1)1072 # Enable model parallelism1073 shift_labels = shift_labels.to(shift_logits.device)1074 loss = loss_fct(shift_logits, shift_labels)1075 1076 if not return_dict:1077 output = (logits,) + outputs[1:]1078 return (loss,) + output if loss is not None else output1079 1080 return CausalLMOutputWithPast(1081 loss=loss,1082 logits=logits,1083 past_key_values=outputs.past_key_values,1084 hidden_states=outputs.hidden_states,1085 attentions=outputs.attentions,1086 )1087 1088 def prepare_inputs_for_generation(1089 self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs1090 ):1091 if past_key_values is not None:1092 past_length = past_key_values[0][0].shape[2]1093 1094 # Some generation methods already pass only the last input ID1095 if input_ids.shape[1] > past_length:1096 remove_prefix_length = past_length1097 else:1098 # Default to old behavior: keep only final ID1099 remove_prefix_length = input_ids.shape[1] - 11100 1101 input_ids = input_ids[:, remove_prefix_length:]1102 1103 position_ids = kwargs.get("position_ids", None)1104 if attention_mask is not None and position_ids is None:1105 # create position_ids on the fly for batch generation1106 position_ids = attention_mask.long().cumsum(-1) - 11107 position_ids.masked_fill_(attention_mask == 0, 1)1108 if past_key_values:1109 position_ids = position_ids[:, -input_ids.shape[1] :]1110 1111 # if `inputs_embeds` are passed, we only want to use them in the 1st generation step1112 if inputs_embeds is not None and past_key_values is None:1113 model_inputs = {"inputs_embeds": inputs_embeds}1114 else:1115 model_inputs = {"input_ids": input_ids}1116 1117 model_inputs.update(1118 {1119 "position_ids": position_ids,1120 "past_key_values": past_key_values,1121 "use_cache": kwargs.get("use_cache"),1122 "attention_mask": attention_mask,1123 }1124 )1125 return model_inputs1126 1127 @staticmethod1128 def _reorder_cache(past_key_values, beam_idx):1129 reordered_past = ()1130 for layer_past in past_key_values:1131 reordered_past += (1132 tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),1133 )1134 return reordered_past1135 1136 def build_inputs(self, tokenizer, query: str, history: List[Tuple[str, str]] = [], meta_instruction=""):1137 if tokenizer.add_bos_token:1138 prompt = ""1139 else:1140 prompt = tokenizer.bos_token1141 if meta_instruction:1142 prompt += f"""<|im_start|>system\n{meta_instruction}<|im_end|>\n"""1143 for record in history:1144 prompt += f"""<|im_start|>user\n{record[0]}<|im_end|>\n<|im_start|>assistant\n{record[1]}<|im_end|>\n"""1145 prompt += f"""<|im_start|>user\n{query}<|im_end|>\n<|im_start|>assistant\n"""1146 return tokenizer([prompt], return_tensors="pt")1147 1148 @torch.no_grad()1149 def chat(1150 self,1151 tokenizer,1152 query: str,1153 history: List[Tuple[str, str]] = [],1154 streamer: Optional[BaseStreamer] = None,1155 max_new_tokens: int = 1024,1156 do_sample: bool = True,1157 temperature: float = 0.8,1158 top_p: float = 0.8,1159 meta_instruction: str = "You are an AI assistant whose name is InternLM (书生·浦语).\n"1160 "- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.\n"1161 "- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文.",1162 **kwargs,1163 ):1164 inputs = self.build_inputs(tokenizer, query, history, meta_instruction)1165 inputs = {k: v.to(self.device) for k, v in inputs.items() if torch.is_tensor(v)}1166 # also add end-of-assistant token in eos token id to avoid unnecessary generation1167 eos_token_id = [tokenizer.eos_token_id, tokenizer.convert_tokens_to_ids(["<|im_end|>"])[0]]1168 outputs = self.generate(1169 **inputs,1170 streamer=streamer,1171 max_new_tokens=max_new_tokens,1172 do_sample=do_sample,1173 temperature=temperature,1174 top_p=top_p,1175 eos_token_id=eos_token_id,1176 **kwargs,1177 )1178 outputs = outputs[0].cpu().tolist()[len(inputs["input_ids"][0]) :]1179 response = tokenizer.decode(outputs, skip_special_tokens=True)1180 response = response.split("<|im_end|>")[0]1181 history = history + [(query, response)]1182 return response, history1183 1184 @torch.no_grad()1185 def stream_chat(1186 self,1187 tokenizer,1188 query: str,1189 history: List[Tuple[str, str]] = [],1190 max_new_tokens: int = 1024,1191 do_sample: bool = True,1192 temperature: float = 0.8,1193 top_p: float = 0.8,1194 **kwargs,1195 ):1196 """1197 Return a generator in format: (response, history)1198 Eg.1199 ('你好，有什么可以帮助您的吗', [('你好', '你好，有什么可以帮助您的吗')])1200 ('你好，有什么可以帮助您的吗？', [('你好', '你好，有什么可以帮助您的吗？')])