katuni4ka/tiny-random-codegen2

1# coding=utf-82# Copyright 2022 Salesforce authors, The EleutherAI, and HuggingFace Teams. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15""" PyTorch CodeGen model."""16 17from typing import Optional, Tuple, Union18 19import torch20import torch.utils.checkpoint21from torch import nn22from torch.nn import CrossEntropyLoss23 24from transformers.activations import ACT2FN25from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast26from transformers.modeling_utils import PreTrainedModel27from transformers.utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging28from .configuration_codegen import CodeGenConfig29 30 31logger = logging.get_logger(__name__)32 33_CHECKPOINT_FOR_DOC = "Salesforce/codegen-2B-mono"34_CONFIG_FOR_DOC = "CodeGenConfig"35_TOKENIZER_FOR_DOC = "GPT2Tokenizer"36 37 38CODEGEN_PRETRAINED_MODEL_ARCHIVE_LIST = [39    "Salesforce/codegen-350M-nl",40    "Salesforce/codegen-350M-multi",41    "Salesforce/codegen-350M-mono",42    "Salesforce/codegen-2B-nl",43    "Salesforce/codegen-2B-multi",44    "Salesforce/codegen-2B-mono",45    "Salesforce/codegen-6B-nl",46    "Salesforce/codegen-6B-multi",47    "Salesforce/codegen-6B-mono",48    "Salesforce/codegen-16B-nl",49    "Salesforce/codegen-16B-multi",50    "Salesforce/codegen-16B-mono",51    # See all CodeGen models at https://huggingface.co/models?filter=codegen52]53 54 55# Copied from transformers.models.gptj.modeling_gptj.fixed_pos_embedding56def fixed_pos_embedding(x, seq_dim=1, seq_len=None):57    dim = x.shape[-1]58    if seq_len is None:59        seq_len = x.shape[seq_dim]60    inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2) / dim))61    sinusoid_inp = (62        torch.einsum("i , j -> i j", torch.arange(seq_len, dtype=torch.float), inv_freq).to(x.device).float()63    )64    return torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)65 66 67# Copied from transformers.models.gptj.modeling_gptj.rotate_every_two68def rotate_every_two(x):69    x1 = x[:, :, :, ::2]70    x2 = x[:, :, :, 1::2]71    x = torch.stack((-x2, x1), dim=-1)72    return x.flatten(-2)  # in einsum notation: rearrange(x, '... d j -> ... (d j)')73 74 75# Copied from transformers.models.gptj.modeling_gptj.duplicate_interleave76def duplicate_interleave(m):77    """78    A simple version of `torch.repeat_interleave` for duplicating a matrix while interleaving the copy.79    """80    dim0 = m.shape[0]81    m = m.view(-1, 1)  # flatten the matrix82    m = m.repeat(1, 2)  # repeat all elements into the 2nd dimension83    m = m.view(dim0, -1)  # reshape into a matrix, interleaving the copy84    return m85 86 87# Copied from transformers.models.gptj.modeling_gptj.apply_rotary_pos_emb88def apply_rotary_pos_emb(x, sincos, offset=0):89    sin, cos = map(lambda t: duplicate_interleave(t)[None, offset : x.shape[1] + offset, None, :], sincos)90    # einsum notation for lambda t: repeat(t[offset:x.shape[1]+offset,:], "n d -> () n () (d j)", j=2)91    return (x * cos) + (rotate_every_two(x) * sin)92 93 94class CodeGenAttention(nn.Module):95    def __init__(self, config):96        super().__init__()97 98        max_positions = config.max_position_embeddings99        self.register_buffer(100            "causal_mask",101            torch.tril(torch.ones((max_positions, max_positions), dtype=torch.bool)).view(102                1, 1, max_positions, max_positions103            ),104        )105 106        self.attn_dropout = nn.Dropout(config.attn_pdrop)107        self.resid_dropout = nn.Dropout(config.resid_pdrop)108 109        self.embed_dim = config.hidden_size110        self.num_attention_heads = config.num_attention_heads111        self.head_dim = self.embed_dim // self.num_attention_heads112        if self.head_dim * self.num_attention_heads != self.embed_dim:113            raise ValueError(114                f"embed_dim must be divisible by num_attention_heads (got `embed_dim`: {self.embed_dim} and"115                f" `num_attention_heads`: {self.num_attention_heads})."116            )117        self.scale_attn = torch.sqrt(torch.tensor(self.head_dim, dtype=torch.float32)).to(torch.get_default_dtype())118        self.qkv_proj = nn.Linear(self.embed_dim, self.embed_dim * 3, bias=False)119 120        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)121        self.rotary_dim = None122        if config.rotary_dim is not None:123            self.rotary_dim = config.rotary_dim124 125    def _split_heads(self, x, n_head, dim_head, mp_num):126        reshaped = x.reshape(x.shape[:-1] + (n_head // mp_num, dim_head))127        reshaped = reshaped.reshape(x.shape[:-2] + (-1,) + reshaped.shape[-1:])128        return reshaped129 130    def _merge_heads(self, tensor, num_attention_heads, attn_head_size):131        """132        Merges attn_head_size dim and num_attn_heads dim into n_ctx133        """134        if len(tensor.shape) == 5:135            tensor = tensor.permute(0, 1, 3, 2, 4).contiguous()136        elif len(tensor.shape) == 4:137            tensor = tensor.permute(0, 2, 1, 3).contiguous()138        else:139            raise ValueError(f"Input tensor rank should be one of [4, 5], but is: {len(tensor.shape)}")140        new_shape = tensor.size()[:-2] + (num_attention_heads * attn_head_size,)141        return tensor.view(new_shape)142 143    def _attn(144        self,145        query,146        key,147        value,148        attention_mask=None,149        head_mask=None,150    ):151 152        # compute causal mask from causal mask buffer153        query_length, key_length = query.size(-2), key.size(-2)154        causal_mask = self.causal_mask[:, :, key_length - query_length : key_length, :key_length]155 156        # Keep the attention weights computation in fp32 to avoid overflow issues157        query = query.to(torch.float32)158        key = key.to(torch.float32)159 160        attn_weights = torch.matmul(query, key.transpose(-1, -2))161 162        attn_weights = attn_weights / self.scale_attn163        mask_value = torch.finfo(attn_weights.dtype).min164        # Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`.165        # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device`166        mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(attn_weights.device)167        attn_weights = torch.where(causal_mask, attn_weights, mask_value)168 169        if attention_mask is not None:170            # Apply the attention mask171            attn_weights = attn_weights + attention_mask172 173        attn_weights = nn.Softmax(dim=-1)(attn_weights)174        attn_weights = attn_weights.to(value.dtype)175        attn_weights = self.attn_dropout(attn_weights)176 177        # Mask heads if we want to178        if head_mask is not None:179            attn_weights = attn_weights * head_mask180 181        attn_output = torch.matmul(attn_weights, value)182 183        return attn_output, attn_weights184 185    def forward(186        self,187        hidden_states: Optional[torch.FloatTensor],188        attention_mask: Optional[torch.FloatTensor] = None,189        layer_past: Optional[Tuple[torch.Tensor]] = None,190        head_mask: Optional[torch.FloatTensor] = None,191        use_cache: Optional[bool] = False,192        output_attentions: Optional[bool] = False,193    ) -> Union[194        Tuple[torch.Tensor, Tuple[torch.Tensor]],195        Optional[Tuple[torch.Tensor, Tuple[torch.Tensor], Tuple[torch.Tensor, ...]]],196    ]:197 198        qkv = self.qkv_proj(hidden_states)199 200        # TPU-v3201        mp_num = 8202        qkv_split = qkv.reshape(qkv.shape[:-1] + (mp_num, -1))203 204        local_dim = self.head_dim * self.num_attention_heads // mp_num205        query, value, key = torch.split(qkv_split, local_dim, dim=-1)206        query = self._split_heads(query, self.num_attention_heads, self.head_dim, mp_num=mp_num)207        key = self._split_heads(key, self.num_attention_heads, self.head_dim, mp_num=mp_num)208 209        value = self._split_heads(value, self.num_attention_heads, self.head_dim, mp_num=mp_num)210        value = value.permute(0, 2, 1, 3)211 212        seq_len = key.shape[1]213        offset = 0214 215        if layer_past is not None:216            offset = layer_past[0].shape[-2]217            seq_len += offset218 219        if self.rotary_dim is not None:220            k_rot = key[:, :, :, : self.rotary_dim]221            k_pass = key[:, :, :, self.rotary_dim :]222 223            q_rot = query[:, :, :, : self.rotary_dim]224            q_pass = query[:, :, :, self.rotary_dim :]225 226            sincos = fixed_pos_embedding(k_rot, 1, seq_len=seq_len)227            k_rot = apply_rotary_pos_emb(k_rot, sincos, offset=offset)228            q_rot = apply_rotary_pos_emb(q_rot, sincos, offset=offset)229 230            key = torch.cat([k_rot, k_pass], dim=-1)231            query = torch.cat([q_rot, q_pass], dim=-1)232        else:233            sincos = fixed_pos_embedding(key, 1, seq_len=seq_len)234            key = apply_rotary_pos_emb(key, sincos, offset=offset)235            query = apply_rotary_pos_emb(query, sincos, offset=offset)236 237        key = key.permute(0, 2, 1, 3)238        query = query.permute(0, 2, 1, 3)239 240        if layer_past is not None:241            past_key = layer_past[0]242            past_value = layer_past[1]243            key = torch.cat((past_key, key), dim=-2)244            value = torch.cat((past_value, value), dim=-2)245 246        if use_cache is True:247            present = (key, value)248        else:249            present = None250 251        # compute self-attention: V x Softmax(QK^T)252        attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask)253 254        attn_output = self._merge_heads(attn_output, self.num_attention_heads, self.head_dim)255        attn_output = self.out_proj(attn_output)256        attn_output = self.resid_dropout(attn_output)257 258        outputs = (attn_output, present)259        if output_attentions:260            outputs += (attn_weights,)261 262        return outputs  # a, present, (attentions)263 264 265# Copied from transformers.models.gptj.modeling_gptj.GPTJMLP with GPTJ->CodeGen266class CodeGenMLP(nn.Module):267    def __init__(self, intermediate_size, config):  # in MLP: intermediate_size= 4 * embed_dim268        super().__init__()269        embed_dim = config.n_embd270 271        self.fc_in = nn.Linear(embed_dim, intermediate_size)272        self.fc_out = nn.Linear(intermediate_size, embed_dim)273 274        self.act = ACT2FN[config.activation_function]275        self.dropout = nn.Dropout(config.resid_pdrop)276 277    def forward(self, hidden_states: Optional[torch.FloatTensor]) -> torch.FloatTensor:278        hidden_states = self.fc_in(hidden_states)279        hidden_states = self.act(hidden_states)280        hidden_states = self.fc_out(hidden_states)281        hidden_states = self.dropout(hidden_states)282        return hidden_states283 284 285# Copied from transformers.models.gptj.modeling_gptj.GPTJBlock with GPTJ->CodeGen286class CodeGenBlock(nn.Module):287    def __init__(self, config):288        super().__init__()289        inner_dim = config.n_inner if config.n_inner is not None else 4 * config.n_embd290        self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)291        self.attn = CodeGenAttention(config)292        self.mlp = CodeGenMLP(inner_dim, config)293 294    def forward(295        self,296        hidden_states: Optional[torch.FloatTensor],297        layer_past: Optional[Tuple[torch.Tensor]] = None,298        attention_mask: Optional[torch.FloatTensor] = None,299        head_mask: Optional[torch.FloatTensor] = None,300        use_cache: Optional[bool] = False,301        output_attentions: Optional[bool] = False,302    ) -> Union[Tuple[torch.Tensor], Optional[Tuple[torch.Tensor, Tuple[torch.FloatTensor, ...]]]]:303        residual = hidden_states304        hidden_states = self.ln_1(hidden_states)305        attn_outputs = self.attn(306            hidden_states,307            layer_past=layer_past,308            attention_mask=attention_mask,309            head_mask=head_mask,310            use_cache=use_cache,311            output_attentions=output_attentions,312        )313        attn_output = attn_outputs[0]  # output_attn: a, present, (attentions)314        outputs = attn_outputs[1:]315 316        feed_forward_hidden_states = self.mlp(hidden_states)317        hidden_states = attn_output + feed_forward_hidden_states + residual318 319        if use_cache:320            outputs = (hidden_states,) + outputs321        else:322            outputs = (hidden_states,) + outputs[1:]323 324        return outputs  # hidden_states, present, (attentions)325 326 327class CodeGenPreTrainedModel(PreTrainedModel):328    """329    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained330    models.331    """332 333    config_class = CodeGenConfig334    base_model_prefix = "transformer"335    supports_gradient_checkpointing = True336    _no_split_modules = ["CodeGenBlock"]337 338    def __init__(self, *inputs, **kwargs):339        super().__init__(*inputs, **kwargs)340 341    def _init_weights(self, module):342        """Initialize the weights."""343        if isinstance(module, (nn.Linear,)):344            # Slightly different from Mesh Transformer JAX which uses truncated_normal for initialization345            # cf https://github.com/pytorch/pytorch/pull/5617346            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)347            if module.bias is not None:348                module.bias.data.zero_()349        elif isinstance(module, nn.Embedding):350            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)351            if module.padding_idx is not None:352                module.weight.data[module.padding_idx].zero_()353        elif isinstance(module, nn.LayerNorm):354            module.bias.data.zero_()355            module.weight.data.fill_(1.0)356 357    def _set_gradient_checkpointing(self, module, value=False):358        if isinstance(module, CodeGenModel):359            module.gradient_checkpointing = value360 361 362CODEGEN_START_DOCSTRING = r"""363    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use364    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and365    behavior.366 367    Parameters:368        config ([`CodeGenConfig`]): Model configuration class with all the parameters of the model.369            Initializing with a config file does not load the weights associated with the model, only the370            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.371"""372 373CODEGEN_INPUTS_DOCSTRING = r"""374    Args:375        input_ids (`torch.LongTensor` of shape `({0})`):376            Indices of input sequence tokens in the vocabulary.377 378            Indices can be obtained using [`GPT2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and379            [`PreTrainedTokenizer.__call__`] for details.380 381            [What are input IDs?](../glossary#input-ids)382        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):383            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:384 385            - 1 for tokens that are **not masked**,386            - 0 for tokens that are **masked**.387 388            [What are attention masks?](../glossary#attention-mask)389        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):390            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,391            1]`:392 393            - 0 corresponds to a *sentence A* token,394            - 1 corresponds to a *sentence B* token.395 396            [What are token type IDs?](../glossary#token-type-ids)397        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):398            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,399            config.n_positions - 1]`.400 401            [What are position IDs?](../glossary#position-ids)402        head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*):403            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:404 405            - 1 indicates the head is **not masked**,406            - 0 indicates the head is **masked**.407 408        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*):409            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This410            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the411            model's internal embedding lookup matrix.412        output_attentions (`bool`, *optional*):413            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned414            tensors for more detail.415        output_hidden_states (`bool`, *optional*):416            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for417            more detail.418        return_dict (`bool`, *optional*):419            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.420"""421 422 423@add_start_docstrings(424    "The bare CodeGen Model transformer outputting raw hidden-states without any specific head on top.",425    CODEGEN_START_DOCSTRING,426)427class CodeGenModel(CodeGenPreTrainedModel):428    def __init__(self, config):429        super().__init__(config)430 431        self.embed_dim = config.n_embd432        self.vocab_size = config.vocab_size433        self.wte = nn.Embedding(config.vocab_size, self.embed_dim)434        self.drop = nn.Dropout(config.embd_pdrop)435        self.h = nn.ModuleList([CodeGenBlock(config) for _ in range(config.n_layer)])436        self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)437        self.rotary_dim = min(config.rotary_dim, config.n_ctx // config.num_attention_heads)438 439        self.gradient_checkpointing = False440 441        # Initialize weights and apply final processing442        self.post_init()443 444    def get_input_embeddings(self):445        return self.wte446 447    def set_input_embeddings(self, new_embeddings):448        self.wte = new_embeddings449 450    @add_start_docstrings_to_model_forward(CODEGEN_INPUTS_DOCSTRING.format("batch_size, sequence_length"))451    @add_code_sample_docstrings(452        processor_class=_TOKENIZER_FOR_DOC,453        checkpoint=_CHECKPOINT_FOR_DOC,454        output_type=BaseModelOutputWithPast,455        config_class=_CONFIG_FOR_DOC,456    )457    def forward(458        self,459        input_ids: Optional[torch.LongTensor] = None,460        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,461        attention_mask: Optional[torch.FloatTensor] = None,462        token_type_ids: Optional[torch.LongTensor] = None,463        position_ids: Optional[torch.LongTensor] = None,464        head_mask: Optional[torch.FloatTensor] = None,465        inputs_embeds: Optional[torch.FloatTensor] = None,466        use_cache: Optional[bool] = None,467        output_attentions: Optional[bool] = None,468        output_hidden_states: Optional[bool] = None,469        return_dict: Optional[bool] = None,470    ) -> Union[Tuple, BaseModelOutputWithPast]:471        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions472        output_hidden_states = (473            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states474        )475        use_cache = use_cache if use_cache is not None else self.config.use_cache476        return_dict = return_dict if return_dict is not None else self.config.use_return_dict477 478        if input_ids is not None and inputs_embeds is not None:479            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")480        elif input_ids is not None:481            input_shape = input_ids.size()482            input_ids = input_ids.view(-1, input_shape[-1])483            batch_size = input_ids.shape[0]484        elif inputs_embeds is not None:485            input_shape = inputs_embeds.size()[:-1]486            batch_size = inputs_embeds.shape[0]487        else:488            raise ValueError("You have to specify either input_ids or inputs_embeds")489 490        device = input_ids.device if input_ids is not None else inputs_embeds.device491 492        if token_type_ids is not None:493            token_type_ids = token_type_ids.view(-1, input_shape[-1])494 495        if position_ids is not None:496            position_ids = position_ids.view(-1, input_shape[-1])497 498        if past_key_values is None:499            past_length = 0500            past_key_values = tuple([None] * len(self.h))501        else:502            past_length = past_key_values[0][0].size(-2)503 504        if position_ids is None:505            position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)506            position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])507 508        # Attention mask.509        if attention_mask is not None:510            if batch_size <= 0:511                raise ValueError("batch_size has to be defined and > 0")512            attention_mask = attention_mask.view(batch_size, -1)513            # We create a 3D attention mask from a 2D tensor mask.514            # Sizes are [batch_size, 1, 1, to_seq_length]515            # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]516            # this attention mask is more simple than the triangular masking of causal attention517            # used in OpenAI GPT, we just need to prepare the broadcast dimension here.518            attention_mask = attention_mask[:, None, None, :]519 520            # Since attention_mask is 1.0 for positions we want to attend and 0.0 for521            # masked positions, this operation will create a tensor which is 0.0 for522            # positions we want to attend and the dtype's smallest value for masked positions.523            # Since we are adding it to the raw scores before the softmax, this is524            # effectively the same as removing these entirely.525            attention_mask = attention_mask.to(dtype=self.dtype)  # fp16 compatibility526            attention_mask = (1.0 - attention_mask) * torch.finfo(self.dtype).min527 528        # Prepare head mask if needed529        # 1.0 in head_mask indicate we keep the head530        # attention_probs has shape bsz x num_attention_heads x N x N531        # head_mask has shape n_layer x batch x num_attention_heads x N x N532        head_mask = self.get_head_mask(head_mask, self.config.n_layer)533 534        if inputs_embeds is None:535            inputs_embeds = self.wte(input_ids)536 537        hidden_states = inputs_embeds538 539        if token_type_ids is not None:540            token_type_embeds = self.wte(token_type_ids)541            hidden_states = hidden_states + token_type_embeds542 543        hidden_states = self.drop(hidden_states)544 545        output_shape = input_shape + (hidden_states.size(-1),)546 547        presents = () if use_cache else None548        all_self_attentions = () if output_attentions else None549        all_hidden_states = () if output_hidden_states else None550        for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):551 552            if output_hidden_states:553                all_hidden_states = all_hidden_states + (hidden_states,)554 555            if self.gradient_checkpointing and self.training:556 557                if use_cache:558                    logger.warning(559                        "`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting "560                        "`use_cache=False`..."561                    )562                    use_cache = False563 564                def create_custom_forward(module):565                    def custom_forward(*inputs):566                        # None for past_key_value567                        return module(*inputs, use_cache, output_attentions)568 569                    return custom_forward570 571                outputs = torch.utils.checkpoint.checkpoint(572                    create_custom_forward(block),573                    hidden_states,574                    None,575                    attention_mask,576                    head_mask[i],577                )578            else:579                outputs = block(580                    hidden_states,581                    layer_past=layer_past,582                    attention_mask=attention_mask,583                    head_mask=head_mask[i],584                    use_cache=use_cache,585                    output_attentions=output_attentions,586                )587 588            hidden_states = outputs[0]589            if use_cache is True:590                presents = presents + (outputs[1],)591 592            if output_attentions:593                all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)594 595        hidden_states = self.ln_f(hidden_states)596 597        hidden_states = hidden_states.view(output_shape)598        # Add last hidden state599        if output_hidden_states:600            all_hidden_states = all_hidden_states + (hidden_states,)601 602        if not return_dict:603            return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)604 605        return BaseModelOutputWithPast(606            last_hidden_state=hidden_states,607            past_key_values=presents,608            hidden_states=all_hidden_states,609            attentions=all_self_attentions,610        )611 612 613@add_start_docstrings(614    """615    The CodeGen Model transformer with a language modeling head on top.616    """,617    CODEGEN_START_DOCSTRING,618)619class CodeGenForCausalLM(CodeGenPreTrainedModel):620    _keys_to_ignore_on_load_missing = [r"h\.\d+\.attn\.masked_bias", r"h\.\d+\.attn\.bias"]621 622    def __init__(self, config):623        super().__init__(config)624        self.transformer = CodeGenModel(config)625        self.lm_head = nn.Linear(config.n_embd, config.vocab_size)626 627        # Initialize weights and apply final processing628        self.post_init()629 630    def get_output_embeddings(self):631        return self.lm_head632 633    def set_output_embeddings(self, new_embeddings):634        self.lm_head = new_embeddings635 636    def prepare_inputs_for_generation(self, input_ids, past=None, **kwargs):637        token_type_ids = kwargs.get("token_type_ids", None)638        # only last token for inputs_ids if past is defined in kwargs639        if past:640            input_ids = input_ids[:, -1].unsqueeze(-1)641            if token_type_ids is not None:642                token_type_ids = token_type_ids[:, -1].unsqueeze(-1)643 644        attention_mask = kwargs.get("attention_mask", None)645        position_ids = kwargs.get("position_ids", None)646 647        if attention_mask is not None and position_ids is None:648            # create position_ids on the fly for batch generation649            position_ids = attention_mask.long().cumsum(-1) - 1650            position_ids.masked_fill_(attention_mask == 0, 1)651            if past:652                position_ids = position_ids[:, -1].unsqueeze(-1)653        else:654            position_ids = None655        return {656            "input_ids": input_ids,657            "past_key_values": past,658            "use_cache": kwargs.get("use_cache"),659            "position_ids": position_ids,660            "attention_mask": attention_mask,661            "token_type_ids": token_type_ids,662        }663 664    @add_start_docstrings_to_model_forward(CODEGEN_INPUTS_DOCSTRING.format("batch_size, sequence_length"))665    @add_code_sample_docstrings(666        processor_class=_TOKENIZER_FOR_DOC,667        checkpoint=_CHECKPOINT_FOR_DOC,668        output_type=CausalLMOutputWithPast,669        config_class=_CONFIG_FOR_DOC,670    )671    def forward(672        self,673        input_ids: Optional[torch.LongTensor] = None,674        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,675        attention_mask: Optional[torch.FloatTensor] = None,676        token_type_ids: Optional[torch.LongTensor] = None,677        position_ids: Optional[torch.LongTensor] = None,678        head_mask: Optional[torch.FloatTensor] = None,679        inputs_embeds: Optional[torch.FloatTensor] = None,680        labels: Optional[torch.LongTensor] = None,681        use_cache: Optional[bool] = None,682        output_attentions: Optional[bool] = None,683        output_hidden_states: Optional[bool] = None,684        return_dict: Optional[bool] = None,685    ) -> Union[Tuple, CausalLMOutputWithPast]:686        r"""687        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):688            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set689            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`690            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`691        """692        return_dict = return_dict if return_dict is not None else self.config.use_return_dict693 694        transformer_outputs = self.transformer(695            input_ids,696            past_key_values=past_key_values,697            attention_mask=attention_mask,698            token_type_ids=token_type_ids,699            position_ids=position_ids,700            head_mask=head_mask,701            inputs_embeds=inputs_embeds,702            use_cache=use_cache,703            output_attentions=output_attentions,704            output_hidden_states=output_hidden_states,705            return_dict=return_dict,706        )707        hidden_states = transformer_outputs[0]708 709        # make sure sampling in fp16 works correctly and710        # compute loss in fp32 to match with mesh-tf version711        # https://github.com/EleutherAI/gpt-neo/blob/89ce74164da2fb16179106f54e2269b5da8db333/models/gpt2/gpt2.py#L179712        lm_logits = self.lm_head(hidden_states).to(torch.float32)713 714        loss = None715        if labels is not None:716            # Shift so that tokens < n predict n717            shift_logits = lm_logits[..., :-1, :].contiguous()718            shift_labels = labels[..., 1:].contiguous()719            # Flatten the tokens720            loss_fct = CrossEntropyLoss()721            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))722 723            loss = loss.to(hidden_states.dtype)724 725        if not return_dict:726            output = (lm_logits,) + transformer_outputs[1:]727            return ((loss,) + output) if loss is not None else output728 729        return CausalLMOutputWithPast(730            loss=loss,731            logits=lm_logits,732            past_key_values=transformer_outputs.past_key_values,733            hidden_states=transformer_outputs.hidden_states,734            attentions=transformer_outputs.attentions,735        )736 737    @staticmethod738    def _reorder_cache(past: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor) -> Tuple[Tuple[torch.Tensor]]:739        """740        This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or741        [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct742        beam_idx at every generation step.743        """744        return tuple(745            tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)746            for layer_past in past747        )748