parallelstudios/mpt-7b-instruct-parallel-colony-memory-importance-ft

1"""A simple, flexible implementation of a GPT model.2Inspired by https://github.com/karpathy/minGPT/blob/master/mingpt/model.py3"""4import math5import warnings6from typing import List, Optional, Tuple, Union7import torch8import torch.nn as nn9import torch.nn.functional as F10from transformers import PreTrainedModel, PreTrainedTokenizer, PreTrainedTokenizerFast11from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast12from .attention import attn_bias_shape, build_attn_bias13from .blocks import MPTBlock14from .custom_embedding import SharedEmbedding15from .norm import NORM_CLASS_REGISTRY16from .configuration_mpt import MPTConfig17from .adapt_tokenizer import AutoTokenizerForMOD, adapt_tokenizer_for_denoising18from .hf_prefixlm_converter import add_bidirectional_mask_if_missing, convert_hf_causal_lm_to_prefix_lm19from .meta_init_context import init_empty_weights20from .param_init_fns import MODEL_INIT_REGISTRY, generic_param_init_fn_21try:22    from .flash_attn_triton import flash_attn_func23except:24    pass25Tokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]26 27class MPTPreTrainedModel(PreTrainedModel):28    config_class = MPTConfig29    base_model_prefix = 'model'30    _no_split_modules = ['MPTBlock']31 32class MPTModel(MPTPreTrainedModel):33 34    def __init__(self, config: MPTConfig):35        config._validate_config()36        super().__init__(config)37        self.attn_impl = config.attn_config['attn_impl']38        self.prefix_lm = config.attn_config['prefix_lm']39        self.attn_uses_sequence_id = config.attn_config['attn_uses_sequence_id']40        self.alibi = config.attn_config['alibi']41        self.alibi_bias_max = config.attn_config['alibi_bias_max']42        if config.init_device == 'mixed':43            if dist.get_local_rank() == 0:44                config.init_device = 'cpu'45            else:46                config.init_device = 'meta'47        if config.norm_type.lower() not in NORM_CLASS_REGISTRY.keys():48            norm_options = ' | '.join(NORM_CLASS_REGISTRY.keys())49            raise NotImplementedError(f'Requested norm type ({config.norm_type}) is not implemented within this repo (Options: {norm_options}).')50        norm_class = NORM_CLASS_REGISTRY[config.norm_type.lower()]51        self.embedding_fraction = config.embedding_fraction52        self.wte = SharedEmbedding(config.vocab_size, config.d_model, device=config.init_device)53        if not self.alibi:54            self.wpe = torch.nn.Embedding(config.max_seq_len, config.d_model, device=config.init_device)55        self.emb_drop = nn.Dropout(config.emb_pdrop)56        self.blocks = nn.ModuleList([MPTBlock(device=config.init_device, **config.to_dict()) for _ in range(config.n_layers)])57        self.norm_f = norm_class(config.d_model, device=config.init_device)58        if config.init_device != 'meta':59            print(f'You are using config.init_device={config.init_device!r}, but you can also use config.init_device="meta" with Composer + FSDP for fast initialization.')60            self.apply(self.param_init_fn)61        self.is_causal = not self.prefix_lm62        self._attn_bias_initialized = False63        self.attn_bias = None64        self.attn_bias_shape = attn_bias_shape(self.attn_impl, config.n_heads, config.max_seq_len, self.alibi, prefix_lm=self.prefix_lm, causal=self.is_causal, use_sequence_id=self.attn_uses_sequence_id)65        if config.no_bias:66            for module in self.modules():67                if hasattr(module, 'bias') and isinstance(module.bias, nn.Parameter):68                    if config.verbose:69                        warnings.warn(f'Removing bias ({module.bias}) from {module}.')70                    module.register_parameter('bias', None)71        if config.verbose and config.verbose > 2:72            print(self)73        if 'verbose' not in self.config.init_config:74            self.config.init_config['verbose'] = self.config.verbose75        if self.config.init_config['verbose'] > 1:76            init_fn_name = self.config.init_config['name']77            warnings.warn(f'Using {init_fn_name} initialization.')78 79    def get_input_embeddings(self):80        return self.wte81 82    def set_input_embeddings(self, value):83        self.wte = value84 85    @torch.no_grad()86    def _attn_bias(self, device, dtype, attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, sequence_id: Optional[torch.LongTensor]=None):87        if not self._attn_bias_initialized:88            if self.attn_bias_shape:89                self.attn_bias = torch.zeros(self.attn_bias_shape, device=device, dtype=dtype)90                self.attn_bias = build_attn_bias(self.attn_impl, self.attn_bias, self.config.n_heads, self.config.max_seq_len, causal=self.is_causal, alibi=self.alibi, alibi_bias_max=self.alibi_bias_max)91            self._attn_bias_initialized = True92        if self.attn_impl == 'flash':93            return (self.attn_bias, attention_mask)94        if self.attn_bias is not None:95            self.attn_bias = self.attn_bias.to(dtype=dtype, device=device)96        attn_bias = self.attn_bias97        if self.prefix_lm:98            assert isinstance(attn_bias, torch.Tensor)99            assert isinstance(prefix_mask, torch.Tensor)100            attn_bias = self._apply_prefix_mask(attn_bias, prefix_mask)101        if self.attn_uses_sequence_id and sequence_id is not None:102            assert isinstance(attn_bias, torch.Tensor)103            attn_bias = self._apply_sequence_id(attn_bias, sequence_id)104        if attention_mask is not None:105            s_k = attention_mask.shape[-1]106            if attn_bias is None:107                attn_bias = torch.zeros((1, 1, 1, s_k), device=device, dtype=dtype)108            else:109                _s_k = max(0, attn_bias.size(-1) - s_k)110                attn_bias = attn_bias[:, :, :, _s_k:]111            if prefix_mask is not None and attention_mask.shape != prefix_mask.shape:112                raise ValueError(f'attention_mask shape={attention_mask.shape} ' + f'and prefix_mask shape={prefix_mask.shape} are not equal.')113            min_val = torch.finfo(attn_bias.dtype).min114            attn_bias = attn_bias.masked_fill(~attention_mask.view(-1, 1, 1, s_k), min_val)115        return (attn_bias, None)116 117    def _apply_prefix_mask(self, attn_bias: torch.Tensor, prefix_mask: torch.Tensor):118        (s_k, s_q) = attn_bias.shape[-2:]119        if s_k != self.config.max_seq_len or s_q != self.config.max_seq_len:120            raise ValueError('attn_bias does not match the expected shape. ' + f'The last two dimensions should both be {self.config.max_length} ' + f'but are {s_k} and {s_q}.')121        seq_len = prefix_mask.shape[-1]122        if seq_len > self.config.max_seq_len:123            raise ValueError(f'prefix_mask sequence length cannot exceed max_seq_len={self.config.max_seq_len}')124        attn_bias = attn_bias[..., :seq_len, :seq_len]125        causal = torch.tril(torch.ones((seq_len, seq_len), dtype=torch.bool, device=prefix_mask.device)).view(1, 1, seq_len, seq_len)126        prefix = prefix_mask.view(-1, 1, 1, seq_len)127        cannot_attend = ~torch.logical_or(causal, prefix.bool())128        min_val = torch.finfo(attn_bias.dtype).min129        attn_bias = attn_bias.masked_fill(cannot_attend, min_val)130        return attn_bias131 132    def _apply_sequence_id(self, attn_bias: torch.Tensor, sequence_id: torch.LongTensor):133        seq_len = sequence_id.shape[-1]134        if seq_len > self.config.max_seq_len:135            raise ValueError(f'sequence_id sequence length cannot exceed max_seq_len={self.config.max_seq_len}')136        attn_bias = attn_bias[..., :seq_len, :seq_len]137        cannot_attend = torch.logical_not(torch.eq(sequence_id.view(-1, seq_len, 1), sequence_id.view(-1, 1, seq_len))).unsqueeze(1)138        min_val = torch.finfo(attn_bias.dtype).min139        attn_bias = attn_bias.masked_fill(cannot_attend, min_val)140        return attn_bias141 142    def forward(self, input_ids: torch.LongTensor, past_key_values: Optional[List[Tuple[torch.FloatTensor]]]=None, attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, sequence_id: Optional[torch.LongTensor]=None, return_dict: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, use_cache: Optional[bool]=None, inputs_embeds: Optional[torch.Tensor]=None):143        return_dict = return_dict if return_dict is not None else self.config.return_dict144        use_cache = use_cache if use_cache is not None else self.config.use_cache145        if attention_mask is not None:146            attention_mask = attention_mask.bool()147        if prefix_mask is not None:148            prefix_mask = prefix_mask.bool()149        if not return_dict:150            raise NotImplementedError('return_dict False is not implemented yet for MPT')151        if output_attentions:152            if self.attn_impl != 'torch':153                raise NotImplementedError('output_attentions is not implemented for MPT when using attn_impl `flash` or `triton`.')154        if attention_mask is not None and attention_mask[:, 0].sum() != attention_mask.shape[0] and self.training:155            raise NotImplementedError('MPT does not support training with left padding.')156        if self.prefix_lm and prefix_mask is None:157            raise ValueError('prefix_mask is a required argument when MPT is configured with prefix_lm=True.')158        if inputs_embeds is not None:159            raise NotImplementedError('inputs_embeds is not implemented for MPT.')160        if self.training:161            if self.attn_uses_sequence_id and sequence_id is None:162                raise ValueError('sequence_id is a required argument when MPT is configured with attn_uses_sequence_id=True ' + 'and the model is in train mode.')163            elif self.attn_uses_sequence_id is False and sequence_id is not None:164                warnings.warn('MPT received non-None input for `sequence_id` but is configured with attn_uses_sequence_id=False. ' + 'This input will be ignored. If you want the model to use `sequence_id`, set attn_uses_sequence_id to True.')165        S = input_ids.size(1)166        assert S <= self.config.max_seq_len, f'Cannot forward input with seq_len={S}, this model only supports seq_len<={self.config.max_seq_len}'167        tok_emb = self.wte(input_ids)168        if self.alibi:169            x = tok_emb170        else:171            past_position = 0172            if past_key_values is not None:173                if len(past_key_values) != self.config.n_layers:174                    raise ValueError(f'past_key_values must provide a past_key_value for each attention ' + f'layer in the network (len(past_key_values)={len(past_key_values)!r}; self.config.n_layers={self.config.n_layers!r}).')175                past_position = past_key_values[0][0].size(1)176                if self.attn_impl == 'torch':177                    past_position = past_key_values[0][0].size(3)178            if S + past_position > self.config.max_seq_len:179                raise ValueError(f'Cannot forward input with past sequence length {past_position} and current sequence length {S + 1}, this model only supports total sequence length <= {self.config.max_seq_len}.')180            pos = torch.arange(past_position, S + past_position, dtype=torch.long, device=input_ids.device).unsqueeze(0)181            if attention_mask is not None:182                pos = torch.clamp(pos - torch.cumsum((~attention_mask).to(torch.int32), dim=1)[:, past_position:], min=0)183            pos_emb = self.wpe(pos)184            x = tok_emb + pos_emb185        if self.embedding_fraction == 1:186            x = self.emb_drop(x)187        else:188            x_shrunk = x * self.embedding_fraction + x.detach() * (1 - self.embedding_fraction)189            assert isinstance(self.emb_drop, nn.Module)190            x = self.emb_drop(x_shrunk)191        (attn_bias, attention_mask) = self._attn_bias(device=x.device, dtype=torch.float32, attention_mask=attention_mask, prefix_mask=prefix_mask, sequence_id=sequence_id)192        if use_cache and past_key_values is None:193            past_key_values = [() for _ in range(self.config.n_layers)]194        all_hidden_states = () if output_hidden_states else None195        all_self_attns = () if output_attentions else None196        for (b_idx, block) in enumerate(self.blocks):197            if output_hidden_states:198                assert all_hidden_states is not None199                all_hidden_states = all_hidden_states + (x,)200            past_key_value = past_key_values[b_idx] if past_key_values is not None else None201            (x, attn_weights, past_key_value) = block(x, past_key_value=past_key_value, attn_bias=attn_bias, attention_mask=attention_mask, is_causal=self.is_causal)202            if past_key_values is not None:203                past_key_values[b_idx] = past_key_value204            if output_attentions:205                assert all_self_attns is not None206                all_self_attns = all_self_attns + (attn_weights,)207        x = self.norm_f(x)208        if output_hidden_states:209            assert all_hidden_states is not None210            all_hidden_states = all_hidden_states + (x,)211        return BaseModelOutputWithPast(last_hidden_state=x, past_key_values=past_key_values, hidden_states=all_hidden_states, attentions=all_self_attns)212 213    def param_init_fn(self, module):214        init_fn_name = self.config.init_config['name']215        MODEL_INIT_REGISTRY[init_fn_name](module=module, n_layers=self.config.n_layers, d_model=self.config.d_model, **self.config.init_config)216 217    def fsdp_wrap_fn(self, module):218        return isinstance(module, MPTBlock)219 220    def activation_checkpointing_fn(self, module):221        return isinstance(module, MPTBlock)222 223class MPTForCausalLM(MPTPreTrainedModel):224 225    def __init__(self, config: MPTConfig):226        super().__init__(config)227        if not config.tie_word_embeddings:228            raise ValueError('MPTForCausalLM only supports tied word embeddings')229        print(f'Instantiating an MPTForCausalLM model from {__file__}')230        self.transformer = MPTModel(config)231        for child in self.transformer.children():232            if isinstance(child, torch.nn.ModuleList):233                continue234            if isinstance(child, torch.nn.Module):235                child._fsdp_wrap = True236        self.logit_scale = None237        if config.logit_scale is not None:238            logit_scale = config.logit_scale239            if isinstance(logit_scale, str):240                if logit_scale == 'inv_sqrt_d_model':241                    logit_scale = 1 / math.sqrt(config.d_model)242                else:243                    raise ValueError(f"logit_scale={logit_scale!r} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'.")244            self.logit_scale = logit_scale245 246    def get_input_embeddings(self):247        return self.transformer.wte248 249    def set_input_embeddings(self, value):250        self.transformer.wte = value251 252    def get_output_embeddings(self):253        return self.transformer.wte254 255    def set_output_embeddings(self, new_embeddings):256        self.transformer.wte = new_embeddings257 258    def set_decoder(self, decoder):259        self.transformer = decoder260 261    def get_decoder(self):262        return self.transformer263 264    def forward(self, input_ids: torch.LongTensor, past_key_values: Optional[List[Tuple[torch.FloatTensor]]]=None, attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, sequence_id: Optional[torch.LongTensor]=None, labels: Optional[torch.LongTensor]=None, return_dict: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, use_cache: Optional[bool]=None, inputs_embeds: Optional[torch.FloatTensor]=None):265        return_dict = return_dict if return_dict is not None else self.config.return_dict266        use_cache = use_cache if use_cache is not None else self.config.use_cache267        if inputs_embeds is not None:268            raise NotImplementedError('inputs_embeds has to be None (for hf/peft support).')269        outputs = self.transformer(input_ids=input_ids, past_key_values=past_key_values, attention_mask=attention_mask, prefix_mask=prefix_mask, sequence_id=sequence_id, return_dict=return_dict, output_attentions=output_attentions, output_hidden_states=output_hidden_states, use_cache=use_cache)270        logits = self.transformer.wte(outputs.last_hidden_state.to(self.transformer.wte.weight.device), True)271        if self.logit_scale is not None:272            if self.logit_scale == 0:273                warnings.warn(f'Multiplying logits by self.logit_scale={self.logit_scale!r}. This will produce uniform (uninformative) outputs.')274            logits *= self.logit_scale275        loss = None276        if labels is not None:277            labels = torch.roll(labels, shifts=-1)278            labels[:, -1] = -100279            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), labels.to(logits.device).view(-1))280        return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions)281 282    def param_init_fn(self, module):283        init_fn_name = self.config.init_config['name']284        MODEL_INIT_REGISTRY[init_fn_name](module=module, n_layers=self.config.n_layers, d_model=self.config.d_model, **self.config.init_config)285 286    def fsdp_wrap_fn(self, module):287        return isinstance(module, MPTBlock)288 289    def activation_checkpointing_fn(self, module):290        return isinstance(module, MPTBlock)291 292    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):293        if inputs_embeds is not None:294            raise NotImplementedError('inputs_embeds is not implemented for MPT yet')295        attention_mask = kwargs['attention_mask'].bool()296        if attention_mask[:, -1].sum() != attention_mask.shape[0]:297            raise NotImplementedError('MPT does not support generation with right padding.')298        if self.transformer.attn_uses_sequence_id and self.training:299            sequence_id = torch.zeros_like(input_ids[:1])300        else:301            sequence_id = None302        if past_key_values is not None:303            input_ids = input_ids[:, -1].unsqueeze(-1)304        if self.transformer.prefix_lm:305            prefix_mask = torch.ones_like(attention_mask)306            if kwargs.get('use_cache') == False:307                raise NotImplementedError('MPT with prefix_lm=True does not support use_cache=False.')308        else:309            prefix_mask = None310        return {'input_ids': input_ids, 'attention_mask': attention_mask, 'prefix_mask': prefix_mask, 'sequence_id': sequence_id, 'past_key_values': past_key_values, 'use_cache': kwargs.get('use_cache', True)}311 312    @staticmethod313    def _reorder_cache(past_key_values, beam_idx):314        """Used by HuggingFace generate when using beam search with kv-caching.315        See https://github.com/huggingface/transformers/blob/3ec7a47664ebe40c40f4b722f6bb1cd30c3821ec/src/transformers/models/gpt2/modeling_gpt2.py#L1122-L1133316        for an example in transformers.317        """318        reordered_past = []319        for layer_past in past_key_values:320            reordered_past += [tuple((past_state.index_select(0, beam_idx) for past_state in layer_past))]321        return reordered_past