sigmareaver/codegen25-7b-multi-4bit-128g-gptq

1# Copyright (c) 2023, salesforce.com, inc.2# All rights reserved.3# SPDX-License-Identifier: Apache-2.04# For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/Apache-2.05"""Tokenization classes for CodeGen2.5."""6 7from typing import List, Optional8 9from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer10from transformers.utils import logging11 12try:13    import tiktoken14except ModuleNotFoundError as e:15    raise ModuleNotFoundError("CodeGen2.5 requires the installation of tiktoken. Please install it via `pip install tiktoken`.") from e16 17 18logger = logging.get_logger(__name__)19 20MAX_MODEL_INPUT_SIZES = {21    "Salesforce/codegen25-7b-multi": 2048,22    "Salesforce/codegen25-7b-mono": 2048,23    "Salesforce/codegen25-7b-instruct": 2048,24}25 26 27def tiktoken_tokenizer(base="gpt2", pad_token=None, add_special=True):28    if not add_special:29        return tiktoken.get_encoding(base)30 31    def include_whitespace(n_min=2, n_max=20):32        whitespaces = [" " * n for n in reversed(range(n_min, n_max))]33        return whitespaces34 35    def include_tabs(n_min=2, n_max=20):36        tabs = ["\t" * n for n in reversed(range(n_min, n_max))]37        return tabs38 39    def include_fim_tokens():40        fim_tokens = [41            "<fim_prefix>",42            "<fim_middle>",43            "<fim_suffix>",44            "<fim_pad>",45            "<filename>",46            "<gh_stars>",47            "<issue_start>",48            "<issue_comment>",49            "<issue_closed>",50            "<jupyter_start>",51            "<jupyter_text>",52            "<jupyter_code>",53            "<jupyter_output>",54            "<empty_output>",55            "<commit_before>",56            "<commit_msg>",57            "<commit_after>",58            "<reponame>"59        ]60        return fim_tokens61 62    def include_codegen2_tokens():63        tokens = []64        tokens += [f"<dummy_{i}>" for i in range(4)]65        tokens.append("<sep>")  # 5031766        tokens.append("<eom>")  # 5031867        tokens += [f"<mask_{i}>" for i in reversed(range(1, 51199-50318+1))]68        return tokens69 70    add_whitespaces = include_whitespace(n_min=2, n_max=32)71    add_tabs = include_tabs(n_min=2, n_max=10)72    fim_tokens = include_fim_tokens()73    codegen2_tokens = include_codegen2_tokens()74 75    tokenizer = tiktoken.get_encoding(base)76 77    idx = tokenizer.n_vocab78 79    bpe_ranks = tokenizer._mergeable_ranks80 81    for wsp in add_whitespaces:82        bpe_ranks[bytes(wsp, 'ascii')] = idx83        idx += 184    for t in add_tabs:85        bpe_ranks[bytes(t, 'ascii')] = idx86        idx += 187 88    special_tokens = dict()89 90    for sp in fim_tokens:91        special_tokens[sp] = idx92        idx += 193    for sp in codegen2_tokens:94        special_tokens[sp] = idx95        idx += 196 97    if pad_token and pad_token not in tokenizer._special_tokens and pad_token not in special_tokens:98        special_tokens[pad_token] = idx99        idx += 1100    # In production, load the arguments directly instead of accessing private attributes101    # See openai_public.py for examples of arguments for specific encodings102    enc = tiktoken.Encoding(103        # If you're changing the set of special tokens, make sure to use a different name104        # It should be clear from the name what behaviour to expect.105        name=base.replace("base", "im"),106        pat_str=tokenizer._pat_str,107        mergeable_ranks=bpe_ranks,108        special_tokens={109            **tokenizer._special_tokens,110            **special_tokens111        }112    )113    return enc114 115 116class CodeGen25Tokenizer(PreTrainedTokenizer):117    """118    Construct a CodeGen2.5 tokenizer. Based on byte-level Byte-Pair-Encoding.119    Args:120        vocab_file (`str`):121            Path to the vocabulary file.122    """123    max_model_input_sizes = MAX_MODEL_INPUT_SIZES124    model_input_names = ["input_ids", "attention_mask"]125 126    def __init__(127            self,128            pad_token=None,129            eos_token="<|endoftext|>",130            add_eos_token=False,131            add_special_tokens=True,132            **kwargs,133    ):134        pad_token_added = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token135        eos_token_added = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token136        super().__init__(137            pad_token=pad_token_added,138            eos_token=eos_token_added,139            add_eos_token=add_eos_token,140            add_special_tokens=add_special_tokens,141            **kwargs,142        )143        self.add_eos_token = add_eos_token144        self.encoder = tiktoken_tokenizer(base="gpt2", pad_token=pad_token, add_special=add_special_tokens)145 146    @property147    def vocab_size(self):148        """Returns vocab size"""149        return self.encoder.n_vocab150 151    def get_vocab(self):152        """Returns vocab as a dict"""153        vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}154        return vocab155 156    def _tokenize(self, text, **kwargs):157        """Returns a tokenized string."""158        return self.encoder.encode(text, allowed_special="all")159 160    def _convert_token_to_id(self, token):161        """Converts a token (str) in an id using the vocab."""162        if isinstance(token, str):163            return self.encoder.encode_single_token(token)164        else:165            return token166 167    def _convert_id_to_token(self, index):168        """Converts an index (integer) in a token (str) using the vocab."""169        return self.encoder.decode_single_token_bytes(index).decode("utf-8")170 171    def _decode(self, token_ids: List[int], skip_special_tokens: bool = False, **kwargs):172        if skip_special_tokens:173            token_ids = [t for t in token_ids if t not in self.all_special_ids]174        return self.encoder.decode(token_ids)175 176    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:177        """Build model inputs from a sequence by appending eos_token_id."""178        eos_token_id = [self.eos_token_id] if self.add_eos_token else []179 180        output = token_ids_0 + eos_token_id181 182        if token_ids_1 is not None:183            output = output + token_ids_1 + eos_token_id184 185        return output186 187    def get_special_tokens_mask(188            self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None,189            already_has_special_tokens: bool = False190    ) -> List[int]:191        """192        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding193        special tokens using the tokenizer `prepare_for_model` method.194        Args:195            token_ids_0 (`List[int]`):196                List of IDs.197            token_ids_1 (`List[int]`, *optional*):198                Optional second list of IDs for sequence pairs.199            already_has_special_tokens (`bool`, *optional*, defaults to `False`):200                Whether the token list is already formatted with special tokens for the model.201        Returns:202            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.203        """204        if already_has_special_tokens:205            return super().get_special_tokens_mask(206                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True207            )208 209        eos_token_id = [1] if self.add_eos_token else []210 211        if token_ids_1 is None:212            return ([0] * len(token_ids_0)) + eos_token_id213        return ([0] * len(token_ids_0)) + eos_token_id + ([0] * len(token_ids_1)) + eos_token_id214 215    def create_token_type_ids_from_sequences(216            self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None217    ) -> List[int]:218        """219        Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT220        sequence pair mask has the following format:221        ```222        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1223        | first sequence    | second sequence |224        ```225        if token_ids_1 is None, only returns the first portion of the mask (0s).226        Args:227            token_ids_0 (`List[int]`):228                List of ids.229            token_ids_1 (`List[int]`, *optional*):230                Optional second list of IDs for sequence pairs.231        Returns:232            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).233        """234        eos_token_id = [self.eos_token_id] if self.add_eos_token else []235 236        output = [0] * len(token_ids_0 + eos_token_id)237 238        if token_ids_1 is not None:239            output += [1] * len(token_ids_1 + eos_token_id)240 241        return output242 243    # has no vocab file244    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):245        return ()246