1bitLLM/bitnet_b1_58-large

1# coding=utf-82# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.3#4# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX5# and OPT implementations in this library. It has been modified from its6# original forms to accommodate minor architectural differences compared7# to GPT-NeoX and OPT used by the Meta AI team that trained the model.8#9# Licensed under the Apache License, Version 2.0 (the "License");10# you may not use this file except in compliance with the License.11# You may obtain a copy of the License at12#13#     http://www.apache.org/licenses/LICENSE-2.014#15# Unless required by applicable law or agreed to in writing, software16# distributed under the License is distributed on an "AS IS" BASIS,17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.18# See the License for the specific language governing permissions and19# limitations under the License.20 21"""Tokenization classes for LLaMA."""22import os23from shutil import copyfile24from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple25 26import sentencepiece as spm27 28from transformers.convert_slow_tokenizer import import_protobuf29from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer30from transformers.utils import logging31 32 33if TYPE_CHECKING:34    from transformers.tokenization_utils_base import TextInput35 36logger = logging.get_logger(__name__)37 38VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}39 40PRETRAINED_VOCAB_FILES_MAP = {41    "vocab_file": {42        "hf-internal-testing/llama-tokenizer": "https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer.model",43    },44    "tokenizer_file": {45        "hf-internal-testing/llama-tokenizer": "https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer_config.json",46    },47}48PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {49    "hf-internal-testing/llama-tokenizer": 2048,50}51SPIECE_UNDERLINE = "▁"52 53B_INST, E_INST = "[INST]", "[/INST]"54B_SYS, E_SYS = "<<SYS>>\n", "\n<</SYS>>\n\n"55 56# fmt: off57DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your \58answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\59 that your responses are socially unbiased and positive in nature.60 61If a question does not make any sense, or is not factually coherent, explain why instead of answering something not \62correct. If you don't know the answer to a question, please don't share false information."""63# fmt: on64 65 66class BitnetTokenizer(PreTrainedTokenizer):67    """68    Construct a Bitnet tokenizer. Based on byte-level Byte-Pair-Encoding. The default padding token is unset as there is69    no padding token in the original model.70 71    Args:72        vocab_file (`str`):73            Path to the vocabulary file.74        unk_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<unk>"`):75            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this76            token instead.77        bos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<s>"`):78            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.79        eos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"</s>"`):80            The end of sequence token.81        pad_token (`str` or `tokenizers.AddedToken`, *optional*):82            A special token used to make arrays of tokens the same size for batching purpose. Will then be ignored by83            attention mechanisms or loss computation.84        sp_model_kwargs (`Dict[str, Any]`, `Optional`, *optional*):85            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for86            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,87            to set:88 89            - `enable_sampling`: Enable subword regularization.90            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.91 92              - `nbest_size = {0,1}`: No sampling is performed.93              - `nbest_size > 1`: samples from the nbest_size results.94              - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)95                using forward-filtering-and-backward-sampling algorithm.96 97            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for98              BPE-dropout.99 100        add_bos_token (`bool`, *optional*, defaults to `True`):101            Whether or not to add an `bos_token` at the start of sequences.102        add_eos_token (`bool`, *optional*, defaults to `False`):103            Whether or not to add an `eos_token` at the end of sequences.104        clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):105            Whether or not to cleanup spaces after decoding, cleanup consists in removing potential artifacts like106            extra spaces.107        use_default_system_prompt (`bool`, *optional*, defaults to `False`):108            Whether or not the default system prompt for Bitnet should be used.109        spaces_between_special_tokens (`bool`, *optional*, defaults to `False`):110            Whether or not to add spaces between special tokens.111        legacy (`bool`, *optional*):112            Whether or not the `legacy` behavior of the tokenizer should be used. Legacy is before the merge of #24622113            and #25224 which includes fixes to properly handle tokens that appear after special tokens. A simple114            example:115 116            - `legacy=True`:117            ```python118            >>> from transformers import T5Tokenizer119 120            >>> tokenizer = T5Tokenizer.from_pretrained("google-t5/t5-base", legacy=True)121            >>> tokenizer.encode("Hello <extra_id_0>.")122            [8774, 32099, 3, 5, 1]123            ```124            - `legacy=False`:125            ```python126            >>> from transformers import T5Tokenizer127 128            >>> tokenizer = T5Tokenizer.from_pretrained("google-t5/t5-base", legacy=False)129            >>> tokenizer.encode("Hello <extra_id_0>.")  # the extra space `[3]` is no longer here130            [8774, 32099, 5, 1]131            ```132            Checkout the [pull request](https://github.com/huggingface/transformers/pull/24565) for more details.133        add_prefix_space (`bool`, *optional*, defaults to `True`):134            Whether or not to add an initial space to the input. This allows to treat the leading word just as any135            other word.136 137    """138 139    vocab_files_names = VOCAB_FILES_NAMES140    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP141    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES142    model_input_names = ["input_ids", "attention_mask"]143 144    def __init__(145        self,146        vocab_file,147        unk_token="<unk>",148        bos_token="<s>",149        eos_token="</s>",150        pad_token=None,151        sp_model_kwargs: Optional[Dict[str, Any]] = None,152        add_bos_token=True,153        add_eos_token=False,154        clean_up_tokenization_spaces=False,155        use_default_system_prompt=False,156        spaces_between_special_tokens=False,157        legacy=None,158        add_prefix_space=True,159        **kwargs,160    ):161        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs162        bos_token = AddedToken(bos_token, normalized=False, special=True) if isinstance(bos_token, str) else bos_token163        eos_token = AddedToken(eos_token, normalized=False, special=True) if isinstance(eos_token, str) else eos_token164        unk_token = AddedToken(unk_token, normalized=False, special=True) if isinstance(unk_token, str) else unk_token165        pad_token = AddedToken(pad_token, normalized=False, special=True) if isinstance(pad_token, str) else pad_token166 167        if legacy is None:168            logger.warning_once(169                f"You are using the default legacy behaviour of the {self.__class__}. This is"170                " expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you."171                " If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it"172                " means, and thoroughly read the reason why this was added as explained in"173                " https://github.com/huggingface/transformers/pull/24565"174            )175            legacy = True176 177        self.legacy = legacy178        self.vocab_file = vocab_file179        self.add_bos_token = add_bos_token180        self.add_eos_token = add_eos_token181        self.use_default_system_prompt = use_default_system_prompt182        self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))183        self.add_prefix_space = add_prefix_space184 185        super().__init__(186            bos_token=bos_token,187            eos_token=eos_token,188            unk_token=unk_token,189            pad_token=pad_token,190            add_bos_token=add_bos_token,191            add_eos_token=add_eos_token,192            sp_model_kwargs=self.sp_model_kwargs,193            clean_up_tokenization_spaces=clean_up_tokenization_spaces,194            use_default_system_prompt=use_default_system_prompt,195            spaces_between_special_tokens=spaces_between_special_tokens,196            legacy=legacy,197            add_prefix_space=add_prefix_space,198            **kwargs,199        )200 201    @property202    def unk_token_length(self):203        return len(self.sp_model.encode(str(self.unk_token)))204 205    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.get_spm_processor206    def get_spm_processor(self, from_slow=False):207        tokenizer = spm.SentencePieceProcessor(**self.sp_model_kwargs)208        if self.legacy or from_slow:  # no dependency on protobuf209            tokenizer.Load(self.vocab_file)210            return tokenizer211 212        with open(self.vocab_file, "rb") as f:213            sp_model = f.read()214            model_pb2 = import_protobuf(f"The new behaviour of {self.__class__.__name__} (with `self.legacy = False`)")215            model = model_pb2.ModelProto.FromString(sp_model)216            normalizer_spec = model_pb2.NormalizerSpec()217            normalizer_spec.add_dummy_prefix = False218            model.normalizer_spec.MergeFrom(normalizer_spec)219            sp_model = model.SerializeToString()220            tokenizer.LoadFromSerializedProto(sp_model)221        return tokenizer222 223    def __getstate__(self):224        state = self.__dict__.copy()225        state["sp_model"] = None226        state["sp_model_proto"] = self.sp_model.serialized_model_proto()227        return state228 229    def __setstate__(self, d):230        self.__dict__ = d231        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)232        self.sp_model.LoadFromSerializedProto(self.sp_model_proto)233 234    @property235    def vocab_size(self):236        """Returns vocab size"""237        return self.sp_model.get_piece_size()238 239    def get_vocab(self):240        """Returns vocab as a dict"""241        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}242        vocab.update(self.added_tokens_encoder)243        return vocab244 245    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.tokenize246    def tokenize(self, text: "TextInput", **kwargs) -> List[str]:247        """248        Converts a string to a list of tokens. If `self.legacy` is set to `False`, a prefix token is added unless the249        first token is special.250        """251        if self.legacy or len(text) == 0:252            return super().tokenize(text, **kwargs)253 254        text = text.replace(SPIECE_UNDERLINE, " ")255        if self.add_prefix_space:256            text = SPIECE_UNDERLINE + text257 258        tokens = super().tokenize(text, **kwargs)259 260        if len(tokens) > 1 and tokens[0] == SPIECE_UNDERLINE and tokens[1] in self.all_special_tokens:261            tokens = tokens[1:]262        return tokens263 264    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer._tokenize265    def _tokenize(self, text, **kwargs):266        """267        Returns a tokenized string.268 269        We de-activated the `add_dummy_prefix` option, thus the sentencepiece internals will always strip any270        SPIECE_UNDERLINE. For example: `self.sp_model.encode(f"{SPIECE_UNDERLINE}Hey", out_type = str)` will give271        `['H', 'e', 'y']` instead of `['▁He', 'y']`. Thus we always encode `f"{unk_token}text"` and strip the272        `unk_token`. Here is an example with `unk_token = "<unk>"` and `unk_token_length = 4`.273        `self.tokenizer.sp_model.encode("<unk> Hey", out_type = str)[4:]`.274        """275        tokens = self.sp_model.encode(text, out_type=str)276        if self.legacy or not text.startswith((SPIECE_UNDERLINE, " ")):277            return tokens278 279        # 1. Encode string + prefix ex: "<unk> Hey"280        tokens = self.sp_model.encode(self.unk_token + text, out_type=str)281        # 2. Remove self.unk_token from ['<','unk','>', '▁Hey']282        return tokens[self.unk_token_length :] if len(tokens) >= self.unk_token_length else tokens283 284    def _convert_token_to_id(self, token):285        """Converts a token (str) in an id using the vocab."""286        return self.sp_model.piece_to_id(token)287 288    def _convert_id_to_token(self, index):289        """Converts an index (integer) in a token (str) using the vocab."""290        token = self.sp_model.IdToPiece(index)291        return token292 293    def convert_tokens_to_string(self, tokens):294        """Converts a sequence of tokens (string) in a single string."""295        # since we manually add the prefix space, we have to remove it when decoding296        if tokens[0].startswith(SPIECE_UNDERLINE) and self.add_prefix_space:297            tokens[0] = tokens[0][1:]298 299        current_sub_tokens = []300        out_string = ""301        prev_is_special = False302        for i, token in enumerate(tokens):303            # make sure that special tokens are not decoded using sentencepiece model304            if token in self.all_special_tokens:305                if not prev_is_special and i != 0 and self.legacy:306                    out_string += " "307                out_string += self.sp_model.decode(current_sub_tokens) + token308                prev_is_special = True309                current_sub_tokens = []310            else:311                current_sub_tokens.append(token)312                prev_is_special = False313        out_string += self.sp_model.decode(current_sub_tokens)314        return out_string315 316    def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:317        """318        Save the vocabulary and special tokens file to a directory.319 320        Args:321            save_directory (`str`):322                The directory in which to save the vocabulary.323 324        Returns:325            `Tuple(str)`: Paths to the files saved.326        """327        if not os.path.isdir(save_directory):328            logger.error(f"Vocabulary path ({save_directory}) should be a directory")329            return330        out_vocab_file = os.path.join(331            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]332        )333 334        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):335            copyfile(self.vocab_file, out_vocab_file)336        elif not os.path.isfile(self.vocab_file):337            with open(out_vocab_file, "wb") as fi:338                content_spiece_model = self.sp_model.serialized_model_proto()339                fi.write(content_spiece_model)340 341        return (out_vocab_file,)342 343    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):344        bos_token_id = [self.bos_token_id] if self.add_bos_token else []345        eos_token_id = [self.eos_token_id] if self.add_eos_token else []346 347        output = bos_token_id + token_ids_0 + eos_token_id348 349        if token_ids_1 is not None:350            output = output + bos_token_id + token_ids_1 + eos_token_id351 352        return output353 354    def get_special_tokens_mask(355        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False356    ) -> List[int]:357        """358        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding359        special tokens using the tokenizer `prepare_for_model` method.360 361        Args:362            token_ids_0 (`List[int]`):363                List of IDs.364            token_ids_1 (`List[int]`, *optional*):365                Optional second list of IDs for sequence pairs.366            already_has_special_tokens (`bool`, *optional*, defaults to `False`):367                Whether or not the token list is already formatted with special tokens for the model.368 369        Returns:370            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.371        """372        if already_has_special_tokens:373            return super().get_special_tokens_mask(374                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True375            )376 377        bos_token_id = [1] if self.add_bos_token else []378        eos_token_id = [1] if self.add_eos_token else []379 380        if token_ids_1 is None:381            return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id382        return (383            bos_token_id384            + ([0] * len(token_ids_0))385            + eos_token_id386            + bos_token_id387            + ([0] * len(token_ids_1))388            + eos_token_id389        )390 391    def create_token_type_ids_from_sequences(392        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None393    ) -> List[int]:394        """395        Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT396        sequence pair mask has the following format:397 398        ```399        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1400        | first sequence    | second sequence |401        ```402 403        if token_ids_1 is None, only returns the first portion of the mask (0s).404 405        Args:406            token_ids_0 (`List[int]`):407                List of ids.408            token_ids_1 (`List[int]`, *optional*):409                Optional second list of IDs for sequence pairs.410 411        Returns:412            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).413        """414        bos_token_id = [self.bos_token_id] if self.add_bos_token else []415        eos_token_id = [self.eos_token_id] if self.add_eos_token else []416 417        output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)418 419        if token_ids_1 is not None:420            output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)421 422        return output423 424    @property425    def default_chat_template(self):426        """427        LLaMA uses [INST] and [/INST] to indicate user messages, and <<SYS>> and <</SYS>> to indicate system messages.428        Assistant messages do not have special tokens, because LLaMA chat models are generally trained with strict429        user/assistant/user/assistant message ordering, and so assistant messages can be identified from the ordering430        rather than needing special tokens. The system message is partly 'embedded' in the first user message, which431        results in an unusual token ordering when it is present. This template should definitely be changed if you wish432        to fine-tune a model with more flexible role ordering!433 434        The output should look something like:435 436        <bos>[INST] B_SYS SystemPrompt E_SYS Prompt [/INST] Answer <eos><bos>[INST] Prompt [/INST] Answer <eos>437        <bos>[INST] Prompt [/INST]438 439        The reference for this chat template is [this code440        snippet](https://github.com/facebookresearch/llama/blob/556949fdfb72da27c2f4a40b7f0e4cf0b8153a28/llama/generation.py#L320-L362)441        in the original repository.442        """443        logger.warning_once(444            "\nNo chat template is defined for this tokenizer - using the default template "445            f"for the {self.__class__.__name__} class. If the default is not appropriate for "446            "your model, please set `tokenizer.chat_template` to an appropriate template. "447            "See https://huggingface.co/docs/transformers/main/chat_templating for more information.\n"448        )449        template = (450            "{% if messages[0]['role'] == 'system' %}"451            "{% set loop_messages = messages[1:] %}"  # Extract system message if it's present452            "{% set system_message = messages[0]['content'] %}"453            "{% elif USE_DEFAULT_PROMPT == true and not '<<SYS>>' in messages[0]['content'] %}"454            "{% set loop_messages = messages %}"  # Or use the default system message if the flag is set455            "{% set system_message = 'DEFAULT_SYSTEM_MESSAGE' %}"456            "{% else %}"457            "{% set loop_messages = messages %}"458            "{% set system_message = false %}"459            "{% endif %}"460            "{% for message in loop_messages %}"  # Loop over all non-system messages461            "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}"462            "{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}"463            "{% endif %}"464            "{% if loop.index0 == 0 and system_message != false %}"  # Embed system message in first message465            "{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}"466            "{% else %}"467            "{% set content = message['content'] %}"468            "{% endif %}"469            "{% if message['role'] == 'user' %}"  # After all of that, handle messages/roles in a fairly normal way470            "{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}"471            "{% elif message['role'] == 'system' %}"472            "{{ '<<SYS>>\\n' + content.strip() + '\\n<</SYS>>\\n\\n' }}"473            "{% elif message['role'] == 'assistant' %}"474            "{{ ' '  + content.strip() + ' ' + eos_token }}"475            "{% endif %}"476            "{% endfor %}"477        )478        template = template.replace("USE_DEFAULT_PROMPT", "true" if self.use_default_system_prompt else "false")479        default_message = DEFAULT_SYSTEM_PROMPT.replace("\n", "\\n").replace("'", "\\'")480        template = template.replace("DEFAULT_SYSTEM_MESSAGE", default_message)481 482        return template