Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2023 MetaAI and the HuggingFace Inc. team. All rights reserved.3#4#5# Licensed under the Apache License, Version 2.0 (the "License");6# you may not use this file except in compliance with the License.7# You may obtain a copy of the License at8#9#     http://www.apache.org/licenses/LICENSE-2.010#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.14# See the License for the specific language governing permissions and15# limitations under the License.16 17"""Tokenization classes for Code LLaMA."""18 19import os20from shutil import copyfile21from typing import Any, Optional22 23import sentencepiece as spm24 25from ...convert_slow_tokenizer import import_protobuf26from ...tokenization_utils import AddedToken, PreTrainedTokenizer27from ...utils import logging, requires_backends28from ...utils.import_utils import requires29 30 31logger = logging.get_logger(__name__)32 33VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}34 35SPIECE_UNDERLINE = "▁"36 37B_INST, E_INST = "[INST]", "[/INST]"38B_SYS, E_SYS = "<<SYS>>\n", "\n<</SYS>>\n\n"39 40# fmt: off41DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your \42answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\43 that your responses are socially unbiased and positive in nature.44 45If a question does not make any sense, or is not factually coherent, explain why instead of answering something not \46correct. If you don't know the answer to a question, please don't share false information."""47# fmt: on48 49 50@requires(backends=("sentencepiece",))51class CodeLlamaTokenizer(PreTrainedTokenizer):52    """53    Construct a CodeLlama tokenizer. Based on byte-level Byte-Pair-Encoding. The default padding token is unset as54    there is no padding token in the original model.55 56    The default configuration match that of57    [codellama/CodeLlama-7b-Instruct-hf](https://huggingface.co/meta-llama/CodeLlama-7b-Instruct-hf/blob/main/tokenizer_config.json)58    which supports prompt infilling.59 60    Args:61        vocab_file (`str`):62            Path to the vocabulary file.63        unk_token (`str`, *optional*, defaults to `"<unk>"`):64            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this65            token instead.66        bos_token (`str`, *optional*, defaults to `"<s>"`):67            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.68        eos_token (`str`, *optional*, defaults to `"</s>"`):69            The end of sequence token.70 71            <Tip>72 73            When building a sequence using special tokens, this is not the token that is used for the end of sequence.74            The token used is the `sep_token`.75 76            </Tip>77 78        prefix_token (`str`, *optional*, defaults to `"▁<PRE>"`):79            Prefix token used for infilling.80        middle_token (`str`, *optional*, defaults to `"▁<MID>"`):81            Middle token used for infilling.82        suffix_token (`str`, *optional*, defaults to `"▁<SUF>"`):83            Suffix token used for infilling.84        eot_token (`str`, *optional*, defaults to `"▁<EOT>"`):85            End of text token used for infilling.86        fill_token (`str`, *optional*, defaults to `"<FILL_ME>"`):87            The token used to split the input between the prefix and suffix.88        suffix_first (`bool`, *optional*, defaults to `False`):89            Whether the input prompt and suffix should be formatted with the suffix first.90        sp_model_kwargs (`dict`, *optional*):91            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for92            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,93            to set:94 95            - `enable_sampling`: Enable subword regularization.96            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.97 98              - `nbest_size = {0,1}`: No sampling is performed.99              - `nbest_size > 1`: samples from the nbest_size results.100              - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)101                using forward-filtering-and-backward-sampling algorithm.102 103            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for104              BPE-dropout.105        add_bos_token (`bool`, *optional*, defaults to `True`):106            Whether to add a beginning of sequence token at the start of sequences.107        add_eos_token (`bool`, *optional*, defaults to `False`):108            Whether to add an end of sequence token at the end of sequences.109        clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):110            Whether or not to clean up the tokenization spaces.111        additional_special_tokens (`list[str]`, *optional*):112            Additional special tokens used by the tokenizer.113        use_default_system_prompt (`bool`, *optional*, defaults to `False`):114            Whether or not the default system prompt for Llama should be used.115    """116 117    vocab_files_names = VOCAB_FILES_NAMES118    model_input_names = ["input_ids", "attention_mask"]119 120    def __init__(121        self,122        vocab_file,123        unk_token="<unk>",124        bos_token="<s>",125        eos_token="</s>",126        prefix_token="▁<PRE>",127        middle_token="▁<MID>",128        suffix_token="▁<SUF>",129        eot_token="▁<EOT>",130        fill_token="<FILL_ME>",131        suffix_first=False,132        sp_model_kwargs: Optional[dict[str, Any]] = None,133        add_bos_token=True,134        add_eos_token=False,135        clean_up_tokenization_spaces=False,136        additional_special_tokens=None,137        use_default_system_prompt=False,138        **kwargs,139    ):140        requires_backends(self, "protobuf")141        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs142        bos_token = AddedToken(bos_token, normalized=False, special=True) if isinstance(bos_token, str) else bos_token143        eos_token = AddedToken(eos_token, normalized=False, special=True) if isinstance(eos_token, str) else eos_token144        unk_token = AddedToken(unk_token, normalized=False, special=True) if isinstance(unk_token, str) else unk_token145 146        self.use_default_system_prompt = use_default_system_prompt147        # mark tokens special to skip them148        additional_special_tokens = additional_special_tokens or []149        for token in [prefix_token, middle_token, suffix_token, eot_token]:150            additional_special_tokens += [token] if token is not None else []151 152        self.vocab_file = vocab_file153        self.add_bos_token = add_bos_token154        self.add_eos_token = add_eos_token155        self._prefix_token = prefix_token156        self._middle_token = middle_token157        self._suffix_token = suffix_token158        self._eot_token = eot_token159        self.fill_token = fill_token160        self.suffix_first = suffix_first161        self.sp_model = self.get_spm_processor()162 163        super().__init__(164            bos_token=bos_token,165            eos_token=eos_token,166            unk_token=unk_token,167            add_bos_token=add_bos_token,168            add_eos_token=add_eos_token,169            prefix_token=prefix_token,170            middle_token=middle_token,171            suffix_token=suffix_token,172            eot_token=eot_token,173            fill_token=fill_token,174            sp_model_kwargs=self.sp_model_kwargs,175            suffix_first=suffix_first,176            clean_up_tokenization_spaces=clean_up_tokenization_spaces,177            additional_special_tokens=additional_special_tokens,178            use_default_system_prompt=use_default_system_prompt,179            **kwargs,180        )181 182    @property183    def unk_token_length(self):184        return len(self.sp_model.encode(str(self.unk_token)))185 186    def get_spm_processor(self):187        tokenizer = spm.SentencePieceProcessor(**self.sp_model_kwargs)188        with open(self.vocab_file, "rb") as f:189            sp_model = f.read()190            model_pb2 = import_protobuf()191            model = model_pb2.ModelProto.FromString(sp_model)192            normalizer_spec = model_pb2.NormalizerSpec()193            normalizer_spec.add_dummy_prefix = False194            model.normalizer_spec.MergeFrom(normalizer_spec)195            sp_model = model.SerializeToString()196            tokenizer.LoadFromSerializedProto(sp_model)197        return tokenizer198 199    @property200    def prefix_token(self):201        return self._prefix_token202 203    @property204    def prefix_id(self):205        if self._prefix_token is None:206            return None207        return self.convert_tokens_to_ids(self.prefix_token)208 209    @property210    def middle_token(self):211        return self._middle_token212 213    @property214    def middle_id(self):215        if self._middle_token is None:216            return None217        return self.convert_tokens_to_ids(self.middle_token)218 219    @property220    def suffix_token(self):221        return self._suffix_token222 223    @property224    def suffix_id(self):225        if self._suffix_token is None:226            return None227        return self.convert_tokens_to_ids(self.suffix_token)228 229    @property230    def eot_token(self):231        return self._eot_token232 233    @property234    def eot_id(self):235        if self._eot_token is None:236            return None237        return self.convert_tokens_to_ids(self.eot_token)238 239    @property240    def vocab_size(self):241        """Returns vocab size"""242        return self.sp_model.get_piece_size()243 244    # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer.get_vocab245    def get_vocab(self):246        """Returns vocab as a dict"""247        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}248        vocab.update(self.added_tokens_encoder)249        return vocab250 251    def tokenize(self, prefix, suffix=None, suffix_first=False, **kwargs) -> list[int]:252        # add a prefix space to `prefix`253        if self.fill_token is not None and self.fill_token in prefix and suffix is None:254            prefix, suffix = prefix.split(self.fill_token)255 256        if len(prefix) > 0:257            prefix = SPIECE_UNDERLINE + prefix.replace(SPIECE_UNDERLINE, " ")258 259        if suffix is None or len(suffix) < 1:260            tokens = super().tokenize(prefix, **kwargs)261            if len(tokens) > 1 and tokens[0] == SPIECE_UNDERLINE and tokens[1] in self.all_special_tokens:262                tokens = tokens[1:]263            return tokens264 265        prefix_tokens = self._tokenize(prefix)  # prefix has an extra `SPIECE_UNDERLINE`266 267        if None in (self.prefix_id, self.middle_id, self.suffix_id):268            raise ValueError(269                "The input either includes a `prefix` and a `suffix` used for the infilling task,"270                f"  or can be split on the {self.fill_token} token, creating a suffix and prefix,"271                " but the model does not support `infilling`."272            )273        suffix_tokens = self._tokenize(suffix)  # make sure CodeLlama sp model does not mess up274 275        suffix_first = suffix_first if suffix_first is not None else self.suffix_first276        if suffix_first:277            # format as " <PRE> <SUF>{suf} <MID> {pre}"278            return [self.prefix_token, self.suffix_token] + suffix_tokens + [self.middle_token] + prefix_tokens279        else:280            # format as " <PRE> {pre} <SUF>{suf} <MID>"281            return [self.prefix_token] + prefix_tokens + [self.suffix_token] + suffix_tokens + [self.middle_token]282 283    def _tokenize(self, text, **kwargs):284        """285        Returns a tokenized string.286 287        We de-activated the `add_dummy_prefix` option, thus the sentencepiece internals will always strip any288        SPIECE_UNDERLINE. For example: `self.sp_model.encode(f"{SPIECE_UNDERLINE}Hey", out_type = str)` will give289        `['H', 'e', 'y']` instead of `['▁He', 'y']`. Thus we always encode `f"{unk_token}text"` and strip the290        `unk_token`. Here is an example with `unk_token = "<unk>"` and `unk_token_length = 4`.291        `self.tokenizer.sp_model.encode("<unk> Hey", out_type = str)[4:]`.292        """293        tokens = self.sp_model.encode(text, out_type=str)294        if not text.startswith((SPIECE_UNDERLINE, " ")):295            return tokens296        # 1. Encode string + prefix ex: "<unk> Hey"297        tokens = self.sp_model.encode(self.unk_token + text, out_type=str)298        # 2. Remove self.unk_token from ['<','unk','>', '▁Hey']299        return tokens[self.unk_token_length :] if len(tokens) >= self.unk_token_length else tokens300 301    # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer._convert_token_to_id302    def _convert_token_to_id(self, token):303        """Converts a token (str) in an id using the vocab."""304        return self.sp_model.piece_to_id(token)305 306    # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer._convert_id_to_token307    def _convert_id_to_token(self, index):308        """Converts an index (integer) in a token (str) using the vocab."""309        token = self.sp_model.IdToPiece(index)310        return token311 312    def convert_tokens_to_string(self, tokens):313        """Converts a sequence of tokens (string) in a single string."""314        # since we manually add the prefix space, we have to remove it when decoding315        if tokens[0].startswith(SPIECE_UNDERLINE):316            tokens[0] = tokens[0][1:]317 318        current_sub_tokens = []319        out_string = ""320        for _, token in enumerate(tokens):321            # make sure that special tokens are not decoded using sentencepiece model322            if token in self.all_special_tokens:323                out_string += self.sp_model.decode(current_sub_tokens) + token324                current_sub_tokens = []325            else:326                current_sub_tokens.append(token)327        out_string += self.sp_model.decode(current_sub_tokens)328        return out_string329 330    # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer.save_vocabulary331    def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> tuple[str]:332        """333        Save the vocabulary and special tokens file to a directory.334 335        Args:336            save_directory (`str`):337                The directory in which to save the vocabulary.338 339        Returns:340            `Tuple(str)`: Paths to the files saved.341        """342        if not os.path.isdir(save_directory):343            logger.error(f"Vocabulary path ({save_directory}) should be a directory")344            return345        out_vocab_file = os.path.join(346            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]347        )348 349        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):350            copyfile(self.vocab_file, out_vocab_file)351        elif not os.path.isfile(self.vocab_file):352            with open(out_vocab_file, "wb") as fi:353                content_spiece_model = self.sp_model.serialized_model_proto()354                fi.write(content_spiece_model)355 356        return (out_vocab_file,)357 358    # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer.build_inputs_with_special_tokens359    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):360        bos_token_id = [self.bos_token_id] if self.add_bos_token else []361        eos_token_id = [self.eos_token_id] if self.add_eos_token else []362 363        output = bos_token_id + token_ids_0 + eos_token_id364 365        if token_ids_1 is not None:366            output = output + bos_token_id + token_ids_1 + eos_token_id367 368        return output369 370    # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer.get_special_tokens_mask371    def get_special_tokens_mask(372        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None, already_has_special_tokens: bool = False373    ) -> list[int]:374        """375        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding376        special tokens using the tokenizer `prepare_for_model` method.377 378        Args:379            token_ids_0 (`list[int]`):380                List of IDs.381            token_ids_1 (`list[int]`, *optional*):382                Optional second list of IDs for sequence pairs.383            already_has_special_tokens (`bool`, *optional*, defaults to `False`):384                Whether or not the token list is already formatted with special tokens for the model.385 386        Returns:387            `list[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.388        """389        if already_has_special_tokens:390            return super().get_special_tokens_mask(391                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True392            )393 394        bos_token_id = [1] if self.add_bos_token else []395        eos_token_id = [1] if self.add_eos_token else []396 397        if token_ids_1 is None:398            return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id399        return (400            bos_token_id401            + ([0] * len(token_ids_0))402            + eos_token_id403            + bos_token_id404            + ([0] * len(token_ids_1))405            + eos_token_id406        )407 408    # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer.create_token_type_ids_from_sequences409    def create_token_type_ids_from_sequences(410        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None411    ) -> list[int]:412        """413        Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT414        sequence pair mask has the following format:415 416        ```417        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1418        | first sequence    | second sequence |419        ```420 421        if token_ids_1 is None, only returns the first portion of the mask (0s).422 423        Args:424            token_ids_0 (`list[int]`):425                List of ids.426            token_ids_1 (`list[int]`, *optional*):427                Optional second list of IDs for sequence pairs.428 429        Returns:430            `list[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).431        """432        bos_token_id = [self.bos_token_id] if self.add_bos_token else []433        eos_token_id = [self.eos_token_id] if self.add_eos_token else []434 435        output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)436 437        if token_ids_1 is not None:438            output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)439 440        return output441 442    def __getstate__(self):443        state = self.__dict__.copy()444        state["sp_model"] = None445        state["sp_model_proto"] = self.sp_model.serialized_model_proto()446        return state447 448    def __setstate__(self, d):449        self.__dict__ = d450        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)451        self.sp_model.LoadFromSerializedProto(self.sp_model_proto)452 453 454__all__ = ["CodeLlamaTokenizer"]455