Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2023 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15import os16from shutil import copyfile17from typing import Optional18 19from tokenizers import normalizers, processors20 21from ...tokenization_utils_fast import PreTrainedTokenizerFast22from ...utils import is_sentencepiece_available, logging23 24 25if is_sentencepiece_available():26    from .tokenization_code_llama import CodeLlamaTokenizer27else:28    CodeLlamaTokenizer = None29 30logger = logging.get_logger(__name__)31VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model", "tokenizer_file": "tokenizer.json"}32 33SPIECE_UNDERLINE = "▁"34 35 36B_INST, E_INST = "[INST]", "[/INST]"37B_SYS, E_SYS = "<<SYS>>\n", "\n<</SYS>>\n\n"38 39# fmt: off40DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your \41answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\42 that your responses are socially unbiased and positive in nature.43 44If a question does not make any sense, or is not factually coherent, explain why instead of answering something not \45correct. If you don't know the answer to a question, please don't share false information."""46# fmt: on47 48 49class CodeLlamaTokenizerFast(PreTrainedTokenizerFast):50    """51    Construct a Llama tokenizer. Based on byte-level Byte-Pair-Encoding.52 53    This uses notably ByteFallback and no normalization.54 55    ```python56    >>> from transformers import CodeLlamaTokenizerFast57 58    >>> tokenizer = CodeLlamaTokenizerFast.from_pretrained("hf-internal-testing/llama-tokenizer")59    >>> tokenizer.encode("Hello this is a test")60    [1, 15043, 445, 338, 263, 1243]61    ```62 63    If you want to change the `bos_token` or the `eos_token`, make sure to specify them when initializing the model, or64    call `tokenizer.update_post_processor()` to make sure that the post-processing is correctly done (otherwise the65    values of the first token and final token of an encoded sequence will not be correct). For more details, checkout66    [post-processors] (https://huggingface.co/docs/tokenizers/api/post-processors) documentation.67 68 69    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should70    refer to this superclass for more information regarding those methods. The default configuration match that of71    [meta-llama/CodeLlama-7b-Instruct-hf](https://huggingface.co/meta-llama/CodeLlama-7b-Instruct-hf/blob/main/tokenizer_config.json)72    which supports prompt infilling.73 74    Args:75        vocab_file (`str`, *optional*):76            [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .model extension) that77            contains the vocabulary necessary to instantiate a tokenizer.78        tokenizer_file (`str`, *optional*):79            [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that80            contains everything needed to load the tokenizer.81        clean_up_tokenization_spaces (`str`, *optional*, defaults to `False`):82            Whether to cleanup spaces after decoding, cleanup consists in removing potential artifacts like extra83            spaces.84        unk_token (`str`, *optional*, defaults to `"<unk>"`):85            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this86            token instead.87        bos_token (`str`, *optional*, defaults to `"<s>"`):88            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.89        eos_token (`str`, *optional*, defaults to `"</s>"`):90            The end of sequence token.91        prefix_token (`str`, *optional*, defaults to `"▁<PRE>"`):92            Prefix token used for infilling.93        middle_token (`str`, *optional*, defaults to `"▁<MID>"`):94            Middle token used for infilling.95        suffix_token (`str`, *optional*, defaults to `"▁<SUF>"`):96            Suffix token used for infilling.97        eot_token (`str`, *optional*, defaults to `"▁<EOT>"`):98            End of text token used for infilling.99        fill_token (`str`, *optional*, defaults to `"<FILL_ME>"`):100            The token used to split the input between the prefix and suffix.101        additional_special_tokens (`list[str]`, *optional*):102            Additional special tokens used by the tokenizer.103        add_bos_token (`bool`, *optional*, defaults to `True`):104            Whether to add a beginning of sequence token at the start of sequences.105        add_eos_token (`bool`, *optional*, defaults to `False`):106            Whether to add an end of sequence token at the end of sequences.107        use_default_system_prompt (`bool`, *optional*, defaults to `False`):108            Whether or not the default system prompt for Llama should be used.109    """110 111    vocab_files_names = VOCAB_FILES_NAMES112    slow_tokenizer_class = CodeLlamaTokenizer113    padding_side = "left"114    model_input_names = ["input_ids", "attention_mask"]115 116    def __init__(117        self,118        vocab_file=None,119        tokenizer_file=None,120        clean_up_tokenization_spaces=False,121        unk_token="<unk>",122        bos_token="<s>",123        eos_token="</s>",124        prefix_token="▁<PRE>",125        middle_token="▁<MID>",126        suffix_token="▁<SUF>",127        eot_token="▁<EOT>",128        fill_token="<FILL_ME>",129        additional_special_tokens=None,130        add_bos_token=True,131        add_eos_token=False,132        use_default_system_prompt=False,133        **kwargs,134    ):135        # mark tokens special to skip them136        additional_special_tokens = additional_special_tokens or []137        for token in [prefix_token, middle_token, suffix_token, eot_token]:138            additional_special_tokens += [token] if token is not None else []139        self.use_default_system_prompt = use_default_system_prompt140 141        super().__init__(142            vocab_file=vocab_file,143            tokenizer_file=tokenizer_file,144            clean_up_tokenization_spaces=clean_up_tokenization_spaces,145            additional_special_tokens=additional_special_tokens,146            unk_token=unk_token,147            bos_token=bos_token,148            eos_token=eos_token,149            add_bos_token=add_bos_token,150            add_eos_token=add_eos_token,151            prefix_token=prefix_token,152            middle_token=middle_token,153            suffix_token=suffix_token,154            eot_token=eot_token,155            fill_token=fill_token,156            use_default_system_prompt=use_default_system_prompt,157            **kwargs,158        )159        self._add_bos_token = add_bos_token160        self._add_eos_token = add_eos_token161        self.update_post_processor()162 163        self.vocab_file = vocab_file164 165        self._prefix_token = prefix_token166        self._middle_token = middle_token167        self._suffix_token = suffix_token168        self._eot_token = eot_token169        self.fill_token = fill_token170 171    # Copied from transformers.models.llama.tokenization_llama_fast.LlamaTokenizerFast.update_post_processor172    def update_post_processor(self):173        """174        Updates the underlying post processor with the current `bos_token` and `eos_token`.175        """176        bos = self.bos_token177        bos_token_id = self.bos_token_id178        if bos is None and self.add_bos_token:179            raise ValueError("add_bos_token = True but bos_token = None")180 181        eos = self.eos_token182        eos_token_id = self.eos_token_id183        if eos is None and self.add_eos_token:184            raise ValueError("add_eos_token = True but eos_token = None")185 186        single = f"{(bos + ':0 ') if self.add_bos_token else ''}$A:0{(' ' + eos + ':0') if self.add_eos_token else ''}"187        pair = f"{single}{(' ' + bos + ':1') if self.add_bos_token else ''} $B:1{(' ' + eos + ':1') if self.add_eos_token else ''}"188 189        special_tokens = []190        if self.add_bos_token:191            special_tokens.append((bos, bos_token_id))192        if self.add_eos_token:193            special_tokens.append((eos, eos_token_id))194        self._tokenizer.post_processor = processors.TemplateProcessing(195            single=single, pair=pair, special_tokens=special_tokens196        )197 198    @property199    def prefix_token(self):200        return self._prefix_token201 202    @property203    def prefix_id(self):204        if self._prefix_token is None:205            return None206        return self.convert_tokens_to_ids(self.prefix_token)207 208    @property209    def middle_token(self):210        return self._middle_token211 212    @property213    def middle_id(self):214        if self._middle_token is None:215            return None216        return self.convert_tokens_to_ids(self.middle_token)217 218    @property219    def suffix_token(self):220        return self._suffix_token221 222    @property223    def suffix_id(self):224        if self._suffix_token is None:225            return None226        return self.convert_tokens_to_ids(self.suffix_token)227 228    @property229    def eot_id(self):230        if self._eot_token is None:231            return None232        return self.convert_tokens_to_ids(self.eot_token)233 234    @property235    def eot_token(self):236        return self._eot_token237 238    @property239    def add_eos_token(self):240        return self._add_eos_token241 242    @property243    def add_bos_token(self):244        return self._add_bos_token245 246    @add_eos_token.setter247    def add_eos_token(self, value):248        self._add_eos_token = value249        self.update_post_processor()250 251    @add_bos_token.setter252    def add_bos_token(self, value):253        self._add_bos_token = value254        self.update_post_processor()255 256    def set_infilling_processor(self, reset, suffix_first=False, add_special_tokens=True):257        """258        Updates the normalizer to make sure the prompt format for `infilling` is respected. The infilling format is the259        following: if suffix_first260            " <PRE> <SUF>{suf} <MID> {pre}"261        else:262            " <PRE> {pre} <SUF>{suf} <MID>"263 264        If `reset` is set to `True`, the `normalizer` and `post_processor` are reset to their "normal" behaviour, which265        is to add a prefix space for the normalizer, and add a `bos_token` to the input text for the `post_processor`.266        """267        if reset:268            self._tokenizer.normalizer = normalizers.Sequence(269                [270                    normalizers.Prepend(prepend="▁"),271                    normalizers.Replace(pattern=" ", content="▁"),272                ]273            )274            self.update_post_processor()275            return276 277        self._tokenizer.normalizer = normalizers.Replace(pattern=" ", content="▁")278        pair = [self.bos_token] if self.add_bos_token and add_special_tokens else []279        special_tokens = [(self.bos_token, self.bos_token_id)] if self.add_bos_token and add_special_tokens else []280        if suffix_first:281            # format as " <PRE> <SUF>{suf} <MID> {pre}"282            pair += [self.prefix_token, self.suffix_token, "$B", self.middle_token, "$A"]283            special_tokens += [284                (self.prefix_token, self.prefix_id),285                (self.suffix_token, self.suffix_id),286                (self.middle_token, self.middle_id),287            ]288        else:289            # format as " <PRE> {pre} <SUF>{suf} <MID>"290            pair += [self.prefix_token, "$A", self.suffix_token, "$B", self.middle_token]291            special_tokens += [292                (self.prefix_token, self.prefix_id),293                (self.suffix_token, self.suffix_id),294                (self.middle_token, self.middle_id),295            ]296 297        if self.add_eos_token and add_special_tokens:298            pair += [self.eos_token]299            special_tokens += [(self.eos_token, self.eos_token_id)]300        self._tokenizer.post_processor = processors.TemplateProcessing(301            single="$A", pair=pair, special_tokens=special_tokens302        )303 304    def encode_plus(self, text, text_pair=None, suffix_first=False, add_special_tokens=True, **kwargs):305        # hack to make sure the input is pre-process but outside rust306        text_pair = kwargs.pop("suffix", text_pair)307        if self.fill_token is not None and self.fill_token in text and text_pair is None:308            text, text_pair = text.split(self.fill_token)309 310        if text_pair is None or len(text_pair) < 1:311            return super().encode_plus(text, text_pair, add_special_tokens=add_special_tokens, **kwargs)312 313        if None in (self.prefix_id, self.middle_id, self.suffix_id):314            raise ValueError(315                "Then input includes a `prefix` and a `suffix` used for the infilling task,"316                " the `prefix_id, middle_id, suffix_id` must all be initialized. Current"317                f" values : {self.prefix_id, self.middle_id, self.suffix_id}"318            )319 320        self.set_infilling_processor(False, suffix_first=suffix_first, add_special_tokens=add_special_tokens)321        tokens = super().encode_plus(" " + text, text_pair=text_pair, add_special_tokens=True, **kwargs)322        self.set_infilling_processor(True)323        return tokens324 325    # Copied from transformers.models.llama.tokenization_llama_fast.LlamaTokenizerFast.save_vocabulary326    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:327        if not self.can_save_slow_tokenizer:328            raise ValueError(329                "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "330                "tokenizer."331            )332 333        if not os.path.isdir(save_directory):334            logger.error(f"Vocabulary path ({save_directory}) should be a directory")335            return336        out_vocab_file = os.path.join(337            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]338        )339 340        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):341            copyfile(self.vocab_file, out_vocab_file)342 343        return (out_vocab_file,)344 345    def build_inputs_with_special_tokens(346        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None347    ) -> list[int]:348        """349        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and350        adding special tokens. The special tokens depend on calling set_lang.351 352        An NLLB sequence has the following format, where `X` represents the sequence:353 354        - `input_ids` (for encoder) `X [eos, src_lang_code]`355        - `decoder_input_ids`: (for decoder) `X [eos, tgt_lang_code]`356 357        BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a358        separator.359 360        Args:361            token_ids_0 (`list[int]`):362                List of IDs to which the special tokens will be added.363            token_ids_1 (`list[int]`, *optional*):364                Optional second list of IDs for sequence pairs.365 366        Returns:367            `list[int]`: list of [input IDs](../glossary#input-ids) with the appropriate special tokens.368        """369        if token_ids_1 is None:370            return self.bos_token_id + token_ids_0 + self.eos_token_id371        return self.bos_token_id + token_ids_0 + token_ids_1 + self.eos_token_id372 373 374__all__ = ["CodeLlamaTokenizerFast"]375