Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2021 The Facebook Inc. and The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Fast Tokenization class for Blenderbot."""16 17import json18from typing import Optional19 20from tokenizers import processors21 22from ...tokenization_utils_base import AddedToken, BatchEncoding23from ...tokenization_utils_fast import PreTrainedTokenizerFast24from ...utils import logging25from .tokenization_blenderbot import BlenderbotTokenizer26 27 28logger = logging.get_logger(__name__)29 30 31VOCAB_FILES_NAMES = {32    "vocab_file": "vocab.json",33    "merges_file": "merges.txt",34    "tokenizer_config_file": "tokenizer_config.json",35}36 37 38class BlenderbotTokenizerFast(PreTrainedTokenizerFast):39    """40    Construct a "fast" Blenderbot tokenizer (backed by HuggingFace's *tokenizers* library), derived from the GPT-241    tokenizer, using byte-level Byte-Pair-Encoding.42 43    This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will44    be encoded differently whether it is at the beginning of the sentence (without space) or not:45 46    ```python47    >>> from transformers import BlenderbotTokenizerFast48 49    >>> tokenizer = BlenderbotTokenizerFast.from_pretrained("facebook/blenderbot-3B")50    >>> tokenizer("Hello world")["input_ids"]51    [6950, 1085, 2]52 53    >>> tokenizer(" Hello world")["input_ids"]54    [6950, 1085, 2]55    ```56 57    You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer or when you58    call it on some text, but since the model was not pretrained this way, it might yield a decrease in performance.59 60    <Tip>61 62    When used with `is_split_into_words=True`, this tokenizer needs to be instantiated with `add_prefix_space=True`.63 64    </Tip>65 66    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should67    refer to this superclass for more information regarding those methods.68 69    Args:70        vocab_file (`str`):71            Path to the vocabulary file.72        merges_file (`str`):73            Path to the merges file.74        errors (`str`, *optional*, defaults to `"replace"`):75            Paradigm to follow when decoding bytes to UTF-8. See76            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.77        bos_token (`str`, *optional*, defaults to `"<s>"`):78            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.79 80            <Tip>81 82            When building a sequence using special tokens, this is not the token that is used for the beginning of83            sequence. The token used is the `cls_token`.84 85            </Tip>86 87        eos_token (`str`, *optional*, defaults to `"</s>"`):88            The end of sequence token.89 90            <Tip>91 92            When building a sequence using special tokens, this is not the token that is used for the end of sequence.93            The token used is the `sep_token`.94 95            </Tip>96 97        sep_token (`str`, *optional*, defaults to `"</s>"`):98            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for99            sequence classification or for a text and a question for question answering. It is also used as the last100            token of a sequence built with special tokens.101        cls_token (`str`, *optional*, defaults to `"<s>"`):102            The classifier token which is used when doing sequence classification (classification of the whole sequence103            instead of per-token classification). It is the first token of the sequence when built with special tokens.104        unk_token (`str`, *optional*, defaults to `"<unk>"`):105            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this106            token instead.107        pad_token (`str`, *optional*, defaults to `"<pad>"`):108            The token used for padding, for example when batching sequences of different lengths.109        mask_token (`str`, *optional*, defaults to `"<mask>"`):110            The token used for masking values. This is the token used when training this model with masked language111            modeling. This is the token which the model will try to predict.112        add_prefix_space (`bool`, *optional*, defaults to `False`):113            Whether or not to add an initial space to the input. This allows to treat the leading word just as any114            other word. (Blenderbot tokenizer detect beginning of words by the preceding space).115        trim_offsets (`bool`, *optional*, defaults to `True`):116            Whether the post processing step should trim offsets to avoid including whitespaces.117    """118 119    vocab_files_names = VOCAB_FILES_NAMES120    model_input_names = ["input_ids", "attention_mask"]121    slow_tokenizer_class = BlenderbotTokenizer122 123    # Copied from transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast.__init__ with Roberta->Blenderbot, RoBERTa->Blenderbot124    def __init__(125        self,126        vocab_file=None,127        merges_file=None,128        tokenizer_file=None,129        errors="replace",130        bos_token="<s>",131        eos_token="</s>",132        sep_token="</s>",133        cls_token="<s>",134        unk_token="<unk>",135        pad_token="<pad>",136        mask_token="<mask>",137        add_prefix_space=False,138        trim_offsets=True,139        **kwargs,140    ):141        mask_token = (142            AddedToken(mask_token, lstrip=True, rstrip=False, normalized=False)143            if isinstance(mask_token, str)144            else mask_token145        )146        super().__init__(147            vocab_file,148            merges_file,149            tokenizer_file=tokenizer_file,150            errors=errors,151            bos_token=bos_token,152            eos_token=eos_token,153            sep_token=sep_token,154            cls_token=cls_token,155            unk_token=unk_token,156            pad_token=pad_token,157            mask_token=mask_token,158            add_prefix_space=add_prefix_space,159            trim_offsets=trim_offsets,160            **kwargs,161        )162 163        tokenizer_component = "post_processor"164        tokenizer_component_instance = getattr(self.backend_tokenizer, tokenizer_component, None)165        if tokenizer_component_instance:166            state = json.loads(tokenizer_component_instance.__getstate__())167 168            # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`169            if "sep" in state:170                state["sep"] = tuple(state["sep"])171            if "cls" in state:172                state["cls"] = tuple(state["cls"])173 174            changes_to_apply = False175 176            if state.get("add_prefix_space", add_prefix_space) != add_prefix_space:177                state["add_prefix_space"] = add_prefix_space178                changes_to_apply = True179 180            if state.get("trim_offsets", trim_offsets) != trim_offsets:181                state["trim_offsets"] = trim_offsets182                changes_to_apply = True183 184            if changes_to_apply:185                component_class = getattr(processors, state.pop("type"))186                new_value = component_class(**state)187                setattr(self.backend_tokenizer, tokenizer_component, new_value)188 189    @property190    # Copied from transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast.mask_token with Roberta->Blenderbot, RoBERTa->Blenderbot191    def mask_token(self) -> str:192        """193        `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while not194        having been set.195 196        Blenderbot tokenizer has a special mask token to be usable in the fill-mask pipeline. The mask token will greedily197        comprise the space before the *<mask>*.198        """199        if self._mask_token is None:200            if self.verbose:201                logger.error("Using mask_token, but it is not set yet.")202            return None203        return str(self._mask_token)204 205    @mask_token.setter206    def mask_token(self, value):207        """208        Overriding the default behavior of the mask token to have it eat the space before it.209 210        This is needed to preserve backward compatibility with all the previously used models based on Roberta.211        """212        # Mask token behave like a normal word, i.e. include the space before it213        # So we set lstrip to True214        value = AddedToken(value, lstrip=True, rstrip=False) if isinstance(value, str) else value215        self._mask_token = value216 217    # Copied from transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast._batch_encode_plus with Roberta->Blenderbot, RoBERTa->Blenderbot218    def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding:219        is_split_into_words = kwargs.get("is_split_into_words", False)220        assert self.add_prefix_space or not is_split_into_words, (221            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "222            "to use it with pretokenized inputs."223        )224 225        return super()._batch_encode_plus(*args, **kwargs)226 227    # Copied from transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast._encode_plus with Roberta->Blenderbot, RoBERTa->Blenderbot228    def _encode_plus(self, *args, **kwargs) -> BatchEncoding:229        is_split_into_words = kwargs.get("is_split_into_words", False)230 231        assert self.add_prefix_space or not is_split_into_words, (232            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "233            "to use it with pretokenized inputs."234        )235 236        return super()._encode_plus(*args, **kwargs)237 238    # Copied from transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast.save_vocabulary with Roberta->Blenderbot, RoBERTa->Blenderbot239    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:240        files = self._tokenizer.model.save(save_directory, name=filename_prefix)241        return tuple(files)242 243    # Copied from transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast.create_token_type_ids_from_sequences with Roberta->Blenderbot, RoBERTa->Blenderbot244    def create_token_type_ids_from_sequences(245        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None246    ) -> list[int]:247        """248        Create a mask from the two sequences passed to be used in a sequence-pair classification task. Blenderbot does not249        make use of token type ids, therefore a list of zeros is returned.250 251        Args:252            token_ids_0 (`list[int]`):253                List of IDs.254            token_ids_1 (`list[int]`, *optional*):255                Optional second list of IDs for sequence pairs.256 257        Returns:258            `list[int]`: List of zeros.259        """260        sep = [self.sep_token_id]261        cls = [self.cls_token_id]262 263        if token_ids_1 is None:264            return len(cls + token_ids_0 + sep) * [0]265        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]266 267    def build_inputs_with_special_tokens(self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None):268        """269        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and270        adding special tokens. A Blenderbot sequence has the following format:271        - single sequence: ` X </s>`272 273        Args:274            token_ids_0 (`list[int]`):275                List of IDs to which the special tokens will be added276            token_ids_1 (`list[int]`, *optional*):277                Will be ignored278        Returns:279            `list[int]`: list of [input IDs](../glossary#input-ids) with the appropriate special tokens.280        """281        return token_ids_0 + [self.eos_token_id]282 283 284__all__ = ["BlenderbotTokenizerFast"]285