Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import json17from typing import Optional18 19from tokenizers import processors20 21from ...tokenization_utils_base import AddedToken, BatchEncoding22from ...tokenization_utils_fast import PreTrainedTokenizerFast23from ...utils import logging24from .tokenization_bart import BartTokenizer25 26 27logger = logging.get_logger(__name__)28 29 30VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}31 32# See all BART models at https://huggingface.co/models?filter=bart33 34 35class BartTokenizerFast(PreTrainedTokenizerFast):36    r"""37    Construct a "fast" BART tokenizer (backed by HuggingFace's *tokenizers* library), derived from the GPT-2 tokenizer,38    using byte-level Byte-Pair-Encoding.39 40    This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will41    be encoded differently whether it is at the beginning of the sentence (without space) or not:42 43    ```python44    >>> from transformers import BartTokenizerFast45 46    >>> tokenizer = BartTokenizerFast.from_pretrained("facebook/bart-base")47    >>> tokenizer("Hello world")["input_ids"]48    [0, 31414, 232, 2]49 50    >>> tokenizer(" Hello world")["input_ids"]51    [0, 20920, 232, 2]52    ```53 54    You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer or when you55    call it on some text, but since the model was not pretrained this way, it might yield a decrease in performance.56 57    <Tip>58 59    When used with `is_split_into_words=True`, this tokenizer needs to be instantiated with `add_prefix_space=True`.60 61    </Tip>62 63    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should64    refer to this superclass for more information regarding those methods.65 66    Args:67        vocab_file (`str`):68            Path to the vocabulary file.69        merges_file (`str`):70            Path to the merges file.71        errors (`str`, *optional*, defaults to `"replace"`):72            Paradigm to follow when decoding bytes to UTF-8. See73            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.74        bos_token (`str`, *optional*, defaults to `"<s>"`):75            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.76 77            <Tip>78 79            When building a sequence using special tokens, this is not the token that is used for the beginning of80            sequence. The token used is the `cls_token`.81 82            </Tip>83 84        eos_token (`str`, *optional*, defaults to `"</s>"`):85            The end of sequence token.86 87            <Tip>88 89            When building a sequence using special tokens, this is not the token that is used for the end of sequence.90            The token used is the `sep_token`.91 92            </Tip>93 94        sep_token (`str`, *optional*, defaults to `"</s>"`):95            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for96            sequence classification or for a text and a question for question answering. It is also used as the last97            token of a sequence built with special tokens.98        cls_token (`str`, *optional*, defaults to `"<s>"`):99            The classifier token which is used when doing sequence classification (classification of the whole sequence100            instead of per-token classification). It is the first token of the sequence when built with special tokens.101        unk_token (`str`, *optional*, defaults to `"<unk>"`):102            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this103            token instead.104        pad_token (`str`, *optional*, defaults to `"<pad>"`):105            The token used for padding, for example when batching sequences of different lengths.106        mask_token (`str`, *optional*, defaults to `"<mask>"`):107            The token used for masking values. This is the token used when training this model with masked language108            modeling. This is the token which the model will try to predict.109        add_prefix_space (`bool`, *optional*, defaults to `False`):110            Whether or not to add an initial space to the input. This allows to treat the leading word just as any111            other word. (BART tokenizer detect beginning of words by the preceding space).112        trim_offsets (`bool`, *optional*, defaults to `True`):113            Whether the post processing step should trim offsets to avoid including whitespaces.114    """115 116    vocab_files_names = VOCAB_FILES_NAMES117    model_input_names = ["input_ids", "attention_mask"]118    slow_tokenizer_class = BartTokenizer119 120    def __init__(121        self,122        vocab_file=None,123        merges_file=None,124        tokenizer_file=None,125        errors="replace",126        bos_token="<s>",127        eos_token="</s>",128        sep_token="</s>",129        cls_token="<s>",130        unk_token="<unk>",131        pad_token="<pad>",132        mask_token="<mask>",133        add_prefix_space=False,134        trim_offsets=True,135        **kwargs,136    ):137        # we have to specify that this tokens is special otherwise adding it will reset the normalized flag to `False` in `add_special_tokens`138        mask_token = (139            AddedToken(mask_token, lstrip=True, normalized=True, special=True)140            if isinstance(mask_token, str)141            else mask_token142        )143        super().__init__(144            vocab_file,145            merges_file,146            tokenizer_file=tokenizer_file,147            errors=errors,148            bos_token=bos_token,149            eos_token=eos_token,150            sep_token=sep_token,151            cls_token=cls_token,152            unk_token=unk_token,153            pad_token=pad_token,154            mask_token=mask_token,155            add_prefix_space=add_prefix_space,156            trim_offsets=trim_offsets,157            **kwargs,158        )159 160        # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`161        tokenizer_component = "post_processor"162        tokenizer_component_instance = getattr(self.backend_tokenizer, tokenizer_component, None)163        if tokenizer_component_instance:164            state = json.loads(tokenizer_component_instance.__getstate__())165 166            # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`167            if "sep" in state:168                state["sep"] = tuple(state["sep"])169            if "cls" in state:170                state["cls"] = tuple(state["cls"])171 172            changes_to_apply = False173 174            if state.get("add_prefix_space", add_prefix_space) != add_prefix_space:175                state["add_prefix_space"] = add_prefix_space176                changes_to_apply = True177 178            if state.get("trim_offsets", trim_offsets) != trim_offsets:179                state["trim_offsets"] = trim_offsets180                changes_to_apply = True181 182            if changes_to_apply:183                component_class = getattr(processors, state.pop("type"))184                new_value = component_class(**state)185                setattr(self.backend_tokenizer, tokenizer_component, new_value)186 187    @property188    def mask_token(self) -> str:189        """190        `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while not191        having been set.192 193        BART tokenizer has a special mask token to be usable in the fill-mask pipeline. The mask token will greedily194        comprise the space before the *<mask>*.195        """196        if self._mask_token is None:197            if self.verbose:198                logger.error("Using mask_token, but it is not set yet.")199            return None200        return str(self._mask_token)201 202    @mask_token.setter203    def mask_token(self, value):204        """205        Overriding the default behavior of the mask token to have it eat the space before it.206 207        This is needed to preserve backward compatibility with all the previously used models based on Bart.208        """209        # Mask token behave like a normal word, i.e. include the space before it210        # So we set lstrip to True211        value = AddedToken(value, lstrip=True, rstrip=False) if isinstance(value, str) else value212        self._mask_token = value213 214    def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding:215        is_split_into_words = kwargs.get("is_split_into_words", False)216 217        if is_split_into_words and not self.add_prefix_space:218            raise ValueError(219                f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "220                "to use it with pretokenized inputs."221            )222 223        return super()._batch_encode_plus(*args, **kwargs)224 225    def _encode_plus(self, *args, **kwargs) -> BatchEncoding:226        is_split_into_words = kwargs.get("is_split_into_words", False)227 228        if is_split_into_words and not self.add_prefix_space:229            raise ValueError(230                f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "231                "to use it with pretokenized inputs."232            )233 234        return super()._encode_plus(*args, **kwargs)235 236    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:237        files = self._tokenizer.model.save(save_directory, name=filename_prefix)238        return tuple(files)239 240    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):241        output = [self.bos_token_id] + token_ids_0 + [self.eos_token_id]242        if token_ids_1 is None:243            return output244 245        return output + [self.eos_token_id] + token_ids_1 + [self.eos_token_id]246 247    def create_token_type_ids_from_sequences(248        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None249    ) -> list[int]:250        """251        Create a mask from the two sequences passed to be used in a sequence-pair classification task. BART does not252        make use of token type ids, therefore a list of zeros is returned.253 254        Args:255            token_ids_0 (`list[int]`):256                List of IDs.257            token_ids_1 (`list[int]`, *optional*):258                Optional second list of IDs for sequence pairs.259 260        Returns:261            `list[int]`: List of zeros.262        """263        sep = [self.sep_token_id]264        cls = [self.cls_token_id]265 266        if token_ids_1 is None:267            return len(cls + token_ids_0 + sep) * [0]268        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]269 270 271__all__ = ["BartTokenizerFast"]272