Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2021 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Tokenization class for Perceiver."""16 17from typing import Optional18 19from ...tokenization_utils import AddedToken, PreTrainedTokenizer20from ...utils import logging21 22 23logger = logging.get_logger(__name__)24 25 26class PerceiverTokenizer(PreTrainedTokenizer):27    """28    Construct a Perceiver tokenizer. The Perceiver simply uses raw bytes utf-8 encoding.29 30    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to31    this superclass for more information regarding those methods.32 33    Args:34        pad_token (`str`, *optional*, defaults to `"[PAD]"`):35            The token used for padding, for example when batching sequences of different lengths.36        bos_token (`str`, *optional*, defaults to `"[BOS]"`):37            The BOS token (reserved in the vocab, but not actually used).38        eos_token (`str`, *optional*, defaults to `"[EOS]"`):39            The end of sequence token (reserved in the vocab, but not actually used).40 41            <Tip>42 43            When building a sequence using special tokens, this is not the token that is used for the end of sequence.44            The token used is the `sep_token`.45 46            </Tip>47 48        mask_token (`str`, *optional*, defaults to `"[MASK]"`):49            The MASK token, useful for masked language modeling.50        cls_token (`str`, *optional*, defaults to `"[CLS]"`):51            The CLS token (reserved in the vocab, but not actually used).52        sep_token (`str`, *optional*, defaults to `"[SEP]"`):53            The separator token, which is used when building a sequence from two sequences.54 55    """56 57    model_input_names = ["input_ids", "attention_mask"]58 59    def __init__(60        self,61        pad_token="[PAD]",62        bos_token="[BOS]",63        eos_token="[EOS]",64        mask_token="[MASK]",65        cls_token="[CLS]",66        sep_token="[SEP]",67        model_max_length=2048,68        **kwargs,69    ) -> None:70        pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token71        bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token72        eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token73        mask_token = AddedToken(mask_token, lstrip=False, rstrip=False) if isinstance(mask_token, str) else mask_token74        cls_token = AddedToken(cls_token, lstrip=False, rstrip=False) if isinstance(cls_token, str) else cls_token75        sep_token = AddedToken(sep_token, lstrip=False, rstrip=False) if isinstance(sep_token, str) else sep_token76 77        self._utf_vocab_size = 2**8  # utf is 8 bits78 79        # Since these tokens are not part of the vocabulary, we manually add them80        self._added_tokens_decoder: dict[str, int] = {81            0: pad_token,82            1: bos_token,83            2: eos_token,84            3: mask_token,85            4: cls_token,86            5: sep_token,87        }88        self._num_special_tokens = len(self._added_tokens_decoder)89        super().__init__(90            pad_token=pad_token,91            bos_token=bos_token,92            eos_token=eos_token,93            mask_token=mask_token,94            cls_token=cls_token,95            sep_token=sep_token,96            model_max_length=model_max_length,97            **kwargs,98        )99 100    def get_vocab(self) -> dict[str, int]:101        vocab = {}102        for i in range(self._utf_vocab_size):103            token = chr(i)104            vocab[token] = i + self._num_special_tokens105        vocab.update(self.added_tokens_encoder)106        return vocab107 108    @property109    def vocab_size(self):110        return self._utf_vocab_size111 112    def get_special_tokens_mask(113        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None, already_has_special_tokens: bool = False114    ) -> list[int]:115        """116        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding117        special tokens using the tokenizer `prepare_for_model` method.118 119        Args:120            token_ids_0 (`list[int]`):121                List of IDs.122            token_ids_1 (`list[int]`, *optional*):123                Optional second list of IDs for sequence pairs.124            already_has_special_tokens (`bool`, *optional*, defaults to `False`):125                Whether or not the token list is already formatted with special tokens for the model.126 127        Returns:128            `list[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.129        """130        if already_has_special_tokens:131            return super().get_special_tokens_mask(132                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True133            )134 135        # normal case: some special tokens136        if token_ids_1 is None:137            return [1] + [0] * len(token_ids_0) + [1]138        return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]139 140    def build_inputs_with_special_tokens(141        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None142    ) -> list[int]:143        """144        Build model inputs from a sequence or a pair of sequence for sequence classification tasks. A sequence has the145        following format:146 147        - single sequence: `[CLS] X [SEP]`148        - pair of sequences: `[CLS] A [SEP] B [SEP]`149 150        Args:151            token_ids_0 (`list[int]`):152                List of IDs to which the special tokens will be added.153            token_ids_1 (`list[int]`, *optional*):154                Optional second list of IDs for sequence pairs.155 156        Returns:157            `list[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.158        """159        if token_ids_1 is None:160            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]161        else:162            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + token_ids_1 + [self.sep_token_id]163 164    def _tokenize(self, text: str) -> list[str]:165        """Take as input a string and return a list of strings (tokens) for words/sub-words"""166        tokens = [chr(i) for i in text.encode("utf-8")]167        return tokens168 169    def _convert_token_to_id(self, token):170        """Converts a token (str) in an id using the vocab."""171        if len(token) != 1:172            token_id = self.unk_token_id173        else:174            token_id = ord(token) + self._num_special_tokens175        return token_id176 177    def _convert_id_to_token(self, index):178        """Converts an index (integer) in a token (str) using the vocab."""179        token = chr(index - self._num_special_tokens)180        return token181 182    # TODO @ArthurZ refactor this as well....183    def convert_tokens_to_string(self, tokens):184        """Converts a sequence of tokens (string) in a single string."""185        bstring = b""186        for token in tokens:187            if token in self.added_tokens_encoder:188                tok_string = str(token).encode("utf-8")189            else:190                tok_string = bytes([ord(token)])191            bstring += tok_string192        string = bstring.decode("utf-8", errors="replace")193        return string194 195    # PerceiverTokenizer has no vocab file196    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:197        return ()198 199 200__all__ = ["PerceiverTokenizer"]201