Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2020 The Trax Authors and The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Tokenization class for model Reformer."""16 17import os18from shutil import copyfile19from typing import Optional20 21from ...tokenization_utils_fast import PreTrainedTokenizerFast22from ...utils import is_sentencepiece_available, logging23 24 25if is_sentencepiece_available():26    from .tokenization_reformer import ReformerTokenizer27else:28    ReformerTokenizer = None29 30 31logger = logging.get_logger(__name__)32 33 34SPIECE_UNDERLINE = "▁"35 36VOCAB_FILES_NAMES = {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}37 38 39class ReformerTokenizerFast(PreTrainedTokenizerFast):40    """41    Construct a "fast" Reformer tokenizer (backed by HuggingFace's *tokenizers* library). Based on42    [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).43 44    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should45    refer to this superclass for more information regarding those methods.46 47    Args:48        vocab_file (`str`):49            [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that50            contains the vocabulary necessary to instantiate a tokenizer.51        eos_token (`str`, *optional*, defaults to `"</s>"`):52            The end of sequence token.53 54            <Tip>55 56            When building a sequence using special tokens, this is not the token that is used for the end of sequence.57            The token used is the `sep_token`.58 59            </Tip>60 61        unk_token (`str`, *optional*, defaults to `"<unk>"`):62            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this63            token instead.64        pad_token (`str`, *optional*, defaults to `"<pad>"`):65            The token used for padding, for example when batching sequences of different lengths.66        additional_special_tokens (`list[str]`, *optional*):67            Additional special tokens used by the tokenizer.68    """69 70    vocab_files_names = VOCAB_FILES_NAMES71    model_input_names = ["input_ids", "attention_mask"]72    slow_tokenizer_class = ReformerTokenizer73 74    def __init__(75        self,76        vocab_file=None,77        tokenizer_file=None,78        eos_token="</s>",79        unk_token="<unk>",80        additional_special_tokens=[],81        **kwargs,82    ):83        super().__init__(84            vocab_file,85            tokenizer_file=tokenizer_file,86            eos_token=eos_token,87            unk_token=unk_token,88            additional_special_tokens=additional_special_tokens,89            **kwargs,90        )91 92        self.vocab_file = vocab_file93 94    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:95        if not self.can_save_slow_tokenizer:96            raise ValueError(97                "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "98                "tokenizer."99            )100 101        if not os.path.isdir(save_directory):102            logger.error(f"Vocabulary path ({save_directory}) should be a directory")103            return104        out_vocab_file = os.path.join(105            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]106        )107 108        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):109            copyfile(self.vocab_file, out_vocab_file)110 111        return (out_vocab_file,)112 113 114__all__ = ["ReformerTokenizerFast"]115