Aluode/PerceptionLabPortable
0
1# Copyright 2020 The HuggingFace Inc. team.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""15Tokenization classes for fast tokenizers (provided by HuggingFace's tokenizers library). For slow (python) tokenizers16see tokenization_utils.py17"""18 19import copy20import json21import os22from collections import defaultdict23from collections.abc import Iterable24from typing import Any, Optional, Union25 26import tokenizers.pre_tokenizers as pre_tokenizers_fast27from tokenizers import Encoding as EncodingFast28from tokenizers import Tokenizer as TokenizerFast29from tokenizers.decoders import Decoder as DecoderFast30from tokenizers.trainers import BpeTrainer, UnigramTrainer, WordLevelTrainer, WordPieceTrainer31 32from .convert_slow_tokenizer import convert_slow_tokenizer33from .integrations.ggml import convert_gguf_tokenizer34from .modeling_gguf_pytorch_utils import load_gguf_checkpoint35from .tokenization_utils import PreTrainedTokenizer36from .tokenization_utils_base import (37 INIT_TOKENIZER_DOCSTRING,38 AddedToken,39 BatchEncoding,40 PreTokenizedInput,41 PreTokenizedInputPair,42 PreTrainedTokenizerBase,43 SpecialTokensMixin,44 TextInput,45 TextInputPair,46 TruncationStrategy,47)48from .utils import PaddingStrategy, add_end_docstrings, logging49 50 51logger = logging.get_logger(__name__)52 53# Fast tokenizers (provided by HuggingFace tokenizer's library) can be saved in a single file54TOKENIZER_FILE = "tokenizer.json"55SPECIAL_TOKENS_MAP_FILE = "special_tokens_map.json"56TOKENIZER_CONFIG_FILE = "tokenizer_config.json"57TIKTOKEN_VOCAB_FILE = "tokenizer.model"58 59# Slow tokenizers have an additional added tokens files60ADDED_TOKENS_FILE = "added_tokens.json"61 62INIT_TOKENIZER_DOCSTRING += """63 tokenizer_object ([`tokenizers.Tokenizer`]):64 A [`tokenizers.Tokenizer`] object from 🤗 tokenizers to instantiate from. See [Using tokenizers from 🤗65 tokenizers](../fast_tokenizers) for more information.66 tokenizer_file ([`str`]):67 A path to a local JSON file representing a previously serialized [`tokenizers.Tokenizer`] object from 🤗68 tokenizers.69"""70 71MODEL_TO_TRAINER_MAPPING = {72 "BPE": BpeTrainer,73 "Unigram": UnigramTrainer,74 "WordLevel": WordLevelTrainer,75 "WordPiece": WordPieceTrainer,76}77 78VOCAB_FILES_NAMES = {"tokenizer_file": TOKENIZER_FILE, "vocab_file": TIKTOKEN_VOCAB_FILE}79 80 81@add_end_docstrings(INIT_TOKENIZER_DOCSTRING)82class PreTrainedTokenizerFast(PreTrainedTokenizerBase):83 """84 Base class for all fast tokenizers (wrapping HuggingFace tokenizers library).85 86 Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`].87 88 Handles all the shared methods for tokenization and special tokens, as well as methods for89 downloading/caching/loading pretrained tokenizers, as well as adding tokens to the vocabulary.90 91 This class also contains the added tokens in a unified way on top of all tokenizers so we don't have to handle the92 specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).93 """94 95 vocab_files_names = VOCAB_FILES_NAMES96 slow_tokenizer_class: Optional[type[PreTrainedTokenizer]] = None97 98 def __init__(self, *args, **kwargs):99 tokenizer_object = kwargs.pop("tokenizer_object", None)100 slow_tokenizer = kwargs.pop("__slow_tokenizer", None)101 gguf_file = kwargs.pop("gguf_file", None)102 fast_tokenizer_file = kwargs.pop("tokenizer_file", None)103 from_slow = kwargs.pop("from_slow", False)104 added_tokens_decoder = kwargs.pop("added_tokens_decoder", {})105 self.add_prefix_space = kwargs.get("add_prefix_space", False)106 107 if from_slow and slow_tokenizer is None and self.slow_tokenizer_class is None:108 raise ValueError(109 "Cannot instantiate this tokenizer from a slow version. If it's based on sentencepiece, make sure you "110 "have sentencepiece installed."111 )112 113 if tokenizer_object is not None:114 fast_tokenizer = copy.deepcopy(tokenizer_object)115 elif fast_tokenizer_file is not None and not from_slow:116 # We have a serialization from tokenizers which let us directly build the backend117 fast_tokenizer = TokenizerFast.from_file(fast_tokenizer_file)118 elif slow_tokenizer:119 # We need to convert a slow tokenizer to build the backend120 fast_tokenizer = convert_slow_tokenizer(slow_tokenizer)121 elif gguf_file is not None:122 # We need to convert a slow tokenizer to build the backend123 gguf_param = load_gguf_checkpoint(kwargs.get("vocab_file"))124 architecture = gguf_param["config"]["model_type"]125 tokenizer_dict = gguf_param["tokenizer"]126 tokenizer_config = gguf_param["tokenizer_config"]127 fast_tokenizer, additional_kwargs = convert_gguf_tokenizer(architecture, tokenizer_dict)128 kwargs.update(tokenizer_config)129 if len(additional_kwargs) > 0:130 kwargs.update(additional_kwargs)131 elif self.slow_tokenizer_class is not None and slow_tokenizer is not False:132 # We need to create and convert a slow tokenizer to build the backend133 slow_tokenizer = self.slow_tokenizer_class(*args, **kwargs)134 fast_tokenizer = convert_slow_tokenizer(slow_tokenizer)135 elif not slow_tokenizer:136 # We tried loading a slow_tokenizer with spm and failed, try to load with tiktoken137 self.vocab_file = kwargs.get("vocab_file")138 self.additional_special_tokens = kwargs.get("additional_special_tokens", [])139 fast_tokenizer = convert_slow_tokenizer(self, from_tiktoken=True)140 slow_tokenizer = None141 else:142 raise ValueError(143 "Couldn't instantiate the backend tokenizer from one of: \n"144 "(1) a `tokenizers` library serialization file, \n"145 "(2) a slow tokenizer instance to convert or \n"146 "(3) an equivalent slow tokenizer class to instantiate and convert. \n"147 "You need to have sentencepiece or tiktoken installed to convert a slow tokenizer to a fast one."148 )149 150 self._tokenizer = fast_tokenizer151 152 if slow_tokenizer is not None:153 kwargs.update(slow_tokenizer.init_kwargs)154 155 self._decode_use_source_tokenizer = False156 157 _truncation = self._tokenizer.truncation158 159 if _truncation is not None:160 self._tokenizer.enable_truncation(**_truncation)161 kwargs.setdefault("max_length", _truncation["max_length"])162 kwargs.setdefault("truncation_side", _truncation["direction"])163 kwargs.setdefault("stride", _truncation["stride"])164 kwargs.setdefault("truncation_strategy", _truncation["strategy"])165 else:166 self._tokenizer.no_truncation()167 168 _padding = self._tokenizer.padding169 if _padding is not None:170 self._tokenizer.enable_padding(**_padding)171 kwargs.setdefault("pad_token", _padding["pad_token"])172 kwargs.setdefault("pad_token_type_id", _padding["pad_type_id"])173 kwargs.setdefault("padding_side", _padding["direction"])174 kwargs.setdefault("max_length", _padding["length"])175 kwargs.setdefault("pad_to_multiple_of", _padding["pad_to_multiple_of"])176 177 # We call this after having initialized the backend tokenizer because we update it.178 super().__init__(**kwargs)179 self._tokenizer.encode_special_tokens = self.split_special_tokens180 181 added_tokens_decoder_hash = {hash(repr(token)) for token in self.added_tokens_decoder}182 tokens_to_add = [183 token184 for index, token in sorted(added_tokens_decoder.items(), key=lambda x: x[0])185 if hash(repr(token)) not in added_tokens_decoder_hash186 ]187 encoder = list(self.added_tokens_encoder.keys()) + [str(token) for token in tokens_to_add]188 # if some of the special tokens are strings, we check if we don't already have a token189 tokens_to_add += [190 token for token in self.all_special_tokens_extended if token not in encoder and token not in tokens_to_add191 ]192 193 if len(tokens_to_add) > 0:194 tokens = []195 special_tokens = self.all_special_tokens196 for token in tokens_to_add:197 is_special = (198 (token.special or str(token) in special_tokens)199 if isinstance(token, AddedToken)200 else str(token) in special_tokens201 )202 if isinstance(token, str):203 token = AddedToken(token, special=is_special)204 else:205 token.special = is_special206 tokens.append(token)207 if tokens:208 self.add_tokens(tokens)209 210 try:211 pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())212 if pre_tok_state.get("add_prefix_space", self.add_prefix_space) != self.add_prefix_space:213 pre_tok_class = getattr(pre_tokenizers_fast, pre_tok_state.pop("type"))214 pre_tok_state["add_prefix_space"] = self.add_prefix_space215 self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state)216 except Exception:217 # We'll get an error if there is no pre_tokenizer, or if it's a custom pre_tokenizer that can218 # not be serialized. In those cases, we just ignore the error as there's no pre_tokenizer219 # for which we need to update the `add_prefix_space` attribute.220 pass221 222 @property223 def is_fast(self) -> bool:224 return True225 226 @property227 def can_save_slow_tokenizer(self) -> bool:228 """229 `bool`: Whether or not the slow tokenizer can be saved. For a sentencepiece based slow tokenizer, this230 can only be `True` if the original `"sentencepiece.model"` was not deleted.231 """232 if "vocab_file" in self.vocab_files_names and self.vocab_files_names["vocab_file"].endswith(".model"):233 if hasattr(self, "vocab_file") and self.vocab_file:234 # If the vocab file is a sentencepiece model, we can save it235 return os.path.isfile(self.vocab_file)236 return False237 else:238 return True239 240 @property241 def vocab_size(self) -> int:242 """243 `int`: Size of the base vocabulary (without the added tokens).244 """245 return self._tokenizer.get_vocab_size(with_added_tokens=False)246 247 def get_vocab(self) -> dict[str, int]:248 return self._tokenizer.get_vocab(with_added_tokens=True)249 250 @property251 def vocab(self) -> dict[str, int]:252 return self.get_vocab()253 254 @property255 def added_tokens_encoder(self) -> dict[str, int]:256 """257 Returns the sorted mapping from string to index. The added tokens encoder is cached for performance258 optimisation in `self._added_tokens_encoder` for the slow tokenizers.259 """260 return {k.content: v for v, k in sorted(self.added_tokens_decoder.items(), key=lambda item: item[0])}261 262 @property263 def added_tokens_decoder(self) -> dict[int, AddedToken]:264 """265 Returns the added tokens in the vocabulary as a dictionary of index to AddedToken.266 267 Returns:268 `dict[str, int]`: The added tokens.269 """270 return self._tokenizer.get_added_tokens_decoder()271 272 def get_added_vocab(self) -> dict[str, int]:273 """274 Returns the added tokens in the vocabulary as a dictionary of token to index.275 276 Returns:277 `dict[str, int]`: The added tokens.278 """279 return {k.content: v for v, k in sorted(self.added_tokens_decoder.items(), key=lambda item: item[0])}280 281 def __bool__(self) -> bool:282 """283 Returns True, to avoid expensive `assert tokenizer` gotchas.284 """285 return True286 287 def __len__(self) -> int:288 """289 Size of the full vocabulary with the added tokens.290 """291 return self._tokenizer.get_vocab_size(with_added_tokens=True)292 293 @property294 def backend_tokenizer(self) -> TokenizerFast:295 """296 `tokenizers.implementations.BaseTokenizer`: The Rust tokenizer used as a backend.297 """298 return self._tokenizer299 300 @property301 def decoder(self) -> DecoderFast:302 """303 `tokenizers.decoders.Decoder`: The Rust decoder for this tokenizer.304 """305 return self._tokenizer.decoder306 307 def _convert_encoding(308 self,309 encoding: EncodingFast,310 return_token_type_ids: Optional[bool] = None,311 return_attention_mask: Optional[bool] = None,312 return_overflowing_tokens: bool = False,313 return_special_tokens_mask: bool = False,314 return_offsets_mapping: bool = False,315 return_length: bool = False,316 verbose: bool = True,317 ) -> tuple[dict[str, Any], list[EncodingFast]]:318 """319 Convert the encoding representation (from low-level HuggingFace tokenizer output) to a python Dict and a list320 of encodings, take care of building a batch from overflowing tokens.321 322 Overflowing tokens are converted to additional examples (like batches) so the output values of the dict are323 lists (overflows) of lists (tokens).324 325 Output shape: (overflows, sequence length)326 """327 if return_token_type_ids is None:328 return_token_type_ids = "token_type_ids" in self.model_input_names329 if return_attention_mask is None:330 return_attention_mask = "attention_mask" in self.model_input_names331 332 if return_overflowing_tokens and encoding.overflowing is not None:333 encodings = [encoding] + encoding.overflowing334 else:335 encodings = [encoding]336 337 encoding_dict = defaultdict(list)338 for e in encodings:339 encoding_dict["input_ids"].append(e.ids)340 341 if return_token_type_ids:342 encoding_dict["token_type_ids"].append(e.type_ids)343 if return_attention_mask:344 encoding_dict["attention_mask"].append(e.attention_mask)345 if return_special_tokens_mask:346 encoding_dict["special_tokens_mask"].append(e.special_tokens_mask)347 if return_offsets_mapping:348 encoding_dict["offset_mapping"].append(e.offsets)349 if return_length:350 encoding_dict["length"].append(len(e.ids))351 352 return encoding_dict, encodings353 354 def convert_tokens_to_ids(self, tokens: Union[str, Iterable[str]]) -> Union[int, list[int]]:355 """356 Converts a token string (or a sequence of tokens) in a single integer id (or a Iterable of ids), using the357 vocabulary.358 359 Args:360 tokens (`str` or `Iterable[str]`): One or several token(s) to convert to token id(s).361 362 Returns:363 `int` or `list[int]`: The token id or list of token ids.364 """365 if isinstance(tokens, str):366 return self._convert_token_to_id_with_added_voc(tokens)367 368 return [self._convert_token_to_id_with_added_voc(token) for token in tokens]369 370 def _convert_token_to_id_with_added_voc(self, token: str) -> int:371 index = self._tokenizer.token_to_id(token)372 if index is None:373 return self.unk_token_id374 return index375 376 def _convert_id_to_token(self, index: int) -> Optional[str]:377 return self._tokenizer.id_to_token(int(index))378 379 def _add_tokens(self, new_tokens: list[Union[str, AddedToken]], special_tokens=False) -> int:380 if special_tokens:381 return self._tokenizer.add_special_tokens(new_tokens)382 383 return self._tokenizer.add_tokens(new_tokens)384 385 def num_special_tokens_to_add(self, pair: bool = False) -> int:386 """387 Returns the number of added tokens when encoding a sequence with special tokens.388 389 <Tip>390 391 This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put392 this inside your training loop.393 394 </Tip>395 396 Args:397 pair (`bool`, *optional*, defaults to `False`):398 Whether the number of added tokens should be computed in the case of a sequence pair or a single399 sequence.400 401 Returns:402 `int`: Number of special tokens added to sequences.403 """404 return self._tokenizer.num_special_tokens_to_add(pair)405 406 def convert_ids_to_tokens(407 self, ids: Union[int, list[int]], skip_special_tokens: bool = False408 ) -> Union[str, list[str]]:409 """410 Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and411 added tokens.412 413 Args:414 ids (`int` or `list[int]`):415 The token id (or token ids) to convert to tokens.416 skip_special_tokens (`bool`, *optional*, defaults to `False`):417 Whether or not to remove special tokens in the decoding.418 419 Returns:420 `str` or `list[str]`: The decoded token(s).421 """422 if isinstance(ids, int):423 return self._tokenizer.id_to_token(ids)424 tokens = []425 # self.all_special_ids is an @property which may be slow, so only compute it once before the loop426 ids_to_skip = set(self.all_special_ids) if skip_special_tokens else set()427 for index in ids:428 index = int(index)429 if index in ids_to_skip:430 continue431 tokens.append(self._tokenizer.id_to_token(index))432 return tokens433 434 def tokenize(self, text: str, pair: Optional[str] = None, add_special_tokens: bool = False, **kwargs) -> list[str]:435 return self.encode_plus(text=text, text_pair=pair, add_special_tokens=add_special_tokens, **kwargs).tokens()436 437 def set_truncation_and_padding(438 self,439 padding_strategy: PaddingStrategy,440 truncation_strategy: TruncationStrategy,441 max_length: int,442 stride: int,443 pad_to_multiple_of: Optional[int],444 padding_side: Optional[str],445 ):446 """447 Define the truncation and the padding strategies for fast tokenizers (provided by HuggingFace tokenizers448 library) and restore the tokenizer settings afterwards.449 450 The provided tokenizer has no padding / truncation strategy before the managed section. If your tokenizer set a451 padding / truncation strategy before, then it will be reset to no padding / truncation when exiting the managed452 section.453 454 Args:455 padding_strategy ([`~utils.PaddingStrategy`]):456 The kind of padding that will be applied to the input457 truncation_strategy ([`~tokenization_utils_base.TruncationStrategy`]):458 The kind of truncation that will be applied to the input459 max_length (`int`):460 The maximum size of a sequence.461 stride (`int`):462 The stride to use when handling overflow.463 pad_to_multiple_of (`int`, *optional*):464 If set will pad the sequence to a multiple of the provided value. This is especially useful to enable465 the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta).466 padding_side (`str`, *optional*):467 The side on which the model should have padding applied. Should be selected between ['right', 'left'].468 Default value is picked from the class attribute of the same name.469 """470 _truncation = self._tokenizer.truncation471 _padding = self._tokenizer.padding472 # Set truncation and padding on the backend tokenizer473 if truncation_strategy == TruncationStrategy.DO_NOT_TRUNCATE:474 if _truncation is not None:475 self._tokenizer.no_truncation()476 else:477 target = {478 "max_length": max_length,479 "stride": stride,480 "strategy": truncation_strategy.value,481 "direction": self.truncation_side,482 }483 484 # _truncation might contain more keys that the target `transformers`485 # supports. Use only the target keys to trigger `enable_truncation`.486 # This should enable this code to works on various `tokenizers`487 # targets.488 if _truncation is None:489 current = None490 else:491 current = {k: _truncation.get(k, None) for k in target}492 493 if current != target:494 self._tokenizer.enable_truncation(**target)495 496 if padding_strategy == PaddingStrategy.DO_NOT_PAD:497 if _padding is not None:498 self._tokenizer.no_padding()499 else:500 length = max_length if padding_strategy == PaddingStrategy.MAX_LENGTH else None501 target = {502 "length": length,503 "direction": padding_side if padding_side is not None else self.padding_side,504 "pad_id": self.pad_token_id,505 "pad_token": self.pad_token,506 "pad_type_id": self.pad_token_type_id,507 "pad_to_multiple_of": pad_to_multiple_of,508 }509 if _padding != target:510 self._tokenizer.enable_padding(**target)511 512 def _batch_encode_plus(513 self,514 batch_text_or_text_pairs: Union[515 list[TextInput], list[TextInputPair], list[PreTokenizedInput], list[PreTokenizedInputPair]516 ],517 add_special_tokens: bool = True,518 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,519 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,520 max_length: Optional[int] = None,521 stride: int = 0,522 is_split_into_words: bool = False,523 pad_to_multiple_of: Optional[int] = None,524 padding_side: Optional[str] = None,525 return_tensors: Optional[str] = None,526 return_token_type_ids: Optional[bool] = None,527 return_attention_mask: Optional[bool] = None,528 return_overflowing_tokens: bool = False,529 return_special_tokens_mask: bool = False,530 return_offsets_mapping: bool = False,531 return_length: bool = False,532 verbose: bool = True,533 split_special_tokens: bool = False,534 ) -> BatchEncoding:535 if not isinstance(batch_text_or_text_pairs, (tuple, list)):536 raise TypeError(537 f"batch_text_or_text_pairs has to be a list or a tuple (got {type(batch_text_or_text_pairs)})"538 )539 540 # Set the truncation and padding strategy and restore the initial configuration541 self.set_truncation_and_padding(542 padding_strategy=padding_strategy,543 truncation_strategy=truncation_strategy,544 max_length=max_length,545 stride=stride,546 pad_to_multiple_of=pad_to_multiple_of,547 padding_side=padding_side,548 )549 550 if self._tokenizer.encode_special_tokens != split_special_tokens:551 self._tokenizer.encode_special_tokens = split_special_tokens552 553 encodings = self._tokenizer.encode_batch(554 batch_text_or_text_pairs,555 add_special_tokens=add_special_tokens,556 is_pretokenized=is_split_into_words,557 )558 559 # Convert encoding to dict560 # `Tokens` has type: tuple[561 # list[dict[str, list[list[int]]]] or list[dict[str, 2D-Tensor]],562 # list[EncodingFast]563 # ]564 # with nested dimensions corresponding to batch, overflows, sequence length565 tokens_and_encodings = [566 self._convert_encoding(567 encoding=encoding,568 return_token_type_ids=return_token_type_ids,569 return_attention_mask=return_attention_mask,570 return_overflowing_tokens=return_overflowing_tokens,571 return_special_tokens_mask=return_special_tokens_mask,572 return_offsets_mapping=return_offsets_mapping,573 return_length=return_length,574 verbose=verbose,575 )576 for encoding in encodings577 ]578 579 # Convert the output to have dict[list] from list[dict] and remove the additional overflows dimension580 # From (variable) shape (batch, overflows, sequence length) to ~ (batch * overflows, sequence length)581 # (we say ~ because the number of overflow varies with the example in the batch)582 #583 # To match each overflowing sample with the original sample in the batch584 # we add an overflow_to_sample_mapping array (see below)585 sanitized_tokens = {}586 for key in tokens_and_encodings[0][0]:587 stack = [e for item, _ in tokens_and_encodings for e in item[key]]588 sanitized_tokens[key] = stack589 sanitized_encodings = [e for _, item in tokens_and_encodings for e in item]590 591 # If returning overflowing tokens, we need to return a mapping592 # from the batch idx to the original sample593 if return_overflowing_tokens:594 overflow_to_sample_mapping = []595 for i, (toks, _) in enumerate(tokens_and_encodings):596 overflow_to_sample_mapping += [i] * len(toks["input_ids"])597 sanitized_tokens["overflow_to_sample_mapping"] = overflow_to_sample_mapping598 599 for input_ids in sanitized_tokens["input_ids"]:600 self._eventual_warn_about_too_long_sequence(input_ids, max_length, verbose)601 return BatchEncoding(sanitized_tokens, sanitized_encodings, tensor_type=return_tensors)602 603 def _encode_plus(604 self,605 text: Union[TextInput, PreTokenizedInput],606 text_pair: Optional[Union[TextInput, PreTokenizedInput]] = None,607 add_special_tokens: bool = True,608 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,609 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,610 max_length: Optional[int] = None,611 stride: int = 0,612 is_split_into_words: bool = False,613 pad_to_multiple_of: Optional[int] = None,614 padding_side: Optional[str] = None,615 return_tensors: Optional[bool] = None,616 return_token_type_ids: Optional[bool] = None,617 return_attention_mask: Optional[bool] = None,618 return_overflowing_tokens: bool = False,619 return_special_tokens_mask: bool = False,620 return_offsets_mapping: bool = False,621 return_length: bool = False,622 verbose: bool = True,623 split_special_tokens: bool = False,624 **kwargs,625 ) -> BatchEncoding:626 batched_input = [(text, text_pair)] if text_pair else [text]627 batched_output = self._batch_encode_plus(628 batched_input,629 is_split_into_words=is_split_into_words,630 add_special_tokens=add_special_tokens,631 padding_strategy=padding_strategy,632 truncation_strategy=truncation_strategy,633 max_length=max_length,634 stride=stride,635 pad_to_multiple_of=pad_to_multiple_of,636 padding_side=padding_side,637 return_tensors=return_tensors,638 return_token_type_ids=return_token_type_ids,639 return_attention_mask=return_attention_mask,640 return_overflowing_tokens=return_overflowing_tokens,641 return_special_tokens_mask=return_special_tokens_mask,642 return_offsets_mapping=return_offsets_mapping,643 return_length=return_length,644 verbose=verbose,645 split_special_tokens=split_special_tokens,646 **kwargs,647 )648 649 # Return tensor is None, then we can remove the leading batch axis650 # Overflowing tokens are returned as a batch of output so we keep them in this case651 if return_tensors is None and not return_overflowing_tokens:652 batched_output = BatchEncoding(653 {654 key: (value[0] if len(value) > 0 and isinstance(value[0], list) else value)655 for key, value in batched_output.items()656 },657 batched_output.encodings,658 )659 660 self._eventual_warn_about_too_long_sequence(batched_output["input_ids"], max_length, verbose)661 662 return batched_output663 664 def convert_tokens_to_string(self, tokens: list[str]) -> str:665 return (666 self.backend_tokenizer.decoder.decode(tokens)667 if self.backend_tokenizer.decoder is not None668 else " ".join(tokens)669 )670 671 def _decode(672 self,673 token_ids: Union[int, list[int]],674 skip_special_tokens: bool = False,675 clean_up_tokenization_spaces: Optional[bool] = None,676 **kwargs,677 ) -> str:678 self._decode_use_source_tokenizer = kwargs.pop("use_source_tokenizer", False)679 680 if isinstance(token_ids, int):681 token_ids = [token_ids]682 text = self._tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)683 684 clean_up_tokenization_spaces = (685 clean_up_tokenization_spaces686 if clean_up_tokenization_spaces is not None687 else self.clean_up_tokenization_spaces688 )689 if clean_up_tokenization_spaces:690 clean_text = self.clean_up_tokenization(text)691 return clean_text692 else:693 return text694 695 def _save_pretrained(696 self,697 save_directory: Union[str, os.PathLike],698 file_names: tuple[str, ...],699 legacy_format: Optional[bool] = None,700 filename_prefix: Optional[str] = None,701 ) -> tuple[str, ...]:702 """703 Save a tokenizer using the slow-tokenizer/legacy format: vocabulary + added tokens as well as in a unique JSON704 file containing {config + vocab + added-tokens}.705 """706 save_directory = str(save_directory)707 708 if self.slow_tokenizer_class is None and legacy_format is True:709 raise ValueError(710 "Your tokenizer does not have a legacy version defined and therefore cannot register this version. You"711 " might consider leaving the legacy_format at `None` or setting it to `False`."712 )713 714 save_slow = (715 (legacy_format is None or legacy_format is True)716 and self.slow_tokenizer_class is not None717 and self.can_save_slow_tokenizer718 )719 save_fast = legacy_format is None or legacy_format is False720 721 if save_slow:722 added_tokens_file = os.path.join(723 save_directory, (filename_prefix + "-" if filename_prefix else "") + ADDED_TOKENS_FILE724 )725 # make sure to be forward compatible726 added_vocab = {tok: index for tok, index in self.added_tokens_encoder.items() if index >= self.vocab_size}727 if added_vocab:728 with open(added_tokens_file, "w", encoding="utf-8") as f:729 out_str = json.dumps(added_vocab, indent=2, sort_keys=True, ensure_ascii=False) + "\n"730 f.write(out_str)731 732 vocab_files = self.save_vocabulary(save_directory, filename_prefix=filename_prefix)733 file_names = file_names + vocab_files + (added_tokens_file,)734 735 if save_fast:736 tokenizer_file = os.path.join(737 save_directory, (filename_prefix + "-" if filename_prefix else "") + TOKENIZER_FILE738 )739 self.backend_tokenizer.save(tokenizer_file)740 file_names = file_names + (tokenizer_file,)741 742 return file_names743 744 def train_new_from_iterator(745 self,746 text_iterator,747 vocab_size,748 length=None,749 new_special_tokens=None,750 special_tokens_map=None,751 **kwargs,752 ):753 """754 Trains a tokenizer on a new corpus with the same defaults (in terms of special tokens or tokenization pipeline)755 as the current one.756 757 Args:758 text_iterator (generator of `list[str]`):759 The training corpus. Should be a generator of batches of texts, for instance a list of lists of texts760 if you have everything in memory.761 vocab_size (`int`):762 The size of the vocabulary you want for your tokenizer.763 length (`int`, *optional*):764 The total number of sequences in the iterator. This is used to provide meaningful progress tracking765 new_special_tokens (list of `str` or `AddedToken`, *optional*):766 A list of new special tokens to add to the tokenizer you are training.767 special_tokens_map (`dict[str, str]`, *optional*):768 If you want to rename some of the special tokens this tokenizer uses, pass along a mapping old special769 token name to new special token name in this argument.770 kwargs (`dict[str, Any]`, *optional*):771 Additional keyword arguments passed along to the trainer from the 🤗 Tokenizers library.772 773 Returns:774 [`PreTrainedTokenizerFast`]: A new tokenizer of the same type as the original one, trained on775 `text_iterator`.776 777 """778 tokenizer_json = json.loads(self._tokenizer.to_str())779 # Remove added tokens for now (uses IDs of tokens)780 added_tokens = tokenizer_json.pop("added_tokens")781 # Remove post processor for now (uses IDs of tokens)782 post_processor = tokenizer_json.pop("post_processor")783 784 unk_token = None785 # Remove vocab786 if tokenizer_json["model"]["type"] == "BPE":787 tokenizer_json["model"]["vocab"] = {}788 tokenizer_json["model"]["merges"] = []789 elif tokenizer_json["model"]["type"] == "Unigram":790 if tokenizer_json["model"]["unk_id"] is not None:791 unk_id = tokenizer_json["model"]["unk_id"]792 unk_token = tokenizer_json["model"]["vocab"][unk_id][0]793 if special_tokens_map is not None and unk_token in special_tokens_map:794 unk_token = special_tokens_map[unk_token]795 tokenizer_json["model"]["unk_id"] = 0796 tokenizer_json["model"]["vocab"] = [[unk_token, 0.0]]797 elif tokenizer_json["model"]["type"] in ["WordLevel", "WordPiece"]:798 tokenizer_json["model"]["vocab"] = {}799 else:800 raise ValueError(801 f"This method does not support this type of tokenizer (found {tokenizer_json['model']['type']}) "802 "only BPE, Unigram, WordLevel and WordPiece."803 )804 805 if (806 special_tokens_map is not None807 and "unk_token" in tokenizer_json["model"]808 and tokenizer_json["model"]["unk_token"] in special_tokens_map809 ):810 tokenizer_json["model"]["unk_token"] = special_tokens_map[tokenizer_json["model"]["unk_token"]]811 812 tokenizer = TokenizerFast.from_str(json.dumps(tokenizer_json))813 814 # Get the special tokens from the current tokenizer if none are specified.815 special_tokens = []816 for added_token in added_tokens:817 special = added_token.pop("special", None)818 _ = added_token.pop("id", None)819 if tokenizer_json["model"]["type"] != "Unigram" and not special:820 continue821 if special_tokens_map is not None and added_token["content"] in special_tokens_map:822 added_token["content"] = special_tokens_map[added_token["content"]]823 special_tokens.append(AddedToken(**added_token))824 825 if new_special_tokens is not None:826 special_tokens.extend(new_special_tokens)827 828 # Trainer needs to know the end of word / continuing subword thingies in BPE829 if (830 tokenizer_json["model"]["type"] == "BPE"831 and "continuing_subword_prefix" not in kwargs832 and tokenizer_json["model"]["continuing_subword_prefix"] is not None833 ):834 kwargs["continuing_subword_prefix"] = tokenizer_json["model"]["continuing_subword_prefix"]835 if (836 tokenizer_json["model"]["type"] == "BPE"837 and "end_of_word_suffix" not in kwargs838 and tokenizer_json["model"]["end_of_word_suffix"] is not None839 ):840 kwargs["end_of_word_suffix"] = tokenizer_json["model"]["end_of_word_suffix"]841 if tokenizer_json["model"]["type"] == "Unigram" and unk_token is not None:842 kwargs["unk_token"] = unk_token843 if tokenizer_json["pre_tokenizer"] is not None:844 if (845 tokenizer_json["pre_tokenizer"]["type"] == "ByteLevel"846 or tokenizer_json["pre_tokenizer"]["type"] == "Sequence"847 and "pretokenizers" in tokenizer_json["pre_tokenizer"]848 and any(849 pretokenizer["type"] == "ByteLevel"850 for pretokenizer in tokenizer_json["pre_tokenizer"]["pretokenizers"]851 )852 ):853 kwargs["initial_alphabet"] = pre_tokenizers_fast.ByteLevel.alphabet()854 855 trainer_class = MODEL_TO_TRAINER_MAPPING[tokenizer_json["model"]["type"]]856 trainer = trainer_class(vocab_size=vocab_size, special_tokens=special_tokens, **kwargs)857 tokenizer.train_from_iterator(text_iterator, length=length, trainer=trainer)858 859 if post_processor is not None:860 trained_tokenizer_json = json.loads(tokenizer.to_str())861 # Almost done, we just have to adjust the token IDs in the post processor862 if "special_tokens" in post_processor:863 for key in post_processor["special_tokens"]:864 tokens = post_processor["special_tokens"][key]["tokens"]865 if special_tokens_map is not None:866 tokens = [special_tokens_map.get(token, token) for token in tokens]867 post_processor["special_tokens"][key]["tokens"] = tokens868 for token in tokens:869 token_id = tokenizer.token_to_id(token)870 if token_id is None:871 raise ValueError(872 "Attempted to set a token in the post processor that does not exist in the mapping"873 )874 875 post_processor["special_tokens"][key]["ids"] = [tokenizer.token_to_id(token) for token in tokens]876 877 for special_token in ["cls", "sep"]:878 if special_token in post_processor:879 token, _ = post_processor[special_token]880 if special_tokens_map is not None and token in special_tokens_map:881 token = special_tokens_map[token]882 token_id = tokenizer.token_to_id(token)883 if token_id is None:884 raise ValueError(885 "Attempted to set a token in the post processor that does not exist in the mapping"886 )887 post_processor[special_token] = [token, token_id]888 889 trained_tokenizer_json["post_processor"] = post_processor890 tokenizer = TokenizerFast.from_str(json.dumps(trained_tokenizer_json))891 892 kwargs = self.init_kwargs.copy()893 # Map pad/cls/mask token at the Transformers level894 special_tokens_list = SpecialTokensMixin.SPECIAL_TOKENS_ATTRIBUTES.copy()895 special_tokens_list.remove("additional_special_tokens")896 for token in special_tokens_list:897 if getattr(self, token) is not None:898 special_token = getattr(self, token)899 if special_tokens_map is not None and special_token in special_tokens_map:900 special_token = special_tokens_map[special_token]901 902 special_token_full = self._special_tokens_map.get(token, None)903 if isinstance(special_token_full, AddedToken):904 # Create an added token with the same parameters except the content905 kwargs[token] = AddedToken(906 special_token,907 single_word=special_token_full.single_word,908 lstrip=special_token_full.lstrip,909 rstrip=special_token_full.rstrip,910 normalized=special_token_full.normalized,911 special=True,912 )913 else:914 kwargs[token] = special_token915 916 additional_special_tokens = self.additional_special_tokens917 if new_special_tokens is not None:918 additional_special_tokens.extend(new_special_tokens)919 if len(additional_special_tokens) > 0:920 kwargs["additional_special_tokens"] = additional_special_tokens921 922 return self.__class__(tokenizer_object=tokenizer, **kwargs)923