Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2020 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""16Base classes common to both the slow and the fast tokenization classes: PreTrainedTokenizerBase (host all the user17fronting encoding methods) Special token mixing (host the special tokens logic) and BatchEncoding (wrap the dictionary18of output with special method for the Fast tokenizers)19"""20 21import copy22import json23import os24import re25import warnings26from collections import UserDict27from collections.abc import Mapping, Sequence, Sized28from contextlib import contextmanager29from dataclasses import dataclass30from pathlib import Path31from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union32 33import numpy as np34from huggingface_hub import list_repo_files35from packaging import version36 37from . import __version__38from .dynamic_module_utils import custom_object_save39from .utils import (40 CHAT_TEMPLATE_DIR,41 CHAT_TEMPLATE_FILE,42 ExplicitEnum,43 PaddingStrategy,44 PushToHubMixin,45 TensorType,46 add_end_docstrings,47 cached_file,48 copy_func,49 download_url,50 extract_commit_hash,51 is_flax_available,52 is_jax_tensor,53 is_mlx_available,54 is_numpy_array,55 is_offline_mode,56 is_protobuf_available,57 is_remote_url,58 is_tf_available,59 is_tf_tensor,60 is_tokenizers_available,61 is_torch_available,62 is_torch_device,63 is_torch_tensor,64 list_repo_templates,65 logging,66 requires_backends,67 to_py_obj,68)69from .utils.chat_template_utils import render_jinja_template70from .utils.import_utils import PROTOBUF_IMPORT_ERROR71 72 73if TYPE_CHECKING:74 if is_torch_available():75 import torch76 if is_tf_available():77 import tensorflow as tf78 if is_flax_available():79 import jax.numpy as jnp # noqa: F40180 81 82def import_protobuf_decode_error(error_message=""):83 if is_protobuf_available():84 from google.protobuf.message import DecodeError85 86 return DecodeError87 else:88 raise ImportError(PROTOBUF_IMPORT_ERROR.format(error_message))89 90 91def flatten(arr: list):92 res = []93 if len(arr) > 0:94 for sub_arr in arr:95 if isinstance(arr[0], (list, tuple)):96 res.extend(flatten(sub_arr))97 else:98 res.append(sub_arr)99 return res100 101 102if is_tokenizers_available() or TYPE_CHECKING:103 from tokenizers import Encoding as EncodingFast104 105if is_tokenizers_available():106 from tokenizers import AddedToken107else:108 109 @dataclass(frozen=False, eq=True)110 class AddedToken:111 """112 AddedToken represents a token to be added to a Tokenizer An AddedToken can have special options defining the113 way it should behave.114 115 The `normalized` will default to `not special` if it is not specified, similarly to the definition in116 `tokenizers`.117 """118 119 def __init__(120 self, content: str, single_word=False, lstrip=False, rstrip=False, special=False, normalized=None121 ):122 self.content = content123 self.single_word = single_word124 self.lstrip = lstrip125 self.rstrip = rstrip126 self.special = special127 self.normalized = normalized if normalized is not None else not special128 129 def __getstate__(self):130 return self.__dict__131 132 def __str__(self):133 return self.content134 135 136logger = logging.get_logger(__name__)137 138VERY_LARGE_INTEGER = int(1e30) # This is used to set the max input length for a model with infinite size input139LARGE_INTEGER = int(1e20) # This is used when we need something big but slightly smaller than VERY_LARGE_INTEGER140 141# Define type aliases and NamedTuples142TextInput = str143PreTokenizedInput = list[str]144EncodedInput = list[int]145TextInputPair = tuple[str, str]146PreTokenizedInputPair = tuple[list[str], list[str]]147EncodedInputPair = tuple[list[int], list[int]]148 149# Define type aliases for text-related non-text modalities150AudioInput = Union[np.ndarray, "torch.Tensor", list[np.ndarray], list["torch.Tensor"]]151 152# Slow tokenizers used to be saved in three separated files153SPECIAL_TOKENS_MAP_FILE = "special_tokens_map.json"154ADDED_TOKENS_FILE = "added_tokens.json"155TOKENIZER_CONFIG_FILE = "tokenizer_config.json"156 157# Fast tokenizers (provided by HuggingFace tokenizer's library) can be saved in a single file158FULL_TOKENIZER_FILE = "tokenizer.json"159_re_tokenizer_file = re.compile(r"tokenizer\.(.*)\.json")160 161 162class TruncationStrategy(ExplicitEnum):163 """164 Possible values for the `truncation` argument in [`PreTrainedTokenizerBase.__call__`]. Useful for tab-completion in165 an IDE.166 """167 168 ONLY_FIRST = "only_first"169 ONLY_SECOND = "only_second"170 LONGEST_FIRST = "longest_first"171 DO_NOT_TRUNCATE = "do_not_truncate"172 173 174class CharSpan(NamedTuple):175 """176 Character span in the original string.177 178 Args:179 start (`int`): Index of the first character in the original string.180 end (`int`): Index of the character following the last character in the original string.181 """182 183 start: int184 end: int185 186 187class TokenSpan(NamedTuple):188 """189 Token span in an encoded string (list of tokens).190 191 Args:192 start (`int`): Index of the first token in the span.193 end (`int`): Index of the token following the last token in the span.194 """195 196 start: int197 end: int198 199 200class BatchEncoding(UserDict):201 """202 Holds the output of the [`~tokenization_utils_base.PreTrainedTokenizerBase.__call__`],203 [`~tokenization_utils_base.PreTrainedTokenizerBase.encode_plus`] and204 [`~tokenization_utils_base.PreTrainedTokenizerBase.batch_encode_plus`] methods (tokens, attention_masks, etc).205 206 This class is derived from a python dictionary and can be used as a dictionary. In addition, this class exposes207 utility methods to map from word/character space to token space.208 209 Args:210 data (`dict`, *optional*):211 Dictionary of lists/arrays/tensors returned by the `__call__`/`encode_plus`/`batch_encode_plus` methods212 ('input_ids', 'attention_mask', etc.).213 encoding (`tokenizers.Encoding` or `Sequence[tokenizers.Encoding]`, *optional*):214 If the tokenizer is a fast tokenizer which outputs additional information like mapping from word/character215 space to token space the `tokenizers.Encoding` instance or list of instance (for batches) hold this216 information.217 tensor_type (`Union[None, str, TensorType]`, *optional*):218 You can give a tensor_type here to convert the lists of integers in PyTorch/TensorFlow/Numpy Tensors at219 initialization.220 prepend_batch_axis (`bool`, *optional*, defaults to `False`):221 Whether or not to add a batch axis when converting to tensors (see `tensor_type` above). Note that this222 parameter has an effect if the parameter `tensor_type` is set, *otherwise has no effect*.223 n_sequences (`Optional[int]`, *optional*):224 You can give a tensor_type here to convert the lists of integers in PyTorch/TensorFlow/Numpy Tensors at225 initialization.226 """227 228 def __init__(229 self,230 data: Optional[dict[str, Any]] = None,231 encoding: Optional[Union[EncodingFast, Sequence[EncodingFast]]] = None,232 tensor_type: Union[None, str, TensorType] = None,233 prepend_batch_axis: bool = False,234 n_sequences: Optional[int] = None,235 ):236 super().__init__(data)237 238 # If encoding is not None, the fast tokenization is used239 if encoding is not None and isinstance(encoding, EncodingFast):240 encoding = [encoding]241 242 self._encodings = encoding243 244 if n_sequences is None and encoding is not None and encoding:245 n_sequences = encoding[0].n_sequences246 247 self._n_sequences = n_sequences248 249 self.convert_to_tensors(tensor_type=tensor_type, prepend_batch_axis=prepend_batch_axis)250 251 @property252 def n_sequences(self) -> Optional[int]:253 """254 `Optional[int]`: The number of sequences used to generate each sample from the batch encoded in this255 [`BatchEncoding`]. Currently can be one of `None` (unknown), `1` (a single sentence) or `2` (a pair of256 sentences)257 """258 return self._n_sequences259 260 @property261 def is_fast(self) -> bool:262 """263 `bool`: Indicate whether this [`BatchEncoding`] was generated from the result of a [`PreTrainedTokenizerFast`]264 or not.265 """266 return self._encodings is not None267 268 def __getitem__(self, item: Union[int, str]) -> Union[Any, EncodingFast]:269 """270 If the key is a string, returns the value of the dict associated to `key` ('input_ids', 'attention_mask',271 etc.).272 273 If the key is an integer, get the `tokenizers.Encoding` for batch item with index `key`.274 275 If the key is a slice, returns the value of the dict associated to `key` ('input_ids', 'attention_mask', etc.)276 with the constraint of slice.277 """278 if isinstance(item, str):279 return self.data[item]280 elif self._encodings is not None:281 return self._encodings[item]282 elif isinstance(item, slice):283 return {key: self.data[key][item] for key in self.data}284 else:285 raise KeyError(286 "Invalid key. Only three types of key are available: "287 "(1) string, (2) integers for backend Encoding, and (3) slices for data subsetting."288 )289 290 def __getattr__(self, item: str):291 try:292 return self.data[item]293 except KeyError:294 raise AttributeError295 296 def __getstate__(self):297 return {"data": self.data, "encodings": self._encodings}298 299 def __setstate__(self, state):300 if "data" in state:301 self.data = state["data"]302 303 if "encodings" in state:304 self._encodings = state["encodings"]305 306 # After this point:307 # Extended properties and methods only available for fast (Rust-based) tokenizers308 # provided by HuggingFace tokenizers library.309 310 @property311 def encodings(self) -> Optional[list[EncodingFast]]:312 """313 `Optional[list[tokenizers.Encoding]]`: The list all encodings from the tokenization process. Returns `None` if314 the input was tokenized through Python (i.e., not a fast) tokenizer.315 """316 return self._encodings317 318 def tokens(self, batch_index: int = 0) -> list[str]:319 """320 Return the list of tokens (sub-parts of the input strings after word/subword splitting and before conversion to321 integer indices) at a given batch index (only works for the output of a fast tokenizer).322 323 Args:324 batch_index (`int`, *optional*, defaults to 0): The index to access in the batch.325 326 Returns:327 `list[str]`: The list of tokens at that index.328 """329 if not self._encodings:330 raise ValueError(331 "tokens() is not available when using non-fast tokenizers (e.g. instance of a `XxxTokenizerFast`"332 " class)."333 )334 return self._encodings[batch_index].tokens335 336 def sequence_ids(self, batch_index: int = 0) -> list[Optional[int]]:337 """338 Return a list mapping the tokens to the id of their original sentences:339 340 - `None` for special tokens added around or between sequences,341 - `0` for tokens corresponding to words in the first sequence,342 - `1` for tokens corresponding to words in the second sequence when a pair of sequences was jointly343 encoded.344 345 Args:346 batch_index (`int`, *optional*, defaults to 0): The index to access in the batch.347 348 Returns:349 `list[Optional[int]]`: A list indicating the sequence id corresponding to each token. Special tokens added350 by the tokenizer are mapped to `None` and other tokens are mapped to the index of their corresponding351 sequence.352 """353 if not self._encodings:354 raise ValueError(355 "sequence_ids() is not available when using non-fast tokenizers (e.g. instance of a `XxxTokenizerFast`"356 " class)."357 )358 return self._encodings[batch_index].sequence_ids359 360 def words(self, batch_index: int = 0) -> list[Optional[int]]:361 """362 Return a list mapping the tokens to their actual word in the initial sentence for a fast tokenizer.363 364 Args:365 batch_index (`int`, *optional*, defaults to 0): The index to access in the batch.366 367 Returns:368 `list[Optional[int]]`: A list indicating the word corresponding to each token. Special tokens added by the369 tokenizer are mapped to `None` and other tokens are mapped to the index of their corresponding word370 (several tokens will be mapped to the same word index if they are parts of that word).371 """372 if not self._encodings:373 raise ValueError(374 "words() is not available when using non-fast tokenizers (e.g. instance of a `XxxTokenizerFast`"375 " class)."376 )377 warnings.warn(378 "`BatchEncoding.words()` property is deprecated and should be replaced with the identical, "379 "but more self-explanatory `BatchEncoding.word_ids()` property.",380 FutureWarning,381 )382 return self.word_ids(batch_index)383 384 def word_ids(self, batch_index: int = 0) -> list[Optional[int]]:385 """386 Return a list mapping the tokens to their actual word in the initial sentence for a fast tokenizer.387 388 Args:389 batch_index (`int`, *optional*, defaults to 0): The index to access in the batch.390 391 Returns:392 `list[Optional[int]]`: A list indicating the word corresponding to each token. Special tokens added by the393 tokenizer are mapped to `None` and other tokens are mapped to the index of their corresponding word394 (several tokens will be mapped to the same word index if they are parts of that word).395 """396 if not self._encodings:397 raise ValueError(398 "word_ids() is not available when using non-fast tokenizers (e.g. instance of a `XxxTokenizerFast`"399 " class)."400 )401 return self._encodings[batch_index].word_ids402 403 def token_to_sequence(self, batch_or_token_index: int, token_index: Optional[int] = None) -> int:404 """405 Get the index of the sequence represented by the given token. In the general use case, this method returns `0`406 for a single sequence or the first sequence of a pair, and `1` for the second sequence of a pair407 408 Can be called as:409 410 - `self.token_to_sequence(token_index)` if batch size is 1411 - `self.token_to_sequence(batch_index, token_index)` if batch size is greater than 1412 413 This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e.,414 words are defined by the user). In this case it allows to easily associate encoded tokens with provided415 tokenized words.416 417 Args:418 batch_or_token_index (`int`):419 Index of the sequence in the batch. If the batch only comprises one sequence, this can be the index of420 the token in the sequence.421 token_index (`int`, *optional*):422 If a batch index is provided in *batch_or_token_index*, this can be the index of the token in the423 sequence.424 425 Returns:426 `int`: Index of the word in the input sequence.427 """428 429 if not self._encodings:430 raise ValueError("token_to_sequence() is not available when using Python based tokenizers")431 if token_index is not None:432 batch_index = batch_or_token_index433 else:434 batch_index = 0435 token_index = batch_or_token_index436 if batch_index < 0:437 batch_index = self._batch_size + batch_index438 if token_index < 0:439 token_index = self._seq_len + token_index440 return self._encodings[batch_index].token_to_sequence(token_index)441 442 def token_to_word(self, batch_or_token_index: int, token_index: Optional[int] = None) -> int:443 """444 Get the index of the word corresponding (i.e. comprising) to an encoded token in a sequence of the batch.445 446 Can be called as:447 448 - `self.token_to_word(token_index)` if batch size is 1449 - `self.token_to_word(batch_index, token_index)` if batch size is greater than 1450 451 This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e.,452 words are defined by the user). In this case it allows to easily associate encoded tokens with provided453 tokenized words.454 455 Args:456 batch_or_token_index (`int`):457 Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of458 the token in the sequence.459 token_index (`int`, *optional*):460 If a batch index is provided in *batch_or_token_index*, this can be the index of the token in the461 sequence.462 463 Returns:464 `int`: Index of the word in the input sequence.465 """466 467 if not self._encodings:468 raise ValueError("token_to_word() is not available when using Python based tokenizers")469 if token_index is not None:470 batch_index = batch_or_token_index471 else:472 batch_index = 0473 token_index = batch_or_token_index474 if batch_index < 0:475 batch_index = self._batch_size + batch_index476 if token_index < 0:477 token_index = self._seq_len + token_index478 return self._encodings[batch_index].token_to_word(token_index)479 480 def word_to_tokens(481 self, batch_or_word_index: int, word_index: Optional[int] = None, sequence_index: int = 0482 ) -> Optional[TokenSpan]:483 """484 Get the encoded token span corresponding to a word in a sequence of the batch.485 486 Token spans are returned as a [`~tokenization_utils_base.TokenSpan`] with:487 488 - **start** -- Index of the first token.489 - **end** -- Index of the token following the last token.490 491 Can be called as:492 493 - `self.word_to_tokens(word_index, sequence_index: int = 0)` if batch size is 1494 - `self.word_to_tokens(batch_index, word_index, sequence_index: int = 0)` if batch size is greater or equal to495 1496 497 This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e. words498 are defined by the user). In this case it allows to easily associate encoded tokens with provided tokenized499 words.500 501 Args:502 batch_or_word_index (`int`):503 Index of the sequence in the batch. If the batch only comprises one sequence, this can be the index of504 the word in the sequence.505 word_index (`int`, *optional*):506 If a batch index is provided in *batch_or_token_index*, this can be the index of the word in the507 sequence.508 sequence_index (`int`, *optional*, defaults to 0):509 If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0510 or 1) the provided word index belongs to.511 512 Returns:513 ([`~tokenization_utils_base.TokenSpan`], *optional*): Span of tokens in the encoded sequence. Returns514 `None` if no tokens correspond to the word. This can happen especially when the token is a special token515 that has been used to format the tokenization. For example when we add a class token at the very beginning516 of the tokenization.517 """518 519 if not self._encodings:520 raise ValueError("word_to_tokens() is not available when using Python based tokenizers")521 if word_index is not None:522 batch_index = batch_or_word_index523 else:524 batch_index = 0525 word_index = batch_or_word_index526 if batch_index < 0:527 batch_index = self._batch_size + batch_index528 if word_index < 0:529 word_index = self._seq_len + word_index530 span = self._encodings[batch_index].word_to_tokens(word_index, sequence_index)531 return TokenSpan(*span) if span is not None else None532 533 def token_to_chars(self, batch_or_token_index: int, token_index: Optional[int] = None) -> Optional[CharSpan]:534 """535 Get the character span corresponding to an encoded token in a sequence of the batch.536 537 Character spans are returned as a [`~tokenization_utils_base.CharSpan`] with:538 539 - **start** -- Index of the first character in the original string associated to the token.540 - **end** -- Index of the character following the last character in the original string associated to the541 token.542 543 Can be called as:544 545 - `self.token_to_chars(token_index)` if batch size is 1546 - `self.token_to_chars(batch_index, token_index)` if batch size is greater or equal to 1547 548 Args:549 batch_or_token_index (`int`):550 Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of551 the token in the sequence.552 token_index (`int`, *optional*):553 If a batch index is provided in *batch_or_token_index*, this can be the index of the token or tokens in554 the sequence.555 556 Returns:557 [`~tokenization_utils_base.CharSpan`]: Span of characters in the original string, or None, if the token558 (e.g. <s>, </s>) doesn't correspond to any chars in the origin string.559 """560 561 if not self._encodings:562 raise ValueError("token_to_chars() is not available when using Python based tokenizers")563 if token_index is not None:564 batch_index = batch_or_token_index565 else:566 batch_index = 0567 token_index = batch_or_token_index568 span_indices = self._encodings[batch_index].token_to_chars(token_index)569 570 return CharSpan(*span_indices) if span_indices is not None else None571 572 def char_to_token(573 self, batch_or_char_index: int, char_index: Optional[int] = None, sequence_index: int = 0574 ) -> int:575 """576 Get the index of the token in the encoded output comprising a character in the original string for a sequence577 of the batch.578 579 Can be called as:580 581 - `self.char_to_token(char_index)` if batch size is 1582 - `self.char_to_token(batch_index, char_index)` if batch size is greater or equal to 1583 584 This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e. words585 are defined by the user). In this case it allows to easily associate encoded tokens with provided tokenized586 words.587 588 Args:589 batch_or_char_index (`int`):590 Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of591 the word in the sequence592 char_index (`int`, *optional*):593 If a batch index is provided in *batch_or_token_index*, this can be the index of the word in the594 sequence.595 sequence_index (`int`, *optional*, defaults to 0):596 If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0597 or 1) the provided character index belongs to.598 599 600 Returns:601 `int`: Index of the token, or None if the char index refers to a whitespace only token and whitespace is602 trimmed with `trim_offsets=True`.603 """604 605 if not self._encodings:606 raise ValueError("char_to_token() is not available when using Python based tokenizers")607 if char_index is not None:608 batch_index = batch_or_char_index609 else:610 batch_index = 0611 char_index = batch_or_char_index612 return self._encodings[batch_index].char_to_token(char_index, sequence_index)613 614 def word_to_chars(615 self, batch_or_word_index: int, word_index: Optional[int] = None, sequence_index: int = 0616 ) -> CharSpan:617 """618 Get the character span in the original string corresponding to given word in a sequence of the batch.619 620 Character spans are returned as a CharSpan NamedTuple with:621 622 - start: index of the first character in the original string623 - end: index of the character following the last character in the original string624 625 Can be called as:626 627 - `self.word_to_chars(word_index)` if batch size is 1628 - `self.word_to_chars(batch_index, word_index)` if batch size is greater or equal to 1629 630 Args:631 batch_or_word_index (`int`):632 Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of633 the word in the sequence634 word_index (`int`, *optional*):635 If a batch index is provided in *batch_or_token_index*, this can be the index of the word in the636 sequence.637 sequence_index (`int`, *optional*, defaults to 0):638 If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0639 or 1) the provided word index belongs to.640 641 Returns:642 `CharSpan` or `list[CharSpan]`: Span(s) of the associated character or characters in the string. CharSpan643 are NamedTuple with:644 645 - start: index of the first character associated to the token in the original string646 - end: index of the character following the last character associated to the token in the original647 string648 """649 650 if not self._encodings:651 raise ValueError("word_to_chars() is not available when using Python based tokenizers")652 if word_index is not None:653 batch_index = batch_or_word_index654 else:655 batch_index = 0656 word_index = batch_or_word_index657 return CharSpan(*(self._encodings[batch_index].word_to_chars(word_index, sequence_index)))658 659 def char_to_word(self, batch_or_char_index: int, char_index: Optional[int] = None, sequence_index: int = 0) -> int:660 """661 Get the word in the original string corresponding to a character in the original string of a sequence of the662 batch.663 664 Can be called as:665 666 - `self.char_to_word(char_index)` if batch size is 1667 - `self.char_to_word(batch_index, char_index)` if batch size is greater than 1668 669 This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e. words670 are defined by the user). In this case it allows to easily associate encoded tokens with provided tokenized671 words.672 673 Args:674 batch_or_char_index (`int`):675 Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of676 the character in the original string.677 char_index (`int`, *optional*):678 If a batch index is provided in *batch_or_token_index*, this can be the index of the character in the679 original string.680 sequence_index (`int`, *optional*, defaults to 0):681 If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0682 or 1) the provided character index belongs to.683 684 685 Returns:686 `int` or `list[int]`: Index or indices of the associated encoded token(s).687 """688 689 if not self._encodings:690 raise ValueError("char_to_word() is not available when using Python based tokenizers")691 if char_index is not None:692 batch_index = batch_or_char_index693 else:694 batch_index = 0695 char_index = batch_or_char_index696 return self._encodings[batch_index].char_to_word(char_index, sequence_index)697 698 def convert_to_tensors(699 self, tensor_type: Optional[Union[str, TensorType]] = None, prepend_batch_axis: bool = False700 ):701 """702 Convert the inner content to tensors.703 704 Args:705 tensor_type (`str` or [`~utils.TensorType`], *optional*):706 The type of tensors to use. If `str`, should be one of the values of the enum [`~utils.TensorType`]. If707 `None`, no modification is done.708 prepend_batch_axis (`int`, *optional*, defaults to `False`):709 Whether or not to add the batch dimension during the conversion.710 """711 if tensor_type is None:712 return self713 714 # Convert to TensorType715 if not isinstance(tensor_type, TensorType):716 tensor_type = TensorType(tensor_type)717 718 # Get a function reference for the correct framework719 if tensor_type == TensorType.TENSORFLOW:720 if not is_tf_available():721 raise ImportError(722 "Unable to convert output to TensorFlow tensors format, TensorFlow is not installed."723 )724 import tensorflow as tf725 726 def as_tensor(value, dtype=None):727 if len(flatten(value)) == 0 and dtype is None:728 dtype = tf.int32729 return tf.constant(value, dtype=dtype)730 731 is_tensor = tf.is_tensor732 733 elif tensor_type == TensorType.PYTORCH:734 if not is_torch_available():735 raise ImportError("Unable to convert output to PyTorch tensors format, PyTorch is not installed.")736 import torch737 738 def as_tensor(value, dtype=None):739 if isinstance(value, list) and len(value) > 0 and isinstance(value[0], np.ndarray):740 return torch.from_numpy(np.array(value))741 if len(flatten(value)) == 0 and dtype is None:742 dtype = torch.int64743 return torch.tensor(value, dtype=dtype)744 745 is_tensor = torch.is_tensor746 747 elif tensor_type == TensorType.JAX:748 if not is_flax_available():749 raise ImportError("Unable to convert output to JAX tensors format, JAX is not installed.")750 import jax.numpy as jnp # noqa: F811751 752 def as_tensor(value, dtype=None):753 if len(flatten(value)) == 0 and dtype is None:754 dtype = jnp.int32755 return jnp.array(value, dtype=dtype)756 757 is_tensor = is_jax_tensor758 759 elif tensor_type == TensorType.MLX:760 if not is_mlx_available():761 raise ImportError("Unable to convert output to MLX tensors format, MLX is not installed.")762 import mlx.core as mx763 764 def as_tensor(value, dtype=None):765 if len(flatten(value)) == 0 and dtype is None:766 dtype = mx.int32767 return mx.array(value, dtype=dtype)768 769 def is_tensor(obj):770 return isinstance(obj, mx.array)771 else:772 773 def as_tensor(value, dtype=None):774 if (775 isinstance(value, (list, tuple))776 and len(value) > 0777 and isinstance(value[0], (list, tuple, np.ndarray))778 ):779 value_lens = [len(val) for val in value]780 if len(set(value_lens)) > 1 and dtype is None:781 # we have a ragged list so handle explicitly782 value = as_tensor([np.asarray(val) for val in value], dtype=object)783 if len(flatten(value)) == 0 and dtype is None:784 dtype = np.int64785 return np.asarray(value, dtype=dtype)786 787 is_tensor = is_numpy_array788 789 # Do the tensor conversion in batch790 for key, value in self.items():791 try:792 if prepend_batch_axis:793 value = [value]794 795 if not is_tensor(value):796 tensor = as_tensor(value)797 798 # Removing this for now in favor of controlling the shape with `prepend_batch_axis`799 # # at-least2d800 # if tensor.ndim > 2:801 # tensor = tensor.squeeze(0)802 # elif tensor.ndim < 2:803 # tensor = tensor[None, :]804 805 self[key] = tensor806 except Exception as e:807 if key == "overflowing_tokens":808 raise ValueError(809 "Unable to create tensor returning overflowing tokens of different lengths. "810 "Please see if a fast version of this tokenizer is available to have this feature available."811 ) from e812 raise ValueError(813 "Unable to create tensor, you should probably activate truncation and/or padding with"814 " 'padding=True' 'truncation=True' to have batched tensors with the same length. Perhaps your"815 f" features (`{key}` in this case) have excessive nesting (inputs type `list` where type `int` is"816 " expected)."817 ) from e818 819 return self820 821 def to(self, device: Union[str, "torch.device"], *, non_blocking: bool = False) -> "BatchEncoding":822 """823 Send all values to device by calling `v.to(device, non_blocking=non_blocking)` (PyTorch only).824 825 Args:826 device (`str` or `torch.device`): The device to put the tensors on.827 non_blocking (`bool`): Whether to perform the copy asynchronously.828 829 Returns:830 [`BatchEncoding`]: The same instance after modification.831 """832 requires_backends(self, ["torch"])833 834 # This check catches things like APEX blindly calling "to" on all inputs to a module835 # Otherwise it passes the casts down and casts the LongTensor containing the token idxs836 # into a HalfTensor837 if isinstance(device, str) or is_torch_device(device) or isinstance(device, int):838 self.data = {839 k: v.to(device=device, non_blocking=non_blocking) if hasattr(v, "to") and callable(v.to) else v840 for k, v in self.data.items()841 }842 else:843 logger.warning(f"Attempting to cast a BatchEncoding to type {str(device)}. This is not supported.")844 return self845 846 847class SpecialTokensMixin:848 """849 A mixin derived by [`PreTrainedTokenizer`] and [`PreTrainedTokenizerFast`] to handle specific behaviors related to850 special tokens. In particular, this class hold the attributes which can be used to directly access these special851 tokens in a model-independent manner and allow to set and update the special tokens.852 853 Args:854 bos_token (`str` or `tokenizers.AddedToken`, *optional*):855 A special token representing the beginning of a sentence.856 eos_token (`str` or `tokenizers.AddedToken`, *optional*):857 A special token representing the end of a sentence.858 unk_token (`str` or `tokenizers.AddedToken`, *optional*):859 A special token representing an out-of-vocabulary token.860 sep_token (`str` or `tokenizers.AddedToken`, *optional*):861 A special token separating two different sentences in the same input (used by BERT for instance).862 pad_token (`str` or `tokenizers.AddedToken`, *optional*):863 A special token used to make arrays of tokens the same size for batching purpose. Will then be ignored by864 attention mechanisms or loss computation.865 cls_token (`str` or `tokenizers.AddedToken`, *optional*):866 A special token representing the class of the input (used by BERT for instance).867 mask_token (`str` or `tokenizers.AddedToken`, *optional*):868 A special token representing a masked token (used by masked-language modeling pretraining objectives, like869 BERT).870 additional_special_tokens (tuple or list of `str` or `tokenizers.AddedToken`, *optional*):871 A tuple or a list of additional tokens, which will be marked as `special`, meaning that they will be872 skipped when decoding if `skip_special_tokens` is set to `True`.873 """874 875 SPECIAL_TOKENS_ATTRIBUTES = [876 "bos_token",877 "eos_token",878 "unk_token",879 "sep_token",880 "pad_token",881 "cls_token",882 "mask_token",883 "additional_special_tokens",884 ]885 886 def __init__(self, verbose=False, **kwargs):887 self._pad_token_type_id = 0888 self.verbose = verbose889 self._special_tokens_map = dict.fromkeys(self.SPECIAL_TOKENS_ATTRIBUTES)890 self._special_tokens_map["additional_special_tokens"] = [] # for BC where it defaults to empty list891 892 # We directly set the hidden value to allow initialization with special tokens893 # which are not yet in the vocabulary. Necessary for serialization/de-serialization894 # TODO clean this up at some point (probably by switching to fast tokenizers)895 896 for key, value in kwargs.items():897 if value is None:898 continue899 if key in self.SPECIAL_TOKENS_ATTRIBUTES:900 if key == "additional_special_tokens":901 assert isinstance(value, (list, tuple)), f"Value {value} is not a list or tuple"902 assert all(isinstance(t, (str, AddedToken)) for t in value), (903 "One of the tokens is not a string or an AddedToken"904 )905 setattr(self, key, value)906 elif isinstance(value, (str, AddedToken)):907 setattr(self, key, value)908 else:909 raise TypeError(f"Special token {key} has to be either str or AddedToken but got: {type(value)}")910 911 def sanitize_special_tokens(self) -> int:912 """913 The `sanitize_special_tokens` is now deprecated kept for backward compatibility and will be removed in914 transformers v5.915 """916 logger.warning_once("The `sanitize_special_tokens` will be removed in transformers v5.")917 return self.add_tokens(self.all_special_tokens_extended, special_tokens=True)918 919 def add_special_tokens(920 self,921 special_tokens_dict: dict[str, Union[str, AddedToken, Sequence[Union[str, AddedToken]]]],922 replace_additional_special_tokens=True,923 ) -> int:924 """925 Add a dictionary of special tokens (eos, pad, cls, etc.) to the encoder and link them to class attributes. If926 special tokens are NOT in the vocabulary, they are added to it (indexed starting from the last index of the927 current vocabulary).928 929 When adding new tokens to the vocabulary, you should make sure to also resize the token embedding matrix of the930 model so that its embedding matrix matches the tokenizer.931 932 In order to do that, please use the [`~PreTrainedModel.resize_token_embeddings`] method.933 934 Using `add_special_tokens` will ensure your special tokens can be used in several ways:935 936 - Special tokens can be skipped when decoding using `skip_special_tokens = True`.937 - Special tokens are carefully handled by the tokenizer (they are never split), similar to `AddedTokens`.938 - You can easily refer to special tokens using tokenizer class attributes like `tokenizer.cls_token`. This939 makes it easy to develop model-agnostic training and fine-tuning scripts.940 941 When possible, special tokens are already registered for provided pretrained models (for instance942 [`BertTokenizer`] `cls_token` is already registered to be `'[CLS]'` and XLM's one is also registered to be943 `'</s>'`).944 945 Args:946 special_tokens_dict (dictionary *str* to *str*, `tokenizers.AddedToken`, or `Sequence[Union[str, AddedToken]]`):947 Keys should be in the list of predefined special attributes: [`bos_token`, `eos_token`, `unk_token`,948 `sep_token`, `pad_token`, `cls_token`, `mask_token`, `additional_special_tokens`].949 950 Tokens are only added if they are not already in the vocabulary (tested by checking if the tokenizer951 assign the index of the `unk_token` to them).952 replace_additional_special_tokens (`bool`, *optional*,, defaults to `True`):953 If `True`, the existing list of additional special tokens will be replaced by the list provided in954 `special_tokens_dict`. Otherwise, `self._special_tokens_map["additional_special_tokens"]` is just extended. In the former955 case, the tokens will NOT be removed from the tokenizer's full vocabulary - they are only being flagged956 as non-special tokens. Remember, this only affects which tokens are skipped during decoding, not the957 `added_tokens_encoder` and `added_tokens_decoder`. This means that the previous958 `additional_special_tokens` are still added tokens, and will not be split by the model.959 960 Returns:961 `int`: Number of tokens added to the vocabulary.962 963 Examples:964 965 ```python966 # Let's see how to add a new classification token to GPT-2967 tokenizer = GPT2Tokenizer.from_pretrained("openai-community/gpt2")968 model = GPT2Model.from_pretrained("openai-community/gpt2")969 970 special_tokens_dict = {"cls_token": "<CLS>"}971 972 num_added_toks = tokenizer.add_special_tokens(special_tokens_dict)973 print("We have added", num_added_toks, "tokens")974 # Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e., the length of the tokenizer.975 model.resize_token_embeddings(len(tokenizer))976 977 assert tokenizer.cls_token == "<CLS>"978 ```"""979 if not special_tokens_dict:980 return 0981 982 added_tokens = []983 for key, value in special_tokens_dict.items():984 assert key in self.SPECIAL_TOKENS_ATTRIBUTES, f"Key {key} is not a special token"985 986 if self.verbose:987 logger.info(f"Assigning {value} to the {key} key of the tokenizer")988 989 if key == "additional_special_tokens":990 assert isinstance(value, (list, tuple)) and all(isinstance(t, (str, AddedToken)) for t in value), (991 f"Tokens {value} for key {key} should all be str or AddedToken instances"992 )993 994 to_add = []995 for token in value:996 if isinstance(token, str):997 # for legacy purpose we default to stripping. `test_add_tokens_tokenizer` depends on this998 token = AddedToken(token, rstrip=False, lstrip=False, normalized=False, special=True)999 if not replace_additional_special_tokens and str(token) in self.additional_special_tokens:1000 continue1001 to_add.append(token)1002 if replace_additional_special_tokens and len(to_add) > 0:1003 setattr(self, key, list(to_add))1004 else:1005 self._special_tokens_map["additional_special_tokens"].extend(to_add)1006 added_tokens += to_add1007 1008 else:1009 if not isinstance(value, (str, AddedToken)):1010 raise ValueError(f"Token {value} for key {key} should be a str or an AddedToken instance")1011 if isinstance(value, (str)):1012 # for legacy purpose we default to stripping. `False` depends on this1013 value = AddedToken(value, rstrip=False, lstrip=False, normalized=False, special=True)1014 if isinstance(value, AddedToken):1015 setattr(self, key, value)1016 if value not in added_tokens:1017 added_tokens.append(value)1018 1019 # if we are adding tokens that were not part of the vocab, we ought to add them1020 added_tokens = self.add_tokens(added_tokens, special_tokens=True)1021 return added_tokens1022 1023 def add_tokens(1024 self, new_tokens: Union[str, AddedToken, Sequence[Union[str, AddedToken]]], special_tokens: bool = False1025 ) -> int:1026 """1027 Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to1028 it with indices starting from length of the current vocabulary and will be isolated before the tokenization1029 algorithm is applied. Added tokens and tokens from the vocabulary of the tokenization algorithm are therefore1030 not treated in the same way.1031 1032 Note, when adding new tokens to the vocabulary, you should make sure to also resize the token embedding matrix1033 of the model so that its embedding matrix matches the tokenizer.1034 1035 In order to do that, please use the [`~PreTrainedModel.resize_token_embeddings`] method.1036 1037 Args:1038 new_tokens (`str`, `tokenizers.AddedToken` or a sequence of *str* or `tokenizers.AddedToken`):1039 Tokens are only added if they are not already in the vocabulary. `tokenizers.AddedToken` wraps a string1040 token to let you personalize its behavior: whether this token should only match against a single word,1041 whether this token should strip all potential whitespaces on the left side, whether this token should1042 strip all potential whitespaces on the right side, etc.1043 special_tokens (`bool`, *optional*, defaults to `False`):1044 Can be used to specify if the token is a special token. This mostly change the normalization behavior1045 (special tokens like CLS or [MASK] are usually not lower-cased for instance).1046 1047 See details for `tokenizers.AddedToken` in HuggingFace tokenizers library.1048 1049 Returns:1050 `int`: Number of tokens added to the vocabulary.1051 1052 Examples:1053 1054 ```python1055 # Let's see how to increase the vocabulary of Bert model and tokenizer1056 tokenizer = BertTokenizerFast.from_pretrained("google-bert/bert-base-uncased")1057 model = BertModel.from_pretrained("google-bert/bert-base-uncased")1058 1059 num_added_toks = tokenizer.add_tokens(["new_tok1", "my_new-tok2"])1060 print("We have added", num_added_toks, "tokens")1061 # Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e., the length of the tokenizer.1062 model.resize_token_embeddings(len(tokenizer))1063 ```"""1064 if not new_tokens:1065 return 01066 1067 if not isinstance(new_tokens, (list, tuple)):1068 new_tokens = [new_tokens]1069 1070 return self._add_tokens(new_tokens, special_tokens=special_tokens)1071 1072 def _add_tokens(self, new_tokens: Union[list[str], list[AddedToken]], special_tokens: bool = False) -> int:1073 raise NotImplementedError1074 1075 @property1076 def pad_token_type_id(self) -> int:1077 """1078 `int`: Id of the padding token type in the vocabulary.1079 """1080 return self._pad_token_type_id1081 1082 def __setattr__(self, key, value):1083 key_without_id = key1084 key_is_special_id = key.endswith("_id") or key.endswith("_ids")1085 if key_is_special_id:1086 key_without_id = key[:-3] if not key.endswith("_ids") else key[:-4]1087 1088 if self.__dict__.get("_special_tokens_map", None) is not None and any(1089 name in self.__dict__["_special_tokens_map"] for name in [key, key_without_id]1090 ):1091 if key_is_special_id:1092 if value is not None:1093 value = (1094 self.convert_ids_to_tokens(value)1095 if key != "additional_special_tokens"1096 else [self.convert_ids_to_tokens(val) for val in value]1097 )1098 key = key_without_id1099 1100 if key != "additional_special_tokens" and not isinstance(value, (str, AddedToken)) and value is not None:1101 raise ValueError(f"Cannot set a non-string value as the {key}")1102 self._special_tokens_map[key] = value1103 else:1104 super().__setattr__(key, value)1105 1106 def __getattr__(self, key):1107 key_without_id = key1108 key_is_special_id = key.endswith("_id") or key.endswith("_ids")1109 if key_is_special_id:1110 key_without_id = key[:-3] if not key.endswith("_ids") else key[:-4]1111 1112 if self.__dict__.get("_special_tokens_map", None) is not None and any(1113 name in self.__dict__["_special_tokens_map"] for name in [key, key_without_id]1114 ):1115 _special_tokens_map = self.__dict__["_special_tokens_map"]1116 if not key_is_special_id:1117 if _special_tokens_map[key] is None:1118 if self.verbose:1119 logger.error(f"Using {key}, but it is not set yet.")1120 return None1121 value = _special_tokens_map[key]1122 return str(value) if key != "additional_special_tokens" else [str(tok) for tok in value]1123 else:1124 attr_as_tokens = getattr(self, key_without_id)1125 return self.convert_tokens_to_ids(attr_as_tokens) if attr_as_tokens is not None else None1126 1127 if key not in self.__dict__:1128 raise AttributeError(f"{self.__class__.__name__} has no attribute {key}")1129 else:1130 return super().__getattr__(key)1131 1132 @property1133 def special_tokens_map(self) -> dict[str, Union[str, list[str]]]:1134 """1135 `dict[str, Union[str, list[str]]]`: A dictionary mapping special token class attributes (`cls_token`,1136 `unk_token`, etc.) to their values (`'<unk>'`, `'<cls>'`, etc.).1137 1138 Convert potential tokens of `tokenizers.AddedToken` type to string.1139 """1140 set_attr = {}1141 for attr in self.SPECIAL_TOKENS_ATTRIBUTES:1142 attr_value = getattr(self, attr)1143 if attr_value:1144 set_attr[attr] = attr_value1145 return set_attr1146 1147 @property1148 def special_tokens_map_extended(self) -> dict[str, Union[str, AddedToken, list[Union[str, AddedToken]]]]:1149 """1150 `dict[str, Union[str, tokenizers.AddedToken, list[Union[str, tokenizers.AddedToken]]]]`: A dictionary mapping1151 special token class attributes (`cls_token`, `unk_token`, etc.) to their values (`'<unk>'`, `'<cls>'`, etc.).1152 1153 Don't convert tokens of `tokenizers.AddedToken` type to string so they can be used to control more finely how1154 special tokens are tokenized.1155 """1156 set_attr = {}1157 for attr in self.SPECIAL_TOKENS_ATTRIBUTES:1158 attr_value = self._special_tokens_map[attr]1159 if attr_value:1160 set_attr[attr] = attr_value1161 return set_attr1162 1163 @property1164 def all_special_tokens_extended(self) -> list[Union[str, AddedToken]]:1165 """1166 `list[Union[str, tokenizers.AddedToken]]`: All the special tokens (`'<unk>'`, `'<cls>'`, etc.), the order has1167 nothing to do with the index of each tokens. If you want to know the correct indices, check1168 `self.added_tokens_encoder`. We can't create an order anymore as the keys are `AddedTokens` and not `Strings`.1169 1170 Don't convert tokens of `tokenizers.AddedToken` type to string so they can be used to control more finely how1171 special tokens are tokenized.1172 """1173 all_tokens = []1174 seen = set()1175 for value in self.special_tokens_map_extended.values():1176 if isinstance(value, (list, tuple)):1177 tokens_to_add = [token for token in value if str(token) not in seen]1178 else:1179 tokens_to_add = [value] if str(value) not in seen else []1180 seen.update(map(str, tokens_to_add))1181 all_tokens.extend(tokens_to_add)1182 return all_tokens1183 1184 @property1185 def all_special_tokens(self) -> list[str]:1186 """1187 `list[str]`: A list of the unique special tokens (`'<unk>'`, `'<cls>'`, ..., etc.).1188 1189 Convert tokens of `tokenizers.AddedToken` type to string.1190 """1191 all_toks = [str(s) for s in self.all_special_tokens_extended]1192 return all_toks1193 1194 @property1195 def all_special_ids(self) -> list[int]:1196 """1197 `list[int]`: List the ids of the special tokens(`'<unk>'`, `'<cls>'`, etc.) mapped to class attributes.1198 """1199 all_toks = self.all_special_tokens1200 all_ids = self.convert_tokens_to_ids(all_toks)