DoruC/Grounded-Segment-Anything
0
1# coding=utf-82# Copyright 2020 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""16Base classes common to both the slow and the fast tokenization classes: PreTrainedTokenizerBase (host all the user17fronting encoding methods) Special token mixing (host the special tokens logic) and BatchEncoding (wrap the dictionary18of output with special method for the Fast tokenizers)19"""20 21import copy22import json23import os24import re25import warnings26from collections import UserDict27from collections.abc import Mapping, Sized28from contextlib import contextmanager29from dataclasses import dataclass30from functools import lru_cache31from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, Union32 33import numpy as np34from packaging import version35 36from . import __version__37from .dynamic_module_utils import custom_object_save38from .utils import (39 ExplicitEnum,40 PaddingStrategy,41 PushToHubMixin,42 TensorType,43 add_end_docstrings,44 add_model_info_to_auto_map,45 cached_file,46 copy_func,47 download_url,48 extract_commit_hash,49 is_flax_available,50 is_jax_tensor,51 is_numpy_array,52 is_offline_mode,53 is_remote_url,54 is_tf_available,55 is_tf_tensor,56 is_tokenizers_available,57 is_torch_available,58 is_torch_device,59 is_torch_tensor,60 logging,61 requires_backends,62 to_py_obj,63)64 65 66if TYPE_CHECKING:67 if is_torch_available():68 import torch69 if is_tf_available():70 import tensorflow as tf71 if is_flax_available():72 import jax.numpy as jnp # noqa: F40173 from .pipelines.conversational import Conversation74 75 76if is_tokenizers_available():77 from tokenizers import AddedToken78 from tokenizers import Encoding as EncodingFast79else:80 81 @dataclass(frozen=False, eq=True)82 class AddedToken:83 """84 AddedToken represents a token to be added to a Tokenizer An AddedToken can have special options defining the85 way it should behave.86 87 The `normalized` will default to `not special` if it is not specified, similarly to the definition in88 `tokenizers`.89 """90 91 def __init__(92 self, content: str, single_word=False, lstrip=False, rstrip=False, special=False, normalized=None93 ):94 self.content = content95 self.single_word = single_word96 self.lstrip = lstrip97 self.rstrip = rstrip98 self.special = special99 self.normalized = normalized if normalized is not None else not special100 101 def __getstate__(self):102 return self.__dict__103 104 def __str__(self):105 return self.content106 107 @dataclass108 class EncodingFast:109 """This is dummy class because without the `tokenizers` library we don't have these objects anyway"""110 111 pass112 113 114logger = logging.get_logger(__name__)115 116VERY_LARGE_INTEGER = int(1e30) # This is used to set the max input length for a model with infinite size input117LARGE_INTEGER = int(1e20) # This is used when we need something big but slightly smaller than VERY_LARGE_INTEGER118 119# Define type aliases and NamedTuples120TextInput = str121PreTokenizedInput = List[str]122EncodedInput = List[int]123TextInputPair = Tuple[str, str]124PreTokenizedInputPair = Tuple[List[str], List[str]]125EncodedInputPair = Tuple[List[int], List[int]]126 127 128# Slow tokenizers used to be saved in three separated files129SPECIAL_TOKENS_MAP_FILE = "special_tokens_map.json"130ADDED_TOKENS_FILE = "added_tokens.json"131TOKENIZER_CONFIG_FILE = "tokenizer_config.json"132 133# Fast tokenizers (provided by HuggingFace tokenizer's library) can be saved in a single file134FULL_TOKENIZER_FILE = "tokenizer.json"135_re_tokenizer_file = re.compile(r"tokenizer\.(.*)\.json")136 137 138class TruncationStrategy(ExplicitEnum):139 """140 Possible values for the `truncation` argument in [`PreTrainedTokenizerBase.__call__`]. Useful for tab-completion in141 an IDE.142 """143 144 ONLY_FIRST = "only_first"145 ONLY_SECOND = "only_second"146 LONGEST_FIRST = "longest_first"147 DO_NOT_TRUNCATE = "do_not_truncate"148 149 150class CharSpan(NamedTuple):151 """152 Character span in the original string.153 154 Args:155 start (`int`): Index of the first character in the original string.156 end (`int`): Index of the character following the last character in the original string.157 """158 159 start: int160 end: int161 162 163class TokenSpan(NamedTuple):164 """165 Token span in an encoded string (list of tokens).166 167 Args:168 start (`int`): Index of the first token in the span.169 end (`int`): Index of the token following the last token in the span.170 """171 172 start: int173 end: int174 175 176class BatchEncoding(UserDict):177 """178 Holds the output of the [`~tokenization_utils_base.PreTrainedTokenizerBase.__call__`],179 [`~tokenization_utils_base.PreTrainedTokenizerBase.encode_plus`] and180 [`~tokenization_utils_base.PreTrainedTokenizerBase.batch_encode_plus`] methods (tokens, attention_masks, etc).181 182 This class is derived from a python dictionary and can be used as a dictionary. In addition, this class exposes183 utility methods to map from word/character space to token space.184 185 Args:186 data (`dict`, *optional*):187 Dictionary of lists/arrays/tensors returned by the `__call__`/`encode_plus`/`batch_encode_plus` methods188 ('input_ids', 'attention_mask', etc.).189 encoding (`tokenizers.Encoding` or `Sequence[tokenizers.Encoding]`, *optional*):190 If the tokenizer is a fast tokenizer which outputs additional information like mapping from word/character191 space to token space the `tokenizers.Encoding` instance or list of instance (for batches) hold this192 information.193 tensor_type (`Union[None, str, TensorType]`, *optional*):194 You can give a tensor_type here to convert the lists of integers in PyTorch/TensorFlow/Numpy Tensors at195 initialization.196 prepend_batch_axis (`bool`, *optional*, defaults to `False`):197 Whether or not to add a batch axis when converting to tensors (see `tensor_type` above).198 n_sequences (`Optional[int]`, *optional*):199 You can give a tensor_type here to convert the lists of integers in PyTorch/TensorFlow/Numpy Tensors at200 initialization.201 """202 203 def __init__(204 self,205 data: Optional[Dict[str, Any]] = None,206 encoding: Optional[Union[EncodingFast, Sequence[EncodingFast]]] = None,207 tensor_type: Union[None, str, TensorType] = None,208 prepend_batch_axis: bool = False,209 n_sequences: Optional[int] = None,210 ):211 super().__init__(data)212 213 if isinstance(encoding, EncodingFast):214 encoding = [encoding]215 216 self._encodings = encoding217 218 if n_sequences is None and encoding is not None and len(encoding):219 n_sequences = encoding[0].n_sequences220 221 self._n_sequences = n_sequences222 223 self.convert_to_tensors(tensor_type=tensor_type, prepend_batch_axis=prepend_batch_axis)224 225 @property226 def n_sequences(self) -> Optional[int]:227 """228 `Optional[int]`: The number of sequences used to generate each sample from the batch encoded in this229 [`BatchEncoding`]. Currently can be one of `None` (unknown), `1` (a single sentence) or `2` (a pair of230 sentences)231 """232 return self._n_sequences233 234 @property235 def is_fast(self) -> bool:236 """237 `bool`: Indicate whether this [`BatchEncoding`] was generated from the result of a [`PreTrainedTokenizerFast`]238 or not.239 """240 return self._encodings is not None241 242 def __getitem__(self, item: Union[int, str]) -> Union[Any, EncodingFast]:243 """244 If the key is a string, returns the value of the dict associated to `key` ('input_ids', 'attention_mask',245 etc.).246 247 If the key is an integer, get the `tokenizers.Encoding` for batch item with index `key`.248 249 If the key is a slice, returns the value of the dict associated to `key` ('input_ids', 'attention_mask', etc.)250 with the constraint of slice.251 """252 if isinstance(item, str):253 return self.data[item]254 elif self._encodings is not None:255 return self._encodings[item]256 elif isinstance(item, slice):257 return {key: self.data[key][item] for key in self.data.keys()}258 else:259 raise KeyError(260 "Invalid key. Only three types of key are available: "261 "(1) string, (2) integers for backend Encoding, and (3) slices for data subsetting."262 )263 264 def __getattr__(self, item: str):265 try:266 return self.data[item]267 except KeyError:268 raise AttributeError269 270 def __getstate__(self):271 return {"data": self.data, "encodings": self._encodings}272 273 def __setstate__(self, state):274 if "data" in state:275 self.data = state["data"]276 277 if "encodings" in state:278 self._encodings = state["encodings"]279 280 def keys(self):281 return self.data.keys()282 283 def values(self):284 return self.data.values()285 286 def items(self):287 return self.data.items()288 289 # After this point:290 # Extended properties and methods only available for fast (Rust-based) tokenizers291 # provided by HuggingFace tokenizers library.292 293 @property294 def encodings(self) -> Optional[List[EncodingFast]]:295 """296 `Optional[List[tokenizers.Encoding]]`: The list all encodings from the tokenization process. Returns `None` if297 the input was tokenized through Python (i.e., not a fast) tokenizer.298 """299 return self._encodings300 301 def tokens(self, batch_index: int = 0) -> List[str]:302 """303 Return the list of tokens (sub-parts of the input strings after word/subword splitting and before conversion to304 integer indices) at a given batch index (only works for the output of a fast tokenizer).305 306 Args:307 batch_index (`int`, *optional*, defaults to 0): The index to access in the batch.308 309 Returns:310 `List[str]`: The list of tokens at that index.311 """312 if not self._encodings:313 raise ValueError(314 "tokens() is not available when using non-fast tokenizers (e.g. instance of a `XxxTokenizerFast`"315 " class)."316 )317 return self._encodings[batch_index].tokens318 319 def sequence_ids(self, batch_index: int = 0) -> List[Optional[int]]:320 """321 Return a list mapping the tokens to the id of their original sentences:322 323 - `None` for special tokens added around or between sequences,324 - `0` for tokens corresponding to words in the first sequence,325 - `1` for tokens corresponding to words in the second sequence when a pair of sequences was jointly326 encoded.327 328 Args:329 batch_index (`int`, *optional*, defaults to 0): The index to access in the batch.330 331 Returns:332 `List[Optional[int]]`: A list indicating the sequence id corresponding to each token. Special tokens added333 by the tokenizer are mapped to `None` and other tokens are mapped to the index of their corresponding334 sequence.335 """336 if not self._encodings:337 raise ValueError(338 "sequence_ids() is not available when using non-fast tokenizers (e.g. instance of a `XxxTokenizerFast`"339 " class)."340 )341 return self._encodings[batch_index].sequence_ids342 343 def words(self, batch_index: int = 0) -> List[Optional[int]]:344 """345 Return a list mapping the tokens to their actual word in the initial sentence for a fast tokenizer.346 347 Args:348 batch_index (`int`, *optional*, defaults to 0): The index to access in the batch.349 350 Returns:351 `List[Optional[int]]`: A list indicating the word corresponding to each token. Special tokens added by the352 tokenizer are mapped to `None` and other tokens are mapped to the index of their corresponding word353 (several tokens will be mapped to the same word index if they are parts of that word).354 """355 if not self._encodings:356 raise ValueError(357 "words() is not available when using non-fast tokenizers (e.g. instance of a `XxxTokenizerFast`"358 " class)."359 )360 warnings.warn(361 "`BatchEncoding.words()` property is deprecated and should be replaced with the identical, "362 "but more self-explanatory `BatchEncoding.word_ids()` property.",363 FutureWarning,364 )365 return self.word_ids(batch_index)366 367 def word_ids(self, batch_index: int = 0) -> List[Optional[int]]:368 """369 Return a list mapping the tokens to their actual word in the initial sentence for a fast tokenizer.370 371 Args:372 batch_index (`int`, *optional*, defaults to 0): The index to access in the batch.373 374 Returns:375 `List[Optional[int]]`: A list indicating the word corresponding to each token. Special tokens added by the376 tokenizer are mapped to `None` and other tokens are mapped to the index of their corresponding word377 (several tokens will be mapped to the same word index if they are parts of that word).378 """379 if not self._encodings:380 raise ValueError(381 "word_ids() is not available when using non-fast tokenizers (e.g. instance of a `XxxTokenizerFast`"382 " class)."383 )384 return self._encodings[batch_index].word_ids385 386 def token_to_sequence(self, batch_or_token_index: int, token_index: Optional[int] = None) -> int:387 """388 Get the index of the sequence represented by the given token. In the general use case, this method returns `0`389 for a single sequence or the first sequence of a pair, and `1` for the second sequence of a pair390 391 Can be called as:392 393 - `self.token_to_sequence(token_index)` if batch size is 1394 - `self.token_to_sequence(batch_index, token_index)` if batch size is greater than 1395 396 This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e.,397 words are defined by the user). In this case it allows to easily associate encoded tokens with provided398 tokenized words.399 400 Args:401 batch_or_token_index (`int`):402 Index of the sequence in the batch. If the batch only comprises one sequence, this can be the index of403 the token in the sequence.404 token_index (`int`, *optional*):405 If a batch index is provided in *batch_or_token_index*, this can be the index of the token in the406 sequence.407 408 Returns:409 `int`: Index of the word in the input sequence.410 """411 412 if not self._encodings:413 raise ValueError("token_to_sequence() is not available when using Python based tokenizers")414 if token_index is not None:415 batch_index = batch_or_token_index416 else:417 batch_index = 0418 token_index = batch_or_token_index419 if batch_index < 0:420 batch_index = self._batch_size + batch_index421 if token_index < 0:422 token_index = self._seq_len + token_index423 return self._encodings[batch_index].token_to_sequence(token_index)424 425 def token_to_word(self, batch_or_token_index: int, token_index: Optional[int] = None) -> int:426 """427 Get the index of the word corresponding (i.e. comprising) to an encoded token in a sequence of the batch.428 429 Can be called as:430 431 - `self.token_to_word(token_index)` if batch size is 1432 - `self.token_to_word(batch_index, token_index)` if batch size is greater than 1433 434 This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e.,435 words are defined by the user). In this case it allows to easily associate encoded tokens with provided436 tokenized words.437 438 Args:439 batch_or_token_index (`int`):440 Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of441 the token in the sequence.442 token_index (`int`, *optional*):443 If a batch index is provided in *batch_or_token_index*, this can be the index of the token in the444 sequence.445 446 Returns:447 `int`: Index of the word in the input sequence.448 """449 450 if not self._encodings:451 raise ValueError("token_to_word() is not available when using Python based tokenizers")452 if token_index is not None:453 batch_index = batch_or_token_index454 else:455 batch_index = 0456 token_index = batch_or_token_index457 if batch_index < 0:458 batch_index = self._batch_size + batch_index459 if token_index < 0:460 token_index = self._seq_len + token_index461 return self._encodings[batch_index].token_to_word(token_index)462 463 def word_to_tokens(464 self, batch_or_word_index: int, word_index: Optional[int] = None, sequence_index: int = 0465 ) -> Optional[TokenSpan]:466 """467 Get the encoded token span corresponding to a word in a sequence of the batch.468 469 Token spans are returned as a [`~tokenization_utils_base.TokenSpan`] with:470 471 - **start** -- Index of the first token.472 - **end** -- Index of the token following the last token.473 474 Can be called as:475 476 - `self.word_to_tokens(word_index, sequence_index: int = 0)` if batch size is 1477 - `self.word_to_tokens(batch_index, word_index, sequence_index: int = 0)` if batch size is greater or equal to478 1479 480 This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e. words481 are defined by the user). In this case it allows to easily associate encoded tokens with provided tokenized482 words.483 484 Args:485 batch_or_word_index (`int`):486 Index of the sequence in the batch. If the batch only comprises one sequence, this can be the index of487 the word in the sequence.488 word_index (`int`, *optional*):489 If a batch index is provided in *batch_or_token_index*, this can be the index of the word in the490 sequence.491 sequence_index (`int`, *optional*, defaults to 0):492 If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0493 or 1) the provided word index belongs to.494 495 Returns:496 ([`~tokenization_utils_base.TokenSpan`], *optional*): Span of tokens in the encoded sequence. Returns497 `None` if no tokens correspond to the word. This can happen especially when the token is a special token498 that has been used to format the tokenization. For example when we add a class token at the very beginning499 of the tokenization.500 """501 502 if not self._encodings:503 raise ValueError("word_to_tokens() is not available when using Python based tokenizers")504 if word_index is not None:505 batch_index = batch_or_word_index506 else:507 batch_index = 0508 word_index = batch_or_word_index509 if batch_index < 0:510 batch_index = self._batch_size + batch_index511 if word_index < 0:512 word_index = self._seq_len + word_index513 span = self._encodings[batch_index].word_to_tokens(word_index, sequence_index)514 return TokenSpan(*span) if span is not None else None515 516 def token_to_chars(self, batch_or_token_index: int, token_index: Optional[int] = None) -> CharSpan:517 """518 Get the character span corresponding to an encoded token in a sequence of the batch.519 520 Character spans are returned as a [`~tokenization_utils_base.CharSpan`] with:521 522 - **start** -- Index of the first character in the original string associated to the token.523 - **end** -- Index of the character following the last character in the original string associated to the524 token.525 526 Can be called as:527 528 - `self.token_to_chars(token_index)` if batch size is 1529 - `self.token_to_chars(batch_index, token_index)` if batch size is greater or equal to 1530 531 Args:532 batch_or_token_index (`int`):533 Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of534 the token in the sequence.535 token_index (`int`, *optional*):536 If a batch index is provided in *batch_or_token_index*, this can be the index of the token or tokens in537 the sequence.538 539 Returns:540 [`~tokenization_utils_base.CharSpan`]: Span of characters in the original string, or None, if the token541 (e.g. <s>, </s>) doesn't correspond to any chars in the origin string.542 """543 544 if not self._encodings:545 raise ValueError("token_to_chars() is not available when using Python based tokenizers")546 if token_index is not None:547 batch_index = batch_or_token_index548 else:549 batch_index = 0550 token_index = batch_or_token_index551 span_indices = self._encodings[batch_index].token_to_chars(token_index)552 553 return CharSpan(*span_indices) if span_indices is not None else None554 555 def char_to_token(556 self, batch_or_char_index: int, char_index: Optional[int] = None, sequence_index: int = 0557 ) -> int:558 """559 Get the index of the token in the encoded output comprising a character in the original string for a sequence560 of the batch.561 562 Can be called as:563 564 - `self.char_to_token(char_index)` if batch size is 1565 - `self.char_to_token(batch_index, char_index)` if batch size is greater or equal to 1566 567 This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e. words568 are defined by the user). In this case it allows to easily associate encoded tokens with provided tokenized569 words.570 571 Args:572 batch_or_char_index (`int`):573 Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of574 the word in the sequence575 char_index (`int`, *optional*):576 If a batch index is provided in *batch_or_token_index*, this can be the index of the word in the577 sequence.578 sequence_index (`int`, *optional*, defaults to 0):579 If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0580 or 1) the provided character index belongs to.581 582 583 Returns:584 `int`: Index of the token.585 """586 587 if not self._encodings:588 raise ValueError("char_to_token() is not available when using Python based tokenizers")589 if char_index is not None:590 batch_index = batch_or_char_index591 else:592 batch_index = 0593 char_index = batch_or_char_index594 return self._encodings[batch_index].char_to_token(char_index, sequence_index)595 596 def word_to_chars(597 self, batch_or_word_index: int, word_index: Optional[int] = None, sequence_index: int = 0598 ) -> CharSpan:599 """600 Get the character span in the original string corresponding to given word in a sequence of the batch.601 602 Character spans are returned as a CharSpan NamedTuple with:603 604 - start: index of the first character in the original string605 - end: index of the character following the last character in the original string606 607 Can be called as:608 609 - `self.word_to_chars(word_index)` if batch size is 1610 - `self.word_to_chars(batch_index, word_index)` if batch size is greater or equal to 1611 612 Args:613 batch_or_word_index (`int`):614 Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of615 the word in the sequence616 word_index (`int`, *optional*):617 If a batch index is provided in *batch_or_token_index*, this can be the index of the word in the618 sequence.619 sequence_index (`int`, *optional*, defaults to 0):620 If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0621 or 1) the provided word index belongs to.622 623 Returns:624 `CharSpan` or `List[CharSpan]`: Span(s) of the associated character or characters in the string. CharSpan625 are NamedTuple with:626 627 - start: index of the first character associated to the token in the original string628 - end: index of the character following the last character associated to the token in the original629 string630 """631 632 if not self._encodings:633 raise ValueError("word_to_chars() is not available when using Python based tokenizers")634 if word_index is not None:635 batch_index = batch_or_word_index636 else:637 batch_index = 0638 word_index = batch_or_word_index639 return CharSpan(*(self._encodings[batch_index].word_to_chars(word_index, sequence_index)))640 641 def char_to_word(self, batch_or_char_index: int, char_index: Optional[int] = None, sequence_index: int = 0) -> int:642 """643 Get the word in the original string corresponding to a character in the original string of a sequence of the644 batch.645 646 Can be called as:647 648 - `self.char_to_word(char_index)` if batch size is 1649 - `self.char_to_word(batch_index, char_index)` if batch size is greater than 1650 651 This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e. words652 are defined by the user). In this case it allows to easily associate encoded tokens with provided tokenized653 words.654 655 Args:656 batch_or_char_index (`int`):657 Index of the sequence in the batch. If the batch only comprise one sequence, this can be the index of658 the character in the original string.659 char_index (`int`, *optional*):660 If a batch index is provided in *batch_or_token_index*, this can be the index of the character in the661 original string.662 sequence_index (`int`, *optional*, defaults to 0):663 If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0664 or 1) the provided character index belongs to.665 666 667 Returns:668 `int` or `List[int]`: Index or indices of the associated encoded token(s).669 """670 671 if not self._encodings:672 raise ValueError("char_to_word() is not available when using Python based tokenizers")673 if char_index is not None:674 batch_index = batch_or_char_index675 else:676 batch_index = 0677 char_index = batch_or_char_index678 return self._encodings[batch_index].char_to_word(char_index, sequence_index)679 680 def convert_to_tensors(681 self, tensor_type: Optional[Union[str, TensorType]] = None, prepend_batch_axis: bool = False682 ):683 """684 Convert the inner content to tensors.685 686 Args:687 tensor_type (`str` or [`~utils.TensorType`], *optional*):688 The type of tensors to use. If `str`, should be one of the values of the enum [`~utils.TensorType`]. If689 `None`, no modification is done.690 prepend_batch_axis (`int`, *optional*, defaults to `False`):691 Whether or not to add the batch dimension during the conversion.692 """693 if tensor_type is None:694 return self695 696 # Convert to TensorType697 if not isinstance(tensor_type, TensorType):698 tensor_type = TensorType(tensor_type)699 700 # Get a function reference for the correct framework701 if tensor_type == TensorType.TENSORFLOW:702 if not is_tf_available():703 raise ImportError(704 "Unable to convert output to TensorFlow tensors format, TensorFlow is not installed."705 )706 import tensorflow as tf707 708 as_tensor = tf.constant709 is_tensor = tf.is_tensor710 elif tensor_type == TensorType.PYTORCH:711 if not is_torch_available():712 raise ImportError("Unable to convert output to PyTorch tensors format, PyTorch is not installed.")713 import torch714 715 is_tensor = torch.is_tensor716 717 def as_tensor(value, dtype=None):718 if isinstance(value, list) and isinstance(value[0], np.ndarray):719 return torch.tensor(np.array(value))720 return torch.tensor(value)721 722 elif tensor_type == TensorType.JAX:723 if not is_flax_available():724 raise ImportError("Unable to convert output to JAX tensors format, JAX is not installed.")725 import jax.numpy as jnp # noqa: F811726 727 as_tensor = jnp.array728 is_tensor = is_jax_tensor729 else:730 731 def as_tensor(value, dtype=None):732 if isinstance(value, (list, tuple)) and isinstance(value[0], (list, tuple, np.ndarray)):733 value_lens = [len(val) for val in value]734 if len(set(value_lens)) > 1 and dtype is None:735 # we have a ragged list so handle explicitly736 value = as_tensor([np.asarray(val) for val in value], dtype=object)737 return np.asarray(value, dtype=dtype)738 739 is_tensor = is_numpy_array740 741 # Do the tensor conversion in batch742 for key, value in self.items():743 try:744 if prepend_batch_axis:745 value = [value]746 747 if not is_tensor(value):748 tensor = as_tensor(value)749 750 # Removing this for now in favor of controlling the shape with `prepend_batch_axis`751 # # at-least2d752 # if tensor.ndim > 2:753 # tensor = tensor.squeeze(0)754 # elif tensor.ndim < 2:755 # tensor = tensor[None, :]756 757 self[key] = tensor758 except Exception as e:759 if key == "overflowing_tokens":760 raise ValueError(761 "Unable to create tensor returning overflowing tokens of different lengths. "762 "Please see if a fast version of this tokenizer is available to have this feature available."763 ) from e764 raise ValueError(765 "Unable to create tensor, you should probably activate truncation and/or padding with"766 " 'padding=True' 'truncation=True' to have batched tensors with the same length. Perhaps your"767 f" features (`{key}` in this case) have excessive nesting (inputs type `list` where type `int` is"768 " expected)."769 ) from e770 771 return self772 773 def to(self, device: Union[str, "torch.device"]) -> "BatchEncoding":774 """775 Send all values to device by calling `v.to(device)` (PyTorch only).776 777 Args:778 device (`str` or `torch.device`): The device to put the tensors on.779 780 Returns:781 [`BatchEncoding`]: The same instance after modification.782 """783 requires_backends(self, ["torch"])784 785 # This check catches things like APEX blindly calling "to" on all inputs to a module786 # Otherwise it passes the casts down and casts the LongTensor containing the token idxs787 # into a HalfTensor788 if isinstance(device, str) or is_torch_device(device) or isinstance(device, int):789 self.data = {k: v.to(device=device) for k, v in self.data.items()}790 else:791 logger.warning(f"Attempting to cast a BatchEncoding to type {str(device)}. This is not supported.")792 return self793 794 795class SpecialTokensMixin:796 """797 A mixin derived by [`PreTrainedTokenizer`] and [`PreTrainedTokenizerFast`] to handle specific behaviors related to798 special tokens. In particular, this class hold the attributes which can be used to directly access these special799 tokens in a model-independent manner and allow to set and update the special tokens.800 801 Args:802 bos_token (`str` or `tokenizers.AddedToken`, *optional*):803 A special token representing the beginning of a sentence.804 eos_token (`str` or `tokenizers.AddedToken`, *optional*):805 A special token representing the end of a sentence.806 unk_token (`str` or `tokenizers.AddedToken`, *optional*):807 A special token representing an out-of-vocabulary token.808 sep_token (`str` or `tokenizers.AddedToken`, *optional*):809 A special token separating two different sentences in the same input (used by BERT for instance).810 pad_token (`str` or `tokenizers.AddedToken`, *optional*):811 A special token used to make arrays of tokens the same size for batching purpose. Will then be ignored by812 attention mechanisms or loss computation.813 cls_token (`str` or `tokenizers.AddedToken`, *optional*):814 A special token representing the class of the input (used by BERT for instance).815 mask_token (`str` or `tokenizers.AddedToken`, *optional*):816 A special token representing a masked token (used by masked-language modeling pretraining objectives, like817 BERT).818 additional_special_tokens (tuple or list of `str` or `tokenizers.AddedToken`, *optional*):819 A tuple or a list of additional tokens, which will be marked as `special`, meaning that they will be820 skipped when decoding if `skip_special_tokens` is set to `True`.821 """822 823 SPECIAL_TOKENS_ATTRIBUTES = [824 "bos_token",825 "eos_token",826 "unk_token",827 "sep_token",828 "pad_token",829 "cls_token",830 "mask_token",831 "additional_special_tokens",832 ]833 834 def __init__(self, verbose=True, **kwargs):835 self._bos_token = None836 self._eos_token = None837 self._unk_token = None838 self._sep_token = None839 self._pad_token = None840 self._cls_token = None841 self._mask_token = None842 self._pad_token_type_id = 0843 self._additional_special_tokens = []844 self.verbose = verbose845 846 # We directly set the hidden value to allow initialization with special tokens847 # which are not yet in the vocabulary. Necessary for serialization/de-serialization848 # TODO clean this up at some point (probably by switching to fast tokenizers)849 850 for key, value in kwargs.items():851 if value is None:852 continue853 if key in self.SPECIAL_TOKENS_ATTRIBUTES:854 if key == "additional_special_tokens":855 # TODO THIS IS NASTY! Will always reset tokens to default rstrip and lstrip because self.set_attr on strings856 # will not check the addedtokens decoder. WILL FIX TOMORROW857 assert isinstance(value, (list, tuple)), f"Value {value} is not a list or tuple"858 assert all(859 isinstance(t, (str, AddedToken)) for t in value860 ), "One of the tokens is not a string or an AddedToken"861 if hasattr(self, "added_tokens_encoder"):862 extended_token = []863 for token in value:864 if isinstance(token, str) and str(token) in self.added_tokens_encoder:865 extended_token.append(self.added_tokens_decoder[self.added_tokens_encoder[str(token)]])866 else:867 extended_token.append(token)868 value = extended_token869 setattr(self, key, value)870 elif isinstance(value, (str)):871 value = AddedToken(value, normalized=False, special=True)872 setattr(self, key, value)873 elif isinstance(value, AddedToken):874 setattr(self, key, value)875 else:876 raise TypeError(f"Special token {key} has to be either str or AddedToken but got: {type(value)}")877 878 def sanitize_special_tokens(self) -> int:879 """880 The `sanitize_special_tokens` is now deprecated kept for backward compatibility and will be removed in881 transformers v5.882 """883 logger.warning_once("The `sanitize_special_tokens` will be removed in transformers v5.")884 return self.add_tokens(self.all_special_tokens_extended, special_tokens=True)885 886 def add_special_tokens(887 self, special_tokens_dict: Dict[str, Union[str, AddedToken]], replace_additional_special_tokens=True888 ) -> int:889 """890 Add a dictionary of special tokens (eos, pad, cls, etc.) to the encoder and link them to class attributes. If891 special tokens are NOT in the vocabulary, they are added to it (indexed starting from the last index of the892 current vocabulary).893 894 When adding new tokens to the vocabulary, you should make sure to also resize the token embedding matrix of the895 model so that its embedding matrix matches the tokenizer.896 897 In order to do that, please use the [`~PreTrainedModel.resize_token_embeddings`] method.898 899 Using `add_special_tokens` will ensure your special tokens can be used in several ways:900 901 - Special tokens can be skipped when decoding using `skip_special_tokens = True`.902 - Special tokens are carefully handled by the tokenizer (they are never split), similar to `AddedTokens`.903 - You can easily refer to special tokens using tokenizer class attributes like `tokenizer.cls_token`. This904 makes it easy to develop model-agnostic training and fine-tuning scripts.905 906 When possible, special tokens are already registered for provided pretrained models (for instance907 [`BertTokenizer`] `cls_token` is already registered to be :obj*'[CLS]'* and XLM's one is also registered to be908 `'</s>'`).909 910 Args:911 special_tokens_dict (dictionary *str* to *str* or `tokenizers.AddedToken`):912 Keys should be in the list of predefined special attributes: [`bos_token`, `eos_token`, `unk_token`,913 `sep_token`, `pad_token`, `cls_token`, `mask_token`, `additional_special_tokens`].914 915 Tokens are only added if they are not already in the vocabulary (tested by checking if the tokenizer916 assign the index of the `unk_token` to them).917 replace_additional_special_tokens (`bool`, *optional*,, defaults to `True`):918 If `True`, the existing list of additional special tokens will be replaced by the list provided in919 `special_tokens_dict`. Otherwise, `self._additional_special_tokens` is just extended. In the former920 case, the tokens will NOT be removed from the tokenizer's full vocabulary - they are only being flagged921 as non-special tokens. Remember, this only affects which tokens are skipped during decoding, not the922 `added_tokens_encoder` and `added_tokens_decoder`. This means that the previous923 `additional_special_tokens` are still added tokens, and will not be split by the model.924 925 Returns:926 `int`: Number of tokens added to the vocabulary.927 928 Examples:929 930 ```python931 # Let's see how to add a new classification token to GPT-2932 tokenizer = GPT2Tokenizer.from_pretrained("gpt2")933 model = GPT2Model.from_pretrained("gpt2")934 935 special_tokens_dict = {"cls_token": "<CLS>"}936 937 num_added_toks = tokenizer.add_special_tokens(special_tokens_dict)938 print("We have added", num_added_toks, "tokens")939 # Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e., the length of the tokenizer.940 model.resize_token_embeddings(len(tokenizer))941 942 assert tokenizer.cls_token == "<CLS>"943 ```"""944 if not special_tokens_dict:945 return 0946 947 added_tokens = []948 for key, value in special_tokens_dict.items():949 assert key in self.SPECIAL_TOKENS_ATTRIBUTES, f"Key {key} is not a special token"950 951 if self.verbose:952 logger.info(f"Assigning {value} to the {key} key of the tokenizer")953 954 if key == "additional_special_tokens":955 assert isinstance(value, (list, tuple)) and all(956 isinstance(t, (str, AddedToken)) for t in value957 ), f"Tokens {value} for key {key} should all be str or AddedToken instances"958 959 to_add = set()960 for token in value:961 if isinstance(token, str):962 # for legacy purpose we default to stripping. `test_add_tokens_tokenizer` depends on this963 token = AddedToken(token, normalized=False, rstrip=True, lstrip=True)964 if str(token) not in self.additional_special_tokens:965 to_add.add(token)966 if replace_additional_special_tokens:967 setattr(self, key, list(to_add))968 else:969 self._additional_special_tokens.extend(to_add)970 added_tokens += to_add971 972 else:973 if not isinstance(value, (str, AddedToken)):974 raise ValueError(f"Token {value} for key {key} should be a str or an AddedToken instance")975 if isinstance(value, (str)):976 # for legacy purpose we default to stripping. `test_add_tokens_tokenizer` depends on this977 value = AddedToken(value, normalized=False, rstrip=True, lstrip=True)978 if isinstance(value, AddedToken):979 setattr(self, key, value)980 if value not in added_tokens:981 added_tokens.append(value)982 983 # if we are adding tokens that were not part of the vocab, we ought to add them984 added_tokens = self.add_tokens(added_tokens, special_tokens=True)985 return added_tokens986 987 def add_tokens(988 self, new_tokens: Union[str, AddedToken, List[Union[str, AddedToken]]], special_tokens: bool = False989 ) -> int:990 """991 Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to992 it with indices starting from length of the current vocabulary and and will be isolated before the tokenization993 algorithm is applied. Added tokens and tokens from the vocabulary of the tokenization algorithm are therefore994 not treated in the same way.995 996 Note, when adding new tokens to the vocabulary, you should make sure to also resize the token embedding matrix997 of the model so that its embedding matrix matches the tokenizer.998 999 In order to do that, please use the [`~PreTrainedModel.resize_token_embeddings`] method.1000 1001 Args:1002 new_tokens (`str`, `tokenizers.AddedToken` or a list of *str* or `tokenizers.AddedToken`):1003 Tokens are only added if they are not already in the vocabulary. `tokenizers.AddedToken` wraps a string1004 token to let you personalize its behavior: whether this token should only match against a single word,1005 whether this token should strip all potential whitespaces on the left side, whether this token should1006 strip all potential whitespaces on the right side, etc.1007 special_tokens (`bool`, *optional*, defaults to `False`):1008 Can be used to specify if the token is a special token. This mostly change the normalization behavior1009 (special tokens like CLS or [MASK] are usually not lower-cased for instance).1010 1011 See details for `tokenizers.AddedToken` in HuggingFace tokenizers library.1012 1013 Returns:1014 `int`: Number of tokens added to the vocabulary.1015 1016 Examples:1017 1018 ```python1019 # Let's see how to increase the vocabulary of Bert model and tokenizer1020 tokenizer = BertTokenizerFast.from_pretrained("bert-base-uncased")1021 model = BertModel.from_pretrained("bert-base-uncased")1022 1023 num_added_toks = tokenizer.add_tokens(["new_tok1", "my_new-tok2"])1024 print("We have added", num_added_toks, "tokens")1025 # Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e., the length of the tokenizer.1026 model.resize_token_embeddings(len(tokenizer))1027 ```"""1028 if not new_tokens:1029 return 01030 1031 if not isinstance(new_tokens, (list, tuple)):1032 new_tokens = [new_tokens]1033 1034 return self._add_tokens(new_tokens, special_tokens=special_tokens)1035 1036 def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:1037 raise NotImplementedError1038 1039 @property1040 def bos_token(self) -> str:1041 """1042 `str`: Beginning of sentence token. Log an error if used while not having been set.1043 """1044 if self._bos_token is None:1045 if self.verbose:1046 logger.error("Using bos_token, but it is not set yet.")1047 return None1048 return str(self._bos_token)1049 1050 @property1051 def eos_token(self) -> str:1052 """1053 `str`: End of sentence token. Log an error if used while not having been set.1054 """1055 if self._eos_token is None:1056 if self.verbose:1057 logger.error("Using eos_token, but it is not set yet.")1058 return None1059 return str(self._eos_token)1060 1061 @property1062 def unk_token(self) -> str:1063 """1064 `str`: Unknown token. Log an error if used while not having been set.1065 """1066 if self._unk_token is None:1067 if self.verbose:1068 logger.error("Using unk_token, but it is not set yet.")1069 return None1070 return str(self._unk_token)1071 1072 @property1073 def sep_token(self) -> str:1074 """1075 `str`: Separation token, to separate context and query in an input sequence. Log an error if used while not1076 having been set.1077 """1078 if self._sep_token is None:1079 if self.verbose:1080 logger.error("Using sep_token, but it is not set yet.")1081 return None1082 return str(self._sep_token)1083 1084 @property1085 def pad_token(self) -> str:1086 """1087 `str`: Padding token. Log an error if used while not having been set.1088 """1089 if self._pad_token is None:1090 if self.verbose:1091 logger.error("Using pad_token, but it is not set yet.")1092 return None1093 return str(self._pad_token)1094 1095 @property1096 def cls_token(self) -> str:1097 """1098 `str`: Classification token, to extract a summary of an input sequence leveraging self-attention along the full1099 depth of the model. Log an error if used while not having been set.1100 """1101 if self._cls_token is None:1102 if self.verbose:1103 logger.error("Using cls_token, but it is not set yet.")1104 return None1105 return str(self._cls_token)1106 1107 @property1108 def mask_token(self) -> str:1109 """1110 `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while not1111 having been set.1112 """1113 if self._mask_token is None:1114 if self.verbose:1115 logger.error("Using mask_token, but it is not set yet.")1116 return None1117 return str(self._mask_token)1118 1119 @property1120 def additional_special_tokens(self) -> List[str]:1121 """1122 `List[str]`: All the additional special tokens you may want to use. Log an error if used while not having been1123 set.1124 """1125 if self._additional_special_tokens is None:1126 if self.verbose:1127 logger.error("Using additional_special_tokens, but it is not set yet.")1128 return None1129 return [str(tok) for tok in self._additional_special_tokens]1130 1131 @bos_token.setter1132 def bos_token(self, value):1133 if isinstance(value, str) and value != "":1134 value = AddedToken(value, normalized=False, rstrip=True, lstrip=True, special=True)1135 elif not isinstance(value, AddedToken) and value is not None:1136 raise ValueError("Cannot set a non-string value as the BOS token")1137 self._bos_token = value1138 1139 @eos_token.setter1140 def eos_token(self, value):1141 if isinstance(value, str) and value != "":1142 value = AddedToken(value, normalized=False, rstrip=True, lstrip=True, special=True)1143 elif not isinstance(value, AddedToken) and value is not None:1144 raise ValueError("Cannot set a non-string value as the EOS token")1145 self._eos_token = value1146 1147 @unk_token.setter1148 def unk_token(self, value):1149 if isinstance(value, str) and value != "":1150 value = AddedToken(value, normalized=False, rstrip=True, lstrip=True, special=True)1151 elif not isinstance(value, AddedToken) and value is not None:1152 raise ValueError("Cannot set a non-string value as the UNK token")1153 self._unk_token = value1154 1155 @sep_token.setter1156 def sep_token(self, value):1157 if isinstance(value, str) and value != "":1158 value = AddedToken(value, normalized=False, rstrip=True, lstrip=True, special=True)1159 elif not isinstance(value, AddedToken) and value is not None:1160 raise ValueError("Cannot set a non-string value as the SEP token")1161 self._sep_token = value1162 1163 @pad_token.setter1164 def pad_token(self, value):1165 if isinstance(value, str) and value != "":1166 value = AddedToken(value, normalized=False, rstrip=True, lstrip=True, special=True)1167 elif not isinstance(value, AddedToken) and value is not None:1168 raise ValueError("Cannot set a non-string value as the PAD token")1169 self._pad_token = value1170 1171 @cls_token.setter1172 def cls_token(self, value):1173 if isinstance(value, str) and value != "":1174 value = AddedToken(value, normalized=False, rstrip=True, lstrip=True, special=True)1175 elif not isinstance(value, AddedToken) and value is not None:1176 raise ValueError("Cannot set a non-string value as the CLS token")1177 self._cls_token = value1178 1179 @mask_token.setter1180 def mask_token(self, value):1181 if isinstance(value, str) and value != "":1182 value = AddedToken(value, normalized=False, rstrip=True, lstrip=True, special=True)1183 elif not isinstance(value, AddedToken) and value is not None:1184 raise ValueError("Cannot set a non-string value as the MASK token")1185 self._mask_token = value1186 1187 @additional_special_tokens.setter1188 def additional_special_tokens(self, value):1189 if value is None:1190 self._additional_special_tokens = value1191 return1192 if self._additional_special_tokens is None:1193 self._additional_special_tokens = []1194 # We store the `AddedToken` to allow adding tokens via `tokenizer.add_special_tokens`1195 for token in value:1196 if isinstance(token, str) and token != "":1197 token = AddedToken(token, normalized=False, rstrip=True, lstrip=True, special=True)1198 elif not isinstance(token, AddedToken):1199 raise ValueError(f"Cannot add instance of type {type(value)} to additional_special_tokens!")1200 self._additional_special_tokens.append(token)