Aluode/PerceptionLabPortable
0
1# Copyright 2025 Mistral AI and The HuggingFace Inc. team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import os16import shutil17import warnings18from collections.abc import Mapping, Sized19from enum import Enum20from pathlib import Path21from typing import Any, Callable, Optional, Union, overload22 23import numpy as np24 25from transformers.audio_utils import load_audio_as26from transformers.tokenization_utils_base import (27 LARGE_INTEGER,28 VERY_LARGE_INTEGER,29 BatchEncoding,30 EncodedInput,31 PreTokenizedInput,32 PreTrainedTokenizerBase,33 TextInput,34 TruncationStrategy,35)36from transformers.utils import PaddingStrategy, TensorType, add_end_docstrings, logging, to_py_obj37from transformers.utils.generic import is_torch_tensor38from transformers.utils.hub import PushToHubMixin39from transformers.utils.import_utils import is_mistral_common_available, is_torch_available, requires40 41 42if is_mistral_common_available():43 from mistral_common.protocol.instruct.request import ChatCompletionRequest44 from mistral_common.protocol.instruct.validator import ValidationMode45 from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy, TokenizerVersion46 from mistral_common.tokens.tokenizers.image import MultiModalVersion47 from mistral_common.tokens.tokenizers.mistral import MistralTokenizer48 from mistral_common.tokens.tokenizers.tekken import Tekkenizer49 from mistral_common.tokens.tokenizers.utils import download_tokenizer_from_hf_hub50 51 52if is_torch_available():53 import torch54 55 56logger = logging.get_logger(__name__)57 58 59ENCODE_KWARGS_DOCSTRING = r"""60 add_special_tokens (`bool`, *optional*, defaults to `True`):61 Whether or not to add special tokens when encoding the sequences. This will use the underlying62 `PretrainedTokenizerBase.build_inputs_with_special_tokens` function, which defines which tokens are63 automatically added to the input ids. This is useful if you want to add `bos` or `eos` tokens64 automatically.65 padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):66 Activates and controls padding. Accepts the following values:67 68 - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single69 sequence is provided).70 - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum71 acceptable input length for the model if that argument is not provided.72 - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different73 lengths).74 truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):75 Activates and controls truncation. Accepts the following values:76 77 - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or78 to the maximum acceptable input length for the model if that argument is not provided.79 - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths80 greater than the model maximum admissible input size).81 max_length (`int`, *optional*):82 Controls the maximum length to use by one of the truncation/padding parameters.83 84 If left unset or set to `None`, this will use the predefined model maximum length if a maximum length85 is required by one of the truncation/padding parameters. If the model has no specific maximum input86 length (like XLNet) truncation/padding to a maximum length will be deactivated.87 stride (`int`, *optional*, defaults to 0):88 If set to a number along with `max_length`, the overflowing tokens returned when89 `return_overflowing_tokens=True` will contain some tokens from the end of the truncated sequence90 returned to provide some overlap between truncated and overflowing sequences. The value of this91 argument defines the number of overlapping tokens.92 pad_to_multiple_of (`int`, *optional*):93 If set will pad the sequence to a multiple of the provided value. Requires `padding` to be activated.94 This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability95 `>= 7.5` (Volta).96 padding_side (`str`, *optional*):97 The side on which the model should have padding applied. Should be selected between ['right', 'left'].98 Default value is picked from the class attribute of the same name.99 return_tensors (`str` or [`~utils.TensorType`], *optional*):100 If set, will return tensors instead of list of python integers. Acceptable values are:101 102 - `'pt'`: Return PyTorch `torch.Tensor` objects.103"""104 105ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""106 return_attention_mask (`bool`, *optional*):107 Whether to return the attention mask. If left to the default, will return the attention mask according108 to the specific tokenizer's default, defined by the `return_outputs` attribute.109 110 [What are attention masks?](../glossary#attention-mask)111 return_overflowing_tokens (`bool`, *optional*, defaults to `False`):112 Whether or not to return overflowing token sequences. If a pair of sequences of input ids (or a batch113 of pairs) is provided with `truncation_strategy = longest_first` or `True`, an error is raised instead114 of returning overflowing tokens.115 return_special_tokens_mask (`bool`, *optional*, defaults to `False`):116 Whether or not to return special tokens mask information.117 return_offsets_mapping (`bool`, *optional*, defaults to `False`):118 Whether or not to return `(char_start, char_end)` for each token.119 120 This is only available on fast tokenizers inheriting from [`PreTrainedTokenizerFast`], if using121 Python's tokenizer, this method will raise `NotImplementedError`.122 return_length (`bool`, *optional*, defaults to `False`):123 Whether or not to return the lengths of the encoded inputs.124 verbose (`bool`, *optional*, defaults to `True`):125 Whether or not to print more information and warnings.126 **kwargs: passed to the `self.tokenize()` method127 128 Return:129 [`BatchEncoding`]: A [`BatchEncoding`] with the following fields:130 131 - **input_ids** -- List of token ids to be fed to a model.132 133 [What are input IDs?](../glossary#input-ids)134 135 - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when136 `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names`).137 138 [What are attention masks?](../glossary#attention-mask)139 140 - **overflowing_tokens** -- List of overflowing tokens sequences (when a `max_length` is specified and141 `return_overflowing_tokens=True`).142 - **num_truncated_tokens** -- Number of tokens truncated (when a `max_length` is specified and143 `return_overflowing_tokens=True`).144 - **special_tokens_mask** -- List of 0s and 1s, with 1 specifying added special tokens and 0 specifying145 regular sequence tokens (when `add_special_tokens=True` and `return_special_tokens_mask=True`).146 - **length** -- The length of the inputs (when `return_length=True`)147"""148 149 150class MistralTokenizerType(str, Enum):151 """Enum for the different type of tokenizer."""152 153 spm = "spm"154 tekken = "tekken"155 156 157@requires(backends=("mistral-common",))158class MistralCommonTokenizer(PushToHubMixin):159 """160 Class to wrap `mistral-common` tokenizers.161 162 `mistral-common` is the official tokenizer library for Mistral AI models. To use it, you need to install it with:163 164 ```bash165 pip install transformers[mistral-common]166 ```167 168 Otherwise the tokenizer falls back to the Transformers implementation of the tokenizer.169 170 For more info on `mistral-common`, see [mistral-common](https://github.com/mistralai/mistral-common).171 172 This class is a wrapper around a `mistral_common.tokens.tokenizers.mistral.MistralTokenizer`.173 It provides a Hugging Face compatible interface to tokenize using the official mistral-common tokenizer.174 175 Supports the following methods from the `PreTrainedTokenizerBase` class:176 177 - [`~MistralCommonTokenizer.get_vocab`]: Returns the vocabulary as a dictionary of token to index.178 - [`~MistralCommonTokenizer.encode`]: Encode a string to a list of integers.179 - [`~MistralCommonTokenizer.decode`]: Decode a list of integers to a string.180 - [`~MistralCommonTokenizer.batch_decode`]: Decode a batch of list of integers to a list of strings.181 - [`~MistralCommonTokenizer.convert_tokens_to_ids`]: Convert a list of tokens to a list of integers.182 - [`~MistralCommonTokenizer.convert_ids_to_tokens`]: Convert a list of integers to a list of tokens.183 - [`~MistralCommonTokenizer.tokenize`]: Tokenize a string.184 - [`~MistralCommonTokenizer.get_special_tokens_mask`]: Get the special tokens mask for a list of tokens.185 - [`~MistralCommonTokenizer.prepare_for_model`]: Prepare a list of inputs for the model.186 - [`~MistralCommonTokenizer.pad`]: Pad a list of inputs to the same length.187 - [`~MistralCommonTokenizer.truncate_sequences`]: Truncate a list of sequences to the same length.188 - [`~MistralCommonTokenizer.apply_chat_template`]: Apply a chat template to a list of messages.189 - [`~MistralCommonTokenizer.__call__`]: Tokenize a string or a list of strings.190 - [`~MistralCommonTokenizer.from_pretrained`]: Download and cache a pretrained tokenizer from the Hugging Face model hub or local directory.191 - [`~MistralCommonTokenizer.save_pretrained`]: Save a tokenizer to a directory, so it can be reloaded using the `from_pretrained` class method.192 - [`~MistralCommonTokenizer.push_to_hub`]: Upload tokenizer to the Hugging Face model hub.193 194 Here are the key differences with the `PreTrainedTokenizerBase` class:195 196 - Pair of sequences are not supported. The signature have been kept for compatibility but all arguments related to pair of sequences are ignored. The return values of pairs are returned as `None`.197 - The `is_split_into_words` argument is not supported.198 - The `return_token_type_ids` argument is not supported.199 - It is not possible to add new tokens to the tokenizer. Also the special tokens are handled differently from Transformers. In `mistral-common`, special tokens are never encoded directly. This means that: `tokenizer.encode("<s>")` will not return the ID of the `<s>` token. Instead, it will return a list of IDs corresponding to the tokenization of the string `"<s>"`. For more information, see the [mistral-common documentation](https://mistralai.github.io/mistral-common/usage/tokenizers/#special-tokens).200 201 If you have suggestions to improve this class, please open an issue on the [mistral-common GitHub repository](https://github.com/mistralai/mistral-common/issues) if it is related to the tokenizer or on the [Transformers GitHub repository](https://github.com/huggingface/transformers/issues) if it is related to the Hugging Face interface.202 """203 204 model_input_names: list[str] = ["input_ids", "attention_mask"]205 padding_side: str = "left"206 truncation_side: str = "right"207 208 def __init__(209 self,210 tokenizer_path: Union[str, os.PathLike, Path],211 mode: ValidationMode = ValidationMode.test,212 model_max_length: int = VERY_LARGE_INTEGER,213 padding_side: str = "left",214 truncation_side: str = "right",215 model_input_names: Optional[list[str]] = None,216 clean_up_tokenization_spaces: bool = False,217 **kwargs,218 ):219 """220 Constructs a `MistralCommonTokenizer`.221 222 - **model_input_names** (`List[str]`) -- A list of inputs expected in the forward pass of the model.223 - **padding_side** (`str`) -- The default value for the side on which the model should have padding applied.224 Should be `'right'` or `'left'`.225 - **truncation_side** (`str`) -- The default value for the side on which the model should have truncation226 applied. Should be `'right'` or `'left'`.227 228 Args:229 tokenizer_path (`str` or `os.PathLike` or `Path`):230 Path to the tokenizer file to load the `MistralTokenizer`.231 mode (`ValidationMode`, *optional*, defaults to `ValidationMode.test`):232 The mode to use for the tokenizer. This will be passed to the `MistralTokenizer` constructor.233 model_max_length (`int`, *optional*):234 The maximum length (in number of tokens) for the inputs to the transformer model. When the tokenizer is235 loaded with [`~tokenization_utils_base.PreTrainedTokenizerBase.from_pretrained`], this will be set to the236 value stored for the associated model in `max_model_input_sizes` (see above). If no value is provided, will237 default to VERY_LARGE_INTEGER (`int(1e30)`).238 padding_side (`str`, *optional*):239 The side on which the model should have padding applied. Should be selected between ['right', 'left'].240 Default value is picked from the class attribute of the same name.241 truncation_side (`str`, *optional*):242 The side on which the model should have truncation applied. Should be selected between ['right', 'left'].243 Default value is picked from the class attribute of the same name.244 model_input_names (`List[string]`, *optional*):245 The list of inputs accepted by the forward pass of the model (like `"token_type_ids"` or246 `"attention_mask"`). Default value is picked from the class attribute of the same name.247 clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):248 Whether or not the model should cleanup the spaces that were added when splitting the input text during the249 tokenization process.250 """251 if kwargs:252 raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported to init `MistralCommonTokenizer`.")253 254 self._tokenizer_path = Path(tokenizer_path)255 self.tokenizer: MistralTokenizer = MistralTokenizer.from_file(str(self._tokenizer_path), mode=mode)256 self._tokenizer_type = (257 MistralTokenizerType.tekken258 if isinstance(self.tokenizer.instruct_tokenizer.tokenizer, Tekkenizer)259 else MistralTokenizerType.spm260 )261 self.truncation_side = truncation_side262 self.padding_side = padding_side263 self.model_max_length = model_max_length264 self.cleanup_tokenization_spaces = clean_up_tokenization_spaces265 self.deprecation_warnings = {} # Use to store when we have already noticed a deprecation warning (avoid overlogging).266 267 if model_input_names is not None:268 if (269 not isinstance(model_input_names, (list, tuple))270 and len(model_input_names) == 0271 and not all(isinstance(i, str) for i in model_input_names)272 ):273 raise ValueError(274 "`model_input_names` should be a non-empty list or tuple of str but got an empty value."275 )276 self.model_input_names = model_input_names277 278 self._cache_get_vocab: Optional[dict[str, int]] = None279 280 @property281 def bos_token_id(self) -> int:282 """283 Id of the beginning of sentence token in the vocabulary.284 """285 return self.tokenizer.instruct_tokenizer.tokenizer.bos_id286 287 @property288 def eos_token_id(self) -> int:289 """290 Id of the end of sentence token in the vocabulary.291 """292 return self.tokenizer.instruct_tokenizer.tokenizer.eos_id293 294 @property295 def unk_token_id(self) -> int:296 """297 Id of the unknown token in the vocabulary.298 """299 return self.tokenizer.instruct_tokenizer.tokenizer.unk_id300 301 @property302 def pad_token_id(self) -> int:303 """304 Id of the padding token in the vocabulary.305 """306 return self.tokenizer.instruct_tokenizer.tokenizer.pad_id307 308 @property309 def bos_token(self) -> str:310 """311 String associated to the beginning of sentence token in the vocabulary.312 """313 return self.convert_ids_to_tokens(self.bos_token_id)314 315 @property316 def eos_token(self) -> str:317 """318 String associated to the end of sentence token in the vocabulary.319 """320 return self.convert_ids_to_tokens(self.eos_token_id)321 322 @property323 def unk_token(self) -> str:324 """325 String associated to the unknown token in the vocabulary.326 """327 return self.convert_ids_to_tokens(self.unk_token_id)328 329 @property330 def pad_token(self) -> str:331 """332 String associated to the padding token in the vocabulary.333 """334 return self.convert_ids_to_tokens(self.pad_token_id)335 336 @property337 def vocab_size(self) -> int:338 """339 Returns the size of the vocabulary.340 341 `int`: Size of the vocabulary.342 """343 return self.tokenizer.instruct_tokenizer.tokenizer.n_words344 345 def get_vocab(self) -> dict[str, int]:346 """347 Returns the vocabulary as a dictionary of token to index.348 349 This is a lossy conversion. There may be multiple token ids that decode to the same350 string due to partial UTF-8 byte sequences being converted to �.351 352 Returns:353 `Dict[str, int]`: The vocabulary.354 """355 if self._cache_get_vocab is None:356 self._cache_get_vocab = {357 token: idx for idx, token in enumerate(self.tokenizer.instruct_tokenizer.tokenizer.vocab())358 }359 return self._cache_get_vocab360 361 def __len__(self):362 """363 Size of the full vocabulary with the added tokens.364 """365 return self.vocab_size366 367 @add_end_docstrings(368 ENCODE_KWARGS_DOCSTRING,369 """370 **kwargs: Not supported by `MistralCommonTokenizer.encode`.371 Will raise an error if used.372 """,373 """374 Returns:375 `List[int]`, `torch.Tensor`: The tokenized ids of the text.376 """,377 )378 def encode(379 self,380 text: Union[TextInput, EncodedInput],381 text_pair: None = None,382 add_special_tokens: bool = True,383 padding: Union[bool, str, PaddingStrategy] = False,384 truncation: Union[bool, str, TruncationStrategy, None] = None,385 max_length: Optional[int] = None,386 stride: int = 0,387 pad_to_multiple_of: Optional[int] = None,388 padding_side: Optional[str] = None,389 return_tensors: Optional[Union[str, TensorType]] = None,390 verbose: bool = True,391 **kwargs,392 ) -> list[int]:393 """394 Converts a string to a sequence of ids (integer), using the tokenizer and vocabulary.395 396 Args:397 text (`str` or `List[int]`):398 The first sequence to be encoded. This can be a string or a list of integers (tokenized string ids).399 text_pair (`None`, *optional*):400 Not supported by `MistralCommonTokenizer.encode`. Kept to match `PreTrainedTokenizerBase.encode` signature.401 """402 if kwargs:403 raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonTokenizer.encode`.")404 if text_pair:405 raise ValueError("`MistralCommonTokenizer.encode` does not support `text_pair`.")406 407 padding_strategy, truncation_strategy, max_length, _ = self._get_padding_truncation_strategies(408 padding=padding,409 truncation=truncation,410 max_length=max_length,411 pad_to_multiple_of=pad_to_multiple_of,412 verbose=verbose,413 )414 415 encoded_inputs = self._encode_plus(416 text,417 add_special_tokens=add_special_tokens,418 padding_strategy=padding_strategy,419 truncation_strategy=truncation_strategy,420 max_length=max_length,421 stride=stride,422 pad_to_multiple_of=pad_to_multiple_of,423 padding_side=padding_side,424 return_tensors=return_tensors,425 return_attention_mask=False,426 return_overflowing_tokens=False,427 return_special_tokens_mask=False,428 return_length=False,429 verbose=verbose,430 )431 432 return encoded_inputs["input_ids"]433 434 def decode(435 self,436 token_ids: Union[int, list[int], np.ndarray, "torch.Tensor"],437 skip_special_tokens: bool = False,438 clean_up_tokenization_spaces: Optional[bool] = None,439 **kwargs,440 ) -> str:441 """442 Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special443 tokens and clean up tokenization spaces.444 445 Args:446 token_ids (`Union[int, List[int], np.ndarray, torch.Tensor]`):447 List of tokenized input ids. Can be obtained using the `__call__` method.448 skip_special_tokens (`bool`, *optional*, defaults to `False`):449 Whether or not to remove special tokens in the decoding.450 clean_up_tokenization_spaces (`bool`, *optional*):451 Whether or not to clean up the tokenization spaces. If `None`, will default to452 `self.clean_up_tokenization_spaces`.453 kwargs (additional keyword arguments, *optional*):454 Not supported by `MistralCommonTokenizer.decode`.455 Will raise an error if used.456 457 Returns:458 `str`: The decoded sentence.459 """460 if kwargs:461 raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonTokenizer.decode`.")462 463 clean_up_tokenization_spaces = clean_up_tokenization_spaces or self.cleanup_tokenization_spaces464 465 # Convert inputs to python lists466 token_ids = to_py_obj(token_ids)467 468 special_token_policy = SpecialTokenPolicy.IGNORE if skip_special_tokens else SpecialTokenPolicy.KEEP469 470 decoded_string = self.tokenizer.decode(token_ids, special_token_policy=special_token_policy)471 if clean_up_tokenization_spaces:472 decoded_string = PreTrainedTokenizerBase.clean_up_tokenization(decoded_string)473 474 return decoded_string475 476 def batch_decode(477 self,478 sequences: Union[list[int], list[list[int]], np.ndarray, "torch.Tensor"],479 skip_special_tokens: bool = False,480 clean_up_tokenization_spaces: Optional[bool] = None,481 **kwargs,482 ) -> list[str]:483 """484 Convert a list of lists of token ids into a list of strings by calling decode.485 486 Args:487 sequences (`Union[List[int], List[List[int]], np.ndarray, torch.Tensor]`):488 List of tokenized input ids. Can be obtained using the `__call__` method.489 skip_special_tokens (`bool`, *optional*, defaults to `False`):490 Whether or not to remove special tokens in the decoding.491 clean_up_tokenization_spaces (`bool`, *optional*):492 Whether or not to clean up the tokenization spaces. If `None`, will default to493 `self.clean_up_tokenization_spaces`.494 kwargs (additional keyword arguments, *optional*):495 Not supported by `MistralCommonTokenizer.batch_decode`.496 Will raise an error if used.497 498 Returns:499 `List[str]`: The list of decoded sentences.500 """501 return [502 self.decode(503 seq,504 skip_special_tokens=skip_special_tokens,505 clean_up_tokenization_spaces=clean_up_tokenization_spaces,506 **kwargs,507 )508 for seq in sequences509 ]510 511 def _is_control_token(self, token_id: int) -> bool:512 if self._tokenizer_type == MistralTokenizerType.spm:513 return token_id in self.tokenizer.instruct_tokenizer.tokenizer._control_tokens()514 elif self._tokenizer_type == MistralTokenizerType.tekken:515 return token_id < self.tokenizer.instruct_tokenizer.tokenizer.num_special_tokens516 else:517 raise ValueError(f"Unknown tokenizer type: {self._tokenizer_type}")518 519 @overload520 def convert_ids_to_tokens(self, ids: int, skip_special_tokens: bool = False) -> str: ...521 @overload522 def convert_ids_to_tokens(self, ids: list[int], skip_special_tokens: bool = False) -> list[str]: ...523 def convert_ids_to_tokens(524 self, ids: Union[int, list[int]], skip_special_tokens: bool = False525 ) -> Union[str, list[str]]:526 """527 Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and528 added tokens.529 530 Args:531 ids (`int` or `List[int]`):532 The token id (or token ids) to convert to tokens.533 skip_special_tokens (`bool`, *optional*, defaults to `False`):534 Whether or not to remove special tokens in the decoding.535 536 Returns:537 `str` or `List[str]`: The decoded token(s).538 """539 540 if isinstance(ids, int):541 one_token = True542 ids = [ids]543 else:544 one_token = False545 546 tokens: list[str] = []547 for token_id in ids:548 if self._is_control_token(token_id) and skip_special_tokens:549 continue550 tokens.append(self.tokenizer.instruct_tokenizer.tokenizer.id_to_piece(token_id))551 552 if one_token:553 if tokens == []:554 raise ValueError(f"Invalid token id {ids}.")555 556 return tokens[0]557 return tokens558 559 def _piece_to_id(self, piece: str) -> int:560 if self._tokenizer_type == MistralTokenizerType.spm:561 return self.tokenizer.instruct_tokenizer.tokenizer._model.piece_to_id(piece)562 elif self._tokenizer_type == MistralTokenizerType.tekken:563 pieces = self.tokenizer.instruct_tokenizer.tokenizer._model.encode(564 piece, allowed_special="all", disallowed_special=set()565 )566 assert len(pieces) == 1, f"Expected to decode 1 token, got {len(pieces)}"567 return pieces[0]568 else:569 raise ValueError(f"Unknown tokenizer type: {self._tokenizer_type}")570 571 def convert_tokens_to_ids(self, tokens: Union[str, list[str]]) -> Union[int, list[int]]:572 """573 Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the574 vocabulary.575 576 Args:577 tokens (`str` or `List[str]`): One or several token(s) to convert to token id(s).578 579 Returns:580 `int` or `List[int]`: The token id or list of token ids.581 """582 583 if isinstance(tokens, str):584 one_token = True585 tokens = [tokens]586 else:587 one_token = False588 589 ids: list[int] = []590 for token in tokens:591 ids.append(self._piece_to_id(token))592 593 if one_token:594 return ids[0]595 return ids596 597 def _text_to_ids(self, text: TextInput, add_special_tokens: bool) -> list[int]:598 """599 Converts a string into a sequence of tokens ids, using the tokenizer.600 """601 tokens_ids = self.tokenizer.instruct_tokenizer.tokenizer.encode(602 text, bos=add_special_tokens, eos=add_special_tokens603 )604 return tokens_ids605 606 def tokenize(self, text: TextInput, **kwargs) -> list[str]:607 """608 Converts a string into a sequence of tokens, using the tokenizer.609 610 Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies.611 612 Args:613 text (`str`):614 The sequence to be encoded.615 **kwargs (additional keyword arguments):616 Not supported by `MistralCommonTokenizer.tokenize`.617 Will raise an error if used.618 619 Returns:620 `List[str]`: The list of tokens.621 """622 if kwargs:623 raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonTokenizer.tokenize`.")624 625 return self.convert_ids_to_tokens(self._text_to_ids(text, add_special_tokens=False), skip_special_tokens=False)626 627 def _encode_plus(628 self,629 text: Union[TextInput, EncodedInput],630 add_special_tokens: bool = True,631 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,632 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,633 max_length: Optional[int] = None,634 stride: int = 0,635 pad_to_multiple_of: Optional[int] = None,636 padding_side: Optional[str] = None,637 return_tensors: Optional[Union[str, TensorType]] = None,638 return_attention_mask: Optional[bool] = None,639 return_overflowing_tokens: bool = False,640 return_special_tokens_mask: bool = False,641 return_length: bool = False,642 verbose: bool = True,643 **kwargs,644 ) -> BatchEncoding:645 if kwargs:646 raise ValueError(647 f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonTokenizer._encode_plus`."648 )649 650 def get_input_ids(text):651 if isinstance(text, str):652 return self._text_to_ids(text, add_special_tokens)653 elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):654 return text655 else:656 raise ValueError(f"Input {text} is not valid. Should be a string, or a list/tuple of integers.")657 658 ids = get_input_ids(text)659 660 return self.prepare_for_model(661 ids,662 add_special_tokens=add_special_tokens,663 padding=padding_strategy.value,664 truncation=truncation_strategy.value,665 max_length=max_length,666 stride=stride,667 pad_to_multiple_of=pad_to_multiple_of,668 padding_side=padding_side,669 return_tensors=return_tensors,670 prepend_batch_axis=True,671 return_attention_mask=return_attention_mask,672 return_overflowing_tokens=return_overflowing_tokens,673 return_special_tokens_mask=return_special_tokens_mask,674 return_length=return_length,675 verbose=verbose,676 )677 678 def _batch_encode_plus(679 self,680 batch_text: Union[681 list[TextInput],682 list[EncodedInput],683 ],684 add_special_tokens: bool = True,685 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,686 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,687 max_length: Optional[int] = None,688 stride: int = 0,689 pad_to_multiple_of: Optional[int] = None,690 padding_side: Optional[str] = None,691 return_tensors: Optional[Union[str, TensorType]] = None,692 return_attention_mask: Optional[bool] = None,693 return_overflowing_tokens: bool = False,694 return_special_tokens_mask: bool = False,695 return_offsets_mapping: bool = False,696 return_length: bool = False,697 verbose: bool = True,698 **kwargs,699 ) -> BatchEncoding:700 def get_input_ids(text):701 if isinstance(text, str):702 return self._text_to_ids(text, add_special_tokens)703 elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):704 return text705 else:706 raise ValueError("Input is not valid. Should be a string or a list/tuple of integers.")707 708 if return_offsets_mapping:709 raise NotImplementedError(710 "return_offset_mapping is not available when using Python tokenizers. "711 "To use this feature, change your tokenizer to one deriving from "712 "transformers.PreTrainedTokenizerFast."713 )714 715 input_ids = []716 for ids in batch_text:717 input_ids.append(get_input_ids(ids))718 719 batch_outputs = self._batch_prepare_for_model(720 input_ids,721 add_special_tokens=add_special_tokens,722 padding_strategy=padding_strategy,723 truncation_strategy=truncation_strategy,724 max_length=max_length,725 stride=stride,726 pad_to_multiple_of=pad_to_multiple_of,727 padding_side=padding_side,728 return_attention_mask=return_attention_mask,729 return_overflowing_tokens=return_overflowing_tokens,730 return_special_tokens_mask=return_special_tokens_mask,731 return_length=return_length,732 return_tensors=return_tensors,733 verbose=verbose,734 )735 736 return BatchEncoding(batch_outputs)737 738 def _all_special_ids(self) -> set[int]:739 if self._tokenizer_type == MistralTokenizerType.tekken:740 return {t["rank"] for t in self.tokenizer.instruct_tokenizer.tokenizer._all_special_tokens}741 elif self._tokenizer_type == MistralTokenizerType.spm:742 return self.tokenizer.instruct_tokenizer.tokenizer._control_tokens()743 else:744 raise ValueError(f"Unknown tokenizer type: {self._tokenizer_type}")745 746 def get_special_tokens_mask(747 self, token_ids_0: list, token_ids_1: None = None, already_has_special_tokens: bool = False748 ) -> list[int]:749 """750 Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding751 special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.752 753 Args:754 token_ids_0 (`List[int]`):755 List of ids of the sequence.756 token_ids_1 (`List[int]`, *optional*):757 Not supported by `MistralCommonTokenizer`. Kept to match the interface of `PreTrainedTokenizerBase`.758 already_has_special_tokens (`bool`, *optional*, defaults to `False`):759 Whether or not the token list is already formatted with special tokens for the model.760 761 Returns:762 A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.763 """764 if token_ids_1 is not None:765 raise ValueError(766 "`token_ids_1` is not supported by `MistralCommonTokenizer` and should be `None`, kept for compatibility."767 )768 if already_has_special_tokens:769 raise ValueError(770 "`already_has_special_tokens` is not supported by `MistralCommonTokenizer` and should be `False`."771 )772 773 all_special_ids = self._all_special_ids() # cache the ids774 775 special_tokens_mask = [1 if token in all_special_ids else 0 for token in token_ids_0]776 return special_tokens_mask777 778 def _batch_prepare_for_model(779 self,780 batch_ids: list[Union[PreTokenizedInput, list[int]]],781 add_special_tokens: bool = True,782 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,783 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,784 max_length: Optional[int] = None,785 stride: int = 0,786 pad_to_multiple_of: Optional[int] = None,787 padding_side: Optional[str] = None,788 return_tensors: Optional[str] = None,789 return_attention_mask: Optional[bool] = None,790 return_overflowing_tokens: bool = False,791 return_special_tokens_mask: bool = False,792 return_length: bool = False,793 verbose: bool = True,794 ) -> BatchEncoding:795 """796 Prepares a sequence of input id so that it can be used by the model. It797 adds special tokens, truncates sequences if overflowing while taking into account the special tokens and798 manages a moving window (with user defined stride) for overflowing tokens.799 800 Args:801 batch_ids: list of tokenized input ids802 """803 804 batch_outputs = {}805 for ids in batch_ids:806 outputs = self.prepare_for_model(807 ids,808 add_special_tokens=add_special_tokens,809 padding=PaddingStrategy.DO_NOT_PAD.value, # we pad in batch afterward810 truncation=truncation_strategy.value,811 max_length=max_length,812 stride=stride,813 pad_to_multiple_of=None, # we pad in batch afterward814 padding_side=None, # we pad in batch afterward815 return_attention_mask=False, # we pad in batch afterward816 return_overflowing_tokens=return_overflowing_tokens,817 return_special_tokens_mask=return_special_tokens_mask,818 return_length=return_length,819 return_tensors=None, # We convert the whole batch to tensors at the end820 prepend_batch_axis=False,821 verbose=verbose,822 )823 824 for key, value in outputs.items():825 if key not in batch_outputs:826 batch_outputs[key] = []827 batch_outputs[key].append(value)828 829 batch_outputs = self.pad(830 batch_outputs,831 padding=padding_strategy.value,832 max_length=max_length,833 pad_to_multiple_of=pad_to_multiple_of,834 padding_side=padding_side,835 return_attention_mask=return_attention_mask,836 )837 838 batch_outputs = BatchEncoding(batch_outputs, tensor_type=return_tensors)839 840 return batch_outputs841 842 @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)843 def prepare_for_model(844 self,845 ids: list[int],846 pair_ids: None = None,847 add_special_tokens: bool = True,848 padding: Union[bool, str, PaddingStrategy] = False,849 truncation: Union[bool, str, TruncationStrategy, None] = None,850 max_length: Optional[int] = None,851 stride: int = 0,852 pad_to_multiple_of: Optional[int] = None,853 padding_side: Optional[str] = None,854 return_tensors: Optional[Union[str, TensorType]] = None,855 return_attention_mask: Optional[bool] = None,856 return_overflowing_tokens: bool = False,857 return_special_tokens_mask: bool = False,858 return_length: bool = False,859 verbose: bool = True,860 prepend_batch_axis: bool = False,861 **kwargs,862 ) -> BatchEncoding:863 """864 Prepares a sequence of input id so that it can be used by the model. It865 adds special tokens, truncates sequences if overflowing while taking into account the special tokens and866 manages a moving window (with user defined stride) for overflowing tokens.867 868 Args:869 ids (`List[int]`):870 Tokenized input ids of the first sequence.871 pair_ids (`None`, *optional*):872 Not supported by `MistralCommonTokenizer`. Kept to match the interface of `PreTrainedTokenizerBase`.873 """874 if pair_ids is not None:875 raise ValueError(876 "`pair_ids` is not supported by `MistralCommonTokenizer` and should be `None`, kept for compatibility."877 )878 if kwargs:879 raise ValueError(880 f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonTokenizer.prepare_for_model`."881 )882 883 padding_strategy, truncation_strategy, max_length, _ = self._get_padding_truncation_strategies(884 padding=padding,885 truncation=truncation,886 max_length=max_length,887 pad_to_multiple_of=pad_to_multiple_of,888 verbose=verbose,889 )890 891 len_ids = len(ids)892 893 # Load from model defaults894 if return_attention_mask is None:895 return_attention_mask = "attention_mask" in self.model_input_names896 897 encoded_inputs = {}898 899 # Truncation: Handle max sequence length900 overflowing_tokens = []901 if truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE and max_length and len_ids > max_length:902 ids, _, overflowing_tokens = self.truncate_sequences(903 ids,904 num_tokens_to_remove=len_ids - max_length,905 truncation_strategy=truncation_strategy,906 stride=stride,907 )908 909 if return_overflowing_tokens:910 encoded_inputs["overflowing_tokens"] = overflowing_tokens911 encoded_inputs["num_truncated_tokens"] = len_ids - max_length912 913 # Build output dictionary914 encoded_inputs[self.model_input_names[0]] = ids915 if return_special_tokens_mask:916 if add_special_tokens:917 encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, None)918 else:919 encoded_inputs["special_tokens_mask"] = [0] * len(ids)920 921 # Padding922 if padding_strategy != PaddingStrategy.DO_NOT_PAD or return_attention_mask:923 encoded_inputs = self.pad(924 encoded_inputs,925 max_length=max_length,926 padding=padding_strategy.value,927 pad_to_multiple_of=pad_to_multiple_of,928 padding_side=padding_side,929 return_attention_mask=return_attention_mask,930 )931 932 if return_length:933 encoded_inputs["length"] = len(encoded_inputs["input_ids"])934 935 batch_outputs = BatchEncoding(936 encoded_inputs, tensor_type=return_tensors, prepend_batch_axis=prepend_batch_axis937 )938 939 return batch_outputs940 941 def _get_padding_truncation_strategies(942 self,943 padding: Union[str, PaddingStrategy, bool] = False,944 truncation: Optional[Union[str, TruncationStrategy, bool]] = None,945 max_length: Optional[int] = None,946 pad_to_multiple_of: Optional[int] = None,947 verbose: bool = True,948 **kwargs,949 ):950 """951 Find the correct padding/truncation strategy.952 """953 954 # Backward compatibility for previous behavior, maybe we should deprecate it:955 # If you only set max_length, it activates truncation for max_length956 if max_length is not None and padding is False and truncation is None:957 if verbose:958 if not self.deprecation_warnings.get("Truncation-not-explicitly-activated", False):959 logger.warning(960 "Truncation was not explicitly activated but `max_length` is provided a specific value, please"961 " use `truncation=True` to explicitly truncate examples to max length. Defaulting to"962 " 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the"963 " tokenizer you can select this strategy more precisely by providing a specific strategy to"964 " `truncation`."965 )966 self.deprecation_warnings["Truncation-not-explicitly-activated"] = True967 truncation = "longest_first"968 969 # Get padding strategy970 if padding is not False:971 if padding is True:972 if verbose:973 if max_length is not None and (974 truncation is None or truncation is False or truncation == "do_not_truncate"975 ):976 warnings.warn(977 "`max_length` is ignored when `padding`=`True` and there is no truncation strategy. "978 "To pad to max length, use `padding='max_length'`."979 )980 padding_strategy = PaddingStrategy.LONGEST # Default to pad to the longest sequence in the batch981 elif not isinstance(padding, PaddingStrategy):982 padding_strategy = PaddingStrategy(padding)983 elif isinstance(padding, PaddingStrategy):984 padding_strategy = padding985 else:986 padding_strategy = PaddingStrategy.DO_NOT_PAD987 988 # Get truncation strategy989 if truncation is not False and truncation is not None:990 if truncation is True:991 truncation_strategy = (992 TruncationStrategy.LONGEST_FIRST993 ) # Default to truncate the longest sequences in pairs of inputs994 elif not isinstance(truncation, TruncationStrategy):995 truncation_strategy = TruncationStrategy(truncation)996 elif isinstance(truncation, TruncationStrategy):997 truncation_strategy = truncation998 if truncation in [TruncationStrategy.ONLY_FIRST, TruncationStrategy.ONLY_SECOND]:999 raise ValueError(1000 "Truncation strategy `only_first` and `only_second` are not supported by `MistralCommonTokenizer`."1001 )1002 else:1003 truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE1004 1005 # Set max length if needed1006 if max_length is None:1007 if padding_strategy == PaddingStrategy.MAX_LENGTH:1008 if self.model_max_length > LARGE_INTEGER:1009 if verbose:1010 if not self.deprecation_warnings.get("Asking-to-pad-to-max_length", False):1011 logger.warning(1012 "Asking to pad to max_length but no maximum length is provided and the model has no"1013 " predefined maximum length. Default to no padding."1014 )1015 self.deprecation_warnings["Asking-to-pad-to-max_length"] = True1016 padding_strategy = PaddingStrategy.DO_NOT_PAD1017 else:1018 max_length = self.model_max_length1019 1020 if truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE:1021 if self.model_max_length > LARGE_INTEGER:1022 if verbose:1023 if not self.deprecation_warnings.get("Asking-to-truncate-to-max_length", False):1024 logger.warning(1025 "Asking to truncate to max_length but no maximum length is provided and the model has"1026 " no predefined maximum length. Default to no truncation."1027 )1028 self.deprecation_warnings["Asking-to-truncate-to-max_length"] = True1029 truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE1030 else:1031 max_length = self.model_max_length1032 1033 # Test if we have a padding token1034 if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.pad_token is None or self.pad_token_id < 0):1035 raise ValueError(1036 "Asking to pad but the tokenizer does not have a padding token. "1037 "Please select a token to use as `pad_token` `(tokenizer.pad_token = tokenizer.eos_token e.g.)` "1038 "or add a new pad token via `tokenizer.add_special_tokens({'pad_token': '[PAD]'})`."1039 )1040 1041 # Check that we will truncate to a multiple of pad_to_multiple_of if both are provided1042 if (1043 truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE1044 and padding_strategy != PaddingStrategy.DO_NOT_PAD1045 and pad_to_multiple_of is not None1046 and max_length is not None1047 and (max_length % pad_to_multiple_of != 0)1048 ):1049 raise ValueError(1050 "Truncation and padding are both activated but "1051 f"truncation length ({max_length}) is not a multiple of pad_to_multiple_of ({pad_to_multiple_of})."1052 )1053 1054 return padding_strategy, truncation_strategy, max_length, kwargs1055 1056 def _pad(1057 self,1058 encoded_inputs: Union[dict[str, EncodedInput], BatchEncoding],1059 max_length: Optional[int] = None,1060 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,1061 pad_to_multiple_of: Optional[int] = None,1062 padding_side: Optional[str] = None,1063 return_attention_mask: Optional[bool] = None,1064 ) -> dict:1065 """1066 Pad encoded inputs (on left/right and up to predefined length or max length in the batch)1067 1068 Args:1069 encoded_inputs:1070 Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).1071 max_length: maximum length of the returned list and optionally padding length (see below).1072 Will truncate by taking into account the special tokens.1073 padding_strategy: PaddingStrategy to use for padding.1074 1075 - PaddingStrategy.LONGEST Pad to the longest sequence in the batch1076 - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)1077 - PaddingStrategy.DO_NOT_PAD: Do not pad1078 The tokenizer padding sides are defined in `padding_side` argument:1079 1080 - 'left': pads on the left of the sequences1081 - 'right': pads on the right of the sequences1082 pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.1083 This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability1084 `>= 7.5` (Volta).1085 padding_side:1086 The side on which the model should have padding applied. Should be selected between ['right', 'left'].1087 Default value is picked from the class attribute of the same name.1088 return_attention_mask:1089 (optional) Set to False to avoid returning attention mask (default: set to model specifics)1090 """1091 # Load from model defaults1092 if return_attention_mask is None:1093 return_attention_mask = "attention_mask" in self.model_input_names1094 1095 required_input = encoded_inputs[self.model_input_names[0]]1096 1097 if padding_strategy == PaddingStrategy.LONGEST:1098 max_length = len(required_input)1099 1100 if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):1101 max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of1102 1103 needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length1104 1105 # Initialize attention mask if not present.1106 if return_attention_mask and "attention_mask" not in encoded_inputs:1107 encoded_inputs["attention_mask"] = [1] * len(required_input)1108 1109 if needs_to_be_padded:1110 difference = max_length - len(required_input)1111 padding_side = padding_side if padding_side is not None else self.padding_side1112 1113 if padding_side == "right":1114 if return_attention_mask:1115 encoded_inputs["attention_mask"] = encoded_inputs["attention_mask"] + [0] * difference1116 if "special_tokens_mask" in encoded_inputs:1117 encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference1118 encoded_inputs[self.model_input_names[0]] = required_input + [self.pad_token_id] * difference1119 elif padding_side == "left":1120 if return_attention_mask:1121 encoded_inputs["attention_mask"] = [0] * difference + encoded_inputs["attention_mask"]1122 if "special_tokens_mask" in encoded_inputs:1123 encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"]1124 encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input1125 else:1126 raise ValueError(f"Invalid padding strategy:{padding_side}")1127 1128 return encoded_inputs1129 1130 def pad(1131 self,1132 encoded_inputs: Union[1133 BatchEncoding,1134 list[BatchEncoding],1135 dict[str, EncodedInput],1136 dict[str, list[EncodedInput]],1137 list[dict[str, EncodedInput]],1138 ],1139 padding: Union[bool, str, PaddingStrategy] = True,1140 max_length: Optional[int] = None,1141 pad_to_multiple_of: Optional[int] = None,1142 padding_side: Optional[str] = None,1143 return_attention_mask: Optional[bool] = None,1144 return_tensors: Optional[Union[str, TensorType]] = None,1145 verbose: bool = True,1146 ) -> BatchEncoding:1147 """1148 Pad a single encoded input or a batch of encoded inputs up to predefined length or to the max sequence length1149 in the batch.1150 1151 Padding side (left/right) padding token ids are defined at the tokenizer level (with `self.padding_side`,1152 `self.pad_token_id`).1153 <Tip>1154 1155 If the `encoded_inputs` passed are dictionary of numpy arrays, PyTorch tensors, the1156 result will use the same type unless you provide a different tensor type with `return_tensors`. In the case of1157 PyTorch tensors, you will lose the specific device of your tensors however.1158 1159 </Tip>1160 1161 Args:1162 encoded_inputs ([`BatchEncoding`], list of [`BatchEncoding`], `Dict[str, List[int]]`, `Dict[str, List[List[int]]` or `List[Dict[str, List[int]]]`):1163 Tokenized inputs. Can represent one input ([`BatchEncoding`] or `Dict[str, List[int]]`) or a batch of1164 tokenized inputs (list of [`BatchEncoding`], *Dict[str, List[List[int]]]* or *List[Dict[str,1165 List[int]]]*) so you can use this method during preprocessing as well as in a PyTorch Dataloader1166 collate function.1167 1168 Instead of `List[int]` you can have tensors (numpy arrays, PyTorch tensors), see1169 the note above for the return type.1170 padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):1171 Select a strategy to pad the returned sequences (according to the model's padding side and padding1172 index) among:1173 1174 - `True` or `'longest'` (default): Pad to the longest sequence in the batch (or no padding if only a single1175 sequence if provided).1176 - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum1177 acceptable input length for the model if that argument is not provided.1178 - `False` or `'do_not_pad'`: No padding (i.e., can output a batch with sequences of different1179 lengths).1180 max_length (`int`, *optional*):1181 Maximum length of the returned list and optionally padding length (see above).1182 pad_to_multiple_of (`int`, *optional*):1183 If set will pad the sequence to a multiple of the provided value.1184 1185 This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability1186 `>= 7.5` (Volta).1187 padding_side (`str`, *optional*):1188 The side on which the model should have padding applied. Should be selected between ['right', 'left'].1189 Default value is picked from the class attribute of the same name.1190 return_attention_mask (`bool`, *optional*):1191 Whether to return the attention mask. If left to the default, will return the attention mask according1192 to the specific tokenizer's default, defined by the `return_outputs` attribute.1193 1194 [What are attention masks?](../glossary#attention-mask)1195 return_tensors (`str` or [`~utils.TensorType`], *optional*):1196 If set, will return tensors instead of list of python integers. Acceptable values are:1197 1198 - `'pt'`: Return PyTorch `torch.Tensor` objects.1199 - `'np'`: Return Numpy `np.ndarray` objects.1200 verbose (`bool`, *optional*, defaults to `True`):