Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2024 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License15"""Tokenization classes for UDOP model."""16 17import os18import re19import warnings20from shutil import copyfile21from typing import Any, Optional, Union22 23import sentencepiece as spm24 25from ...tokenization_utils import PreTrainedTokenizer26from ...tokenization_utils_base import (27 AddedToken,28 BatchEncoding,29 EncodedInput,30 PreTokenizedInput,31 TextInput,32 TextInputPair,33 TruncationStrategy,34)35from ...utils import PaddingStrategy, TensorType, add_end_docstrings, logging36from ...utils.import_utils import requires37 38 39logger = logging.get_logger(__name__)40 41 42SPIECE_UNDERLINE = "▁"43 44 45UDOP_ENCODE_KWARGS_DOCSTRING = r"""46 add_special_tokens (`bool`, *optional*, defaults to `True`):47 Whether or not to encode the sequences with the special tokens relative to their model.48 padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `False`):49 Activates and controls padding. Accepts the following values:50 51 - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single52 sequence if provided).53 - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum54 acceptable input length for the model if that argument is not provided.55 - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different56 lengths).57 truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):58 Activates and controls truncation. Accepts the following values:59 60 - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or61 to the maximum acceptable input length for the model if that argument is not provided. This will62 truncate token by token, removing a token from the longest sequence in the pair if a pair of63 sequences (or a batch of pairs) is provided.64 - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the65 maximum acceptable input length for the model if that argument is not provided. This will only66 truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.67 - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the68 maximum acceptable input length for the model if that argument is not provided. This will only69 truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.70 - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths71 greater than the model maximum admissible input size).72 max_length (`int`, *optional*):73 Controls the maximum length to use by one of the truncation/padding parameters.74 75 If left unset or set to `None`, this will use the predefined model maximum length if a maximum length76 is required by one of the truncation/padding parameters. If the model has no specific maximum input77 length (like XLNet) truncation/padding to a maximum length will be deactivated.78 stride (`int`, *optional*, defaults to 0):79 If set to a number along with `max_length`, the overflowing tokens returned when80 `return_overflowing_tokens=True` will contain some tokens from the end of the truncated sequence81 returned to provide some overlap between truncated and overflowing sequences. The value of this82 argument defines the number of overlapping tokens.83 pad_to_multiple_of (`int`, *optional*):84 If set will pad the sequence to a multiple of the provided value. This is especially useful to enable85 the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta).86 return_tensors (`str` or [`~file_utils.TensorType`], *optional*):87 If set, will return tensors instead of list of python integers. Acceptable values are:88 89 - `'tf'`: Return TensorFlow `tf.constant` objects.90 - `'pt'`: Return PyTorch `torch.Tensor` objects.91 - `'np'`: Return Numpy `np.ndarray` objects.92 return_token_type_ids (`bool`, *optional*):93 Whether to return token type IDs. If left to the default, will return the token type IDs according to94 the specific tokenizer's default, defined by the `return_outputs` attribute.95 96 [What are token type IDs?](../glossary#token-type-ids)97 return_attention_mask (`bool`, *optional*):98 Whether to return the attention mask. If left to the default, will return the attention mask according99 to the specific tokenizer's default, defined by the `return_outputs` attribute.100 101 [What are attention masks?](../glossary#attention-mask)102 return_overflowing_tokens (`bool`, *optional*, defaults to `False`):103 Whether or not to return overflowing token sequences. If a pair of sequences of input ids (or a batch104 of pairs) is provided with `truncation_strategy = longest_first` or `True`, an error is raised instead105 of returning overflowing tokens.106 return_special_tokens_mask (`bool`, *optional*, defaults to `False`):107 Whether or not to return special tokens mask information.108 return_offsets_mapping (`bool`, *optional*, defaults to `False`):109 Whether or not to return `(char_start, char_end)` for each token.110 111 This is only available on fast tokenizers inheriting from [`PreTrainedTokenizerFast`], if using112 Python's tokenizer, this method will raise `NotImplementedError`.113 return_length (`bool`, *optional*, defaults to `False`):114 Whether or not to return the lengths of the encoded inputs.115 verbose (`bool`, *optional*, defaults to `True`):116 Whether or not to print more information and warnings.117 **kwargs: passed to the `self.tokenize()` method118 119 Return:120 [`BatchEncoding`]: A [`BatchEncoding`] with the following fields:121 122 - **input_ids** -- List of token ids to be fed to a model.123 124 [What are input IDs?](../glossary#input-ids)125 126 - **bbox** -- List of bounding boxes to be fed to a model.127 128 - **token_type_ids** -- List of token type ids to be fed to a model (when `return_token_type_ids=True` or129 if *"token_type_ids"* is in `self.model_input_names`).130 131 [What are token type IDs?](../glossary#token-type-ids)132 133 - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when134 `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names`).135 136 [What are attention masks?](../glossary#attention-mask)137 138 - **labels** -- List of labels to be fed to a model. (when `word_labels` is specified).139 - **overflowing_tokens** -- List of overflowing tokens sequences (when a `max_length` is specified and140 `return_overflowing_tokens=True`).141 - **num_truncated_tokens** -- Number of tokens truncated (when a `max_length` is specified and142 `return_overflowing_tokens=True`).143 - **special_tokens_mask** -- List of 0s and 1s, with 1 specifying added special tokens and 0 specifying144 regular sequence tokens (when `add_special_tokens=True` and `return_special_tokens_mask=True`).145 - **length** -- The length of the inputs (when `return_length=True`).146"""147 148VOCAB_FILES_NAMES = {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}149 150 151@requires(backends=("sentencepiece",))152class UdopTokenizer(PreTrainedTokenizer):153 """154 Adapted from [`LayoutXLMTokenizer`] and [`T5Tokenizer`]. Based on155 [SentencePiece](https://github.com/google/sentencepiece).156 157 This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to158 this superclass for more information regarding those methods.159 160 Args:161 vocab_file (`str`):162 Path to the vocabulary file.163 164 eos_token (`str`, *optional*, defaults to `"</s>"`):165 The end of sequence token.166 167 <Tip>168 169 When building a sequence using special tokens, this is not the token that is used for the end of sequence.170 The token used is the `sep_token`.171 172 </Tip>173 174 unk_token (`str`, *optional*, defaults to `"<unk>"`):175 The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this176 token instead.177 178 sep_token (`str`, *optional*, defaults to `"</s>"`):179 The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for180 sequence classification or for a text and a question for question answering. It is also used as the last181 token of a sequence built with special tokens.182 183 pad_token (`str`, *optional*, defaults to `"<pad>"`):184 The token used for padding, for example when batching sequences of different lengths.185 sep_token_box (`list[int]`, *optional*, defaults to `[1000, 1000, 1000, 1000]`):186 The bounding box to use for the special [SEP] token.187 pad_token_box (`list[int]`, *optional*, defaults to `[0, 0, 0, 0]`):188 The bounding box to use for the special [PAD] token.189 pad_token_label (`int`, *optional*, defaults to -100):190 The label to use for padding tokens. Defaults to -100, which is the `ignore_index` of PyTorch's191 CrossEntropyLoss.192 only_label_first_subword (`bool`, *optional*, defaults to `True`):193 Whether or not to only label the first subword, in case word labels are provided.194 additional_special_tokens (`list[str]`, *optional*, defaults to `["<s>NOTUSED", "</s>NOTUSED"]`):195 Additional special tokens used by the tokenizer.196 197 sp_model_kwargs (`dict`, *optional*):198 Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for199 SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,200 to set:201 202 - `enable_sampling`: Enable subword regularization.203 - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.204 205 - `nbest_size = {0,1}`: No sampling is performed.206 - `nbest_size > 1`: samples from the nbest_size results.207 - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)208 using forward-filtering-and-backward-sampling algorithm.209 210 - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for211 BPE-dropout.212 legacy (`bool`, *optional*, defaults to `True`):213 Whether or not the `legacy` behaviour of the tokenizer should be used. Legacy is before the merge of #24622214 which includes fixes to properly handle tokens that appear after special tokens. A simple example:215 - `legacy=True`:216 ```python217 >>> from transformers import T5Tokenizer218 219 >>> tokenizer = T5Tokenizer.from_pretrained("t5-base", legacy=True)220 >>> tokenizer.encode("Hello <extra_id_0>.")221 [8774, 32099, 3, 5, 1]222 ```223 - `legacy=False`:224 ```python225 >>> from transformers import T5Tokenizer226 227 >>> tokenizer = T5Tokenizer.from_pretrained("t5-base", legacy=False)228 >>> tokenizer.encode("Hello <extra_id_0>.") # the extra space `[3]` is no longer here229 [8774, 32099, 5, 1]230 ```231 Checkout the pull request and the issue [here](https://github.com/huggingface/transformers/pull/24565) for232 more details.233 add_prefix_space (`bool`, *optional*, defaults to `True`):234 Whether or not to add an initial space to the input. This allows to treat the leading word just as any235 other word.236 237 238 Attributes:239 sp_model (`SentencePieceProcessor`):240 The *SentencePiece* processor that is used for every conversion (string, tokens and IDs).241 """242 243 vocab_files_names = VOCAB_FILES_NAMES244 model_input_names = ["input_ids", "attention_mask"]245 246 def __init__(247 self,248 vocab_file,249 eos_token="</s>",250 unk_token="<unk>",251 sep_token="</s>",252 pad_token="<pad>",253 sep_token_box=[1000, 1000, 1000, 1000],254 pad_token_box=[0, 0, 0, 0],255 pad_token_label=-100,256 only_label_first_subword=True,257 additional_special_tokens=None,258 sp_model_kwargs: Optional[dict[str, Any]] = None,259 legacy=True,260 add_prefix_space=True,261 **kwargs,262 ) -> None:263 eos_token = AddedToken(eos_token, special=True) if isinstance(eos_token, str) else eos_token264 unk_token = AddedToken(unk_token, special=True) if isinstance(unk_token, str) else unk_token265 sep_token = AddedToken(sep_token, special=True) if isinstance(sep_token, str) else sep_token266 pad_token = AddedToken(pad_token, special=True) if isinstance(pad_token, str) else pad_token267 268 self.legacy = legacy269 self.add_prefix_space = add_prefix_space270 self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs271 272 self.vocab_file = vocab_file273 274 self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)275 self.sp_model.Load(vocab_file)276 277 # additional properties278 self.sep_token_box = sep_token_box279 self.pad_token_box = pad_token_box280 self.pad_token_label = pad_token_label281 self.only_label_first_subword = only_label_first_subword282 283 super().__init__(284 eos_token=eos_token,285 unk_token=unk_token,286 sep_token=sep_token,287 pad_token=pad_token,288 sep_token_box=sep_token_box,289 pad_token_box=pad_token_box,290 pad_token_label=pad_token_label,291 only_label_first_subword=only_label_first_subword,292 additional_special_tokens=additional_special_tokens,293 sp_model_kwargs=self.sp_model_kwargs,294 legacy=legacy,295 add_prefix_space=add_prefix_space,296 **kwargs,297 )298 299 @property300 def vocab_size(self):301 return len(self.sp_model)302 303 # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.get_vocab304 def get_vocab(self):305 vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}306 vocab.update(self.added_tokens_encoder)307 return vocab308 309 # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.get_special_tokens_mask310 def get_special_tokens_mask(311 self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None, already_has_special_tokens: bool = False312 ) -> list[int]:313 """314 Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding315 special tokens using the tokenizer `prepare_for_model` method.316 317 Args:318 token_ids_0 (`list[int]`):319 List of IDs.320 token_ids_1 (`list[int]`, *optional*):321 Optional second list of IDs for sequence pairs.322 already_has_special_tokens (`bool`, *optional*, defaults to `False`):323 Whether or not the token list is already formatted with special tokens for the model.324 325 Returns:326 `list[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.327 """328 if already_has_special_tokens:329 return super().get_special_tokens_mask(330 token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True331 )332 333 # normal case: some special tokens334 if token_ids_1 is None:335 return ([0] * len(token_ids_0)) + [1]336 return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]337 338 # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.get_sentinel_tokens339 def get_sentinel_tokens(self):340 return list(341 set(filter(lambda x: bool(re.search(r"<extra_id_\d+>", x)) is not None, self.additional_special_tokens))342 )343 344 # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.get_sentinel_token_ids345 def get_sentinel_token_ids(self):346 return [self.convert_tokens_to_ids(token) for token in self.get_sentinel_tokens()]347 348 # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer._add_eos_if_not_present349 def _add_eos_if_not_present(self, token_ids: list[int]) -> list[int]:350 """Do not add eos again if user already added it."""351 if len(token_ids) > 0 and token_ids[-1] == self.eos_token_id:352 warnings.warn(353 f"This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"354 " eos tokens being added."355 )356 return token_ids357 else:358 return token_ids + [self.eos_token_id]359 360 # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.create_token_type_ids_from_sequences361 def create_token_type_ids_from_sequences(362 self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None363 ) -> list[int]:364 """365 Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make366 use of token type ids, therefore a list of zeros is returned.367 368 Args:369 token_ids_0 (`list[int]`):370 List of IDs.371 token_ids_1 (`list[int]`, *optional*):372 Optional second list of IDs for sequence pairs.373 374 Returns:375 `list[int]`: List of zeros.376 """377 eos = [self.eos_token_id]378 379 if token_ids_1 is None:380 return len(token_ids_0 + eos) * [0]381 return len(token_ids_0 + eos + token_ids_1 + eos) * [0]382 383 # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.build_inputs_with_special_tokens384 def build_inputs_with_special_tokens(385 self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None386 ) -> list[int]:387 """388 Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and389 adding special tokens. A sequence has the following format:390 391 - single sequence: `X </s>`392 - pair of sequences: `A </s> B </s>`393 394 Args:395 token_ids_0 (`list[int]`):396 List of IDs to which the special tokens will be added.397 token_ids_1 (`list[int]`, *optional*):398 Optional second list of IDs for sequence pairs.399 400 Returns:401 `list[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.402 """403 token_ids_0 = self._add_eos_if_not_present(token_ids_0)404 if token_ids_1 is None:405 return token_ids_0406 else:407 token_ids_1 = self._add_eos_if_not_present(token_ids_1)408 return token_ids_0 + token_ids_1409 410 # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.__getstate__411 def __getstate__(self):412 state = self.__dict__.copy()413 state["sp_model"] = None414 return state415 416 def __setstate__(self, d):417 self.__dict__.update(d)418 self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)419 self.sp_model.Load(self.vocab_file)420 421 # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.tokenize422 def tokenize(self, text: "TextInput", **kwargs) -> list[str]:423 """424 Converts a string to a list of tokens. If `self.legacy` is set to `False`, a prefix token is added unless the425 first token is special.426 """427 if self.legacy or len(text) == 0:428 return super().tokenize(text, **kwargs)429 430 text = text.replace(SPIECE_UNDERLINE, " ")431 if self.add_prefix_space:432 text = SPIECE_UNDERLINE + text433 434 tokens = super().tokenize(text, **kwargs)435 436 if len(tokens) > 1 and tokens[0] == SPIECE_UNDERLINE and tokens[1] in self.all_special_tokens:437 tokens = tokens[1:]438 return tokens439 440 # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer._tokenize441 def _tokenize(self, text, **kwargs):442 """443 Returns a tokenized string.444 445 We de-activated the `add_dummy_prefix` option, thus the sentencepiece internals will always strip any446 SPIECE_UNDERLINE. For example: `self.sp_model.encode(f"{SPIECE_UNDERLINE}Hey", out_type = str)` will give447 `['H', 'e', 'y']` instead of `['▁He', 'y']`. Thus we always encode `f"{unk_token}text"` and strip the448 `unk_token`. Here is an example with `unk_token = "<unk>"` and `unk_token_length = 4`.449 `self.tokenizer.sp_model.encode("<unk> Hey", out_type = str)[4:]`.450 """451 if self.legacy or not text.startswith((SPIECE_UNDERLINE, " ")):452 return self.sp_model.encode(text, out_type=str)453 454 # 1. Encode string + prefix ex: "<unk> Hey"455 tokens = self.sp_model.encode(self.unk_token + text, out_type=str)456 # 2. Remove self.unk_token from ['<','unk','>', '▁Hey']457 return tokens[self.unk_token_length :] if len(tokens) >= self.unk_token_length else tokens458 459 def _convert_token_to_id(self, token):460 """Converts a token (str) in an id using the vocab."""461 return self.sp_model.piece_to_id(token)462 463 def _convert_id_to_token(self, index):464 """Converts an index (integer) in a token (str) using the vocab."""465 return self.sp_model.IdToPiece(index)466 467 # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.convert_tokens_to_string468 def convert_tokens_to_string(self, tokens):469 """Converts a sequence of tokens (string) in a single string."""470 # since we manually add the prefix space, we have to remove it when decoding471 if tokens[0].startswith(SPIECE_UNDERLINE) and self.add_prefix_space:472 tokens[0] = tokens[0][1:]473 474 current_sub_tokens = []475 out_string = ""476 prev_is_special = False477 for token in tokens:478 # make sure that special tokens are not decoded using sentencepiece model479 if token in self.all_special_tokens:480 if not prev_is_special:481 out_string += " "482 out_string += self.sp_model.decode(current_sub_tokens) + token483 prev_is_special = True484 current_sub_tokens = []485 else:486 current_sub_tokens.append(token)487 prev_is_special = False488 out_string += self.sp_model.decode(current_sub_tokens)489 return out_string.strip()490 491 # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.save_vocabulary492 def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:493 if not os.path.isdir(save_directory):494 logger.error(f"Vocabulary path ({save_directory}) should be a directory")495 return496 out_vocab_file = os.path.join(497 save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]498 )499 500 if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):501 copyfile(self.vocab_file, out_vocab_file)502 elif not os.path.isfile(self.vocab_file):503 with open(out_vocab_file, "wb") as fi:504 content_spiece_model = self.sp_model.serialized_model_proto()505 fi.write(content_spiece_model)506 507 return (out_vocab_file,)508 509 @add_end_docstrings(UDOP_ENCODE_KWARGS_DOCSTRING)510 def __call__(511 self,512 text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]] = None,513 text_pair: Optional[Union[PreTokenizedInput, list[PreTokenizedInput]]] = None,514 boxes: Optional[Union[list[list[int]], list[list[list[int]]]]] = None,515 word_labels: Optional[Union[list[int], list[list[int]]]] = None,516 text_target: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]] = None,517 text_pair_target: Optional[518 Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]]519 ] = None,520 **kwargs,521 ) -> BatchEncoding:522 if text is None and text_target is None:523 raise ValueError("You need to specify either `text` or `text_target`.")524 if text is not None:525 # The context manager will send the inputs as normal texts and not text_target, but we shouldn't change the526 # input mode in this case.527 if not self._in_target_context_manager:528 self._switch_to_input_mode()529 encodings = self.call_boxes(text=text, text_pair=text_pair, boxes=boxes, word_labels=word_labels, **kwargs)530 if text_target is not None:531 self._switch_to_target_mode()532 target_encodings = self._call_one(text=text_target, text_pair=text_pair_target, **kwargs)533 # Leave back tokenizer in input mode534 self._switch_to_input_mode()535 536 if text_target is None:537 return encodings538 elif text is None:539 return target_encodings540 else:541 encodings["labels"] = target_encodings["input_ids"]542 return encodings543 544 def call_boxes(545 self,546 text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]],547 text_pair: Optional[Union[PreTokenizedInput, list[PreTokenizedInput]]] = None,548 boxes: Optional[Union[list[list[int]], list[list[list[int]]]]] = None,549 word_labels: Optional[Union[list[int], list[list[int]]]] = None,550 add_special_tokens: bool = True,551 padding: Union[bool, str, PaddingStrategy] = False,552 truncation: Union[bool, str, TruncationStrategy] = None,553 max_length: Optional[int] = None,554 stride: int = 0,555 pad_to_multiple_of: Optional[int] = None,556 padding_side: Optional[str] = None,557 return_tensors: Optional[Union[str, TensorType]] = None,558 return_token_type_ids: Optional[bool] = None,559 return_attention_mask: Optional[bool] = None,560 return_overflowing_tokens: bool = False,561 return_special_tokens_mask: bool = False,562 return_offsets_mapping: bool = False,563 return_length: bool = False,564 verbose: bool = True,565 **kwargs,566 ) -> BatchEncoding:567 """568 Main method to tokenize and prepare for the model one or several sequence(s) or one or several pair(s) of569 sequences with word-level normalized bounding boxes and optional labels.570 571 Args:572 text (`str`, `list[str]`, `list[list[str]]`):573 The sequence or batch of sequences to be encoded. Each sequence can be a string, a list of strings574 (words of a single example or questions of a batch of examples) or a list of list of strings (batch of575 words).576 text_pair (`list[str]`, `list[list[str]]`):577 The sequence or batch of sequences to be encoded. Each sequence should be a list of strings578 (pretokenized string).579 boxes (`list[list[int]]`, `list[list[list[int]]]`):580 Word-level bounding boxes. Each bounding box should be normalized to be on a 0-1000 scale.581 word_labels (`list[int]`, `list[list[int]]`, *optional*):582 Word-level integer labels (for token classification tasks such as FUNSD, CORD).583 """584 585 # Input type checking for clearer error586 def _is_valid_text_input(t):587 if isinstance(t, str):588 # Strings are fine589 return True590 elif isinstance(t, (list, tuple)):591 # List are fine as long as they are...592 if len(t) == 0:593 # ... empty594 return True595 elif isinstance(t[0], str):596 # ... list of strings597 return True598 elif isinstance(t[0], (list, tuple)):599 # ... list with an empty list or with a list of strings600 return len(t[0]) == 0 or isinstance(t[0][0], str)601 else:602 return False603 else:604 return False605 606 if text_pair is not None:607 # in case text + text_pair are provided, text = questions, text_pair = words608 if not _is_valid_text_input(text):609 raise ValueError("text input must of type `str` (single example) or `list[str]` (batch of examples). ")610 if not isinstance(text_pair, (list, tuple)):611 raise ValueError(612 "words must of type `list[str]` (single pretokenized example), "613 "or `list[list[str]]` (batch of pretokenized examples)."614 )615 else:616 # in case only text is provided => must be words617 if not isinstance(text, (list, tuple)):618 raise ValueError(619 "Words must of type `list[str]` (single pretokenized example), "620 "or `list[list[str]]` (batch of pretokenized examples)."621 )622 623 if text_pair is not None:624 is_batched = isinstance(text, (list, tuple))625 else:626 is_batched = isinstance(text, (list, tuple)) and text and isinstance(text[0], (list, tuple))627 628 words = text if text_pair is None else text_pair629 if boxes is None:630 raise ValueError("You must provide corresponding bounding boxes")631 if is_batched:632 if len(words) != len(boxes):633 raise ValueError("You must provide words and boxes for an equal amount of examples")634 for words_example, boxes_example in zip(words, boxes):635 if len(words_example) != len(boxes_example):636 raise ValueError("You must provide as many words as there are bounding boxes")637 else:638 if len(words) != len(boxes):639 raise ValueError("You must provide as many words as there are bounding boxes")640 641 if is_batched:642 if text_pair is not None and len(text) != len(text_pair):643 raise ValueError(644 f"batch length of `text`: {len(text)} does not match batch length of `text_pair`:"645 f" {len(text_pair)}."646 )647 batch_text_or_text_pairs = list(zip(text, text_pair)) if text_pair is not None else text648 is_pair = bool(text_pair is not None)649 return self.batch_encode_plus_boxes(650 batch_text_or_text_pairs=batch_text_or_text_pairs,651 is_pair=is_pair,652 boxes=boxes,653 word_labels=word_labels,654 add_special_tokens=add_special_tokens,655 padding=padding,656 truncation=truncation,657 max_length=max_length,658 stride=stride,659 pad_to_multiple_of=pad_to_multiple_of,660 padding_side=padding_side,661 return_tensors=return_tensors,662 return_token_type_ids=return_token_type_ids,663 return_attention_mask=return_attention_mask,664 return_overflowing_tokens=return_overflowing_tokens,665 return_special_tokens_mask=return_special_tokens_mask,666 return_offsets_mapping=return_offsets_mapping,667 return_length=return_length,668 verbose=verbose,669 **kwargs,670 )671 else:672 return self.encode_plus_boxes(673 text=text,674 text_pair=text_pair,675 boxes=boxes,676 word_labels=word_labels,677 add_special_tokens=add_special_tokens,678 padding=padding,679 truncation=truncation,680 max_length=max_length,681 stride=stride,682 pad_to_multiple_of=pad_to_multiple_of,683 padding_side=padding_side,684 return_tensors=return_tensors,685 return_token_type_ids=return_token_type_ids,686 return_attention_mask=return_attention_mask,687 return_overflowing_tokens=return_overflowing_tokens,688 return_special_tokens_mask=return_special_tokens_mask,689 return_offsets_mapping=return_offsets_mapping,690 return_length=return_length,691 verbose=verbose,692 **kwargs,693 )694 695 def batch_encode_plus_boxes(696 self,697 batch_text_or_text_pairs: Union[698 list[TextInput],699 list[TextInputPair],700 list[PreTokenizedInput],701 ],702 is_pair: Optional[bool] = None,703 boxes: Optional[list[list[list[int]]]] = None,704 word_labels: Optional[list[list[int]]] = None,705 add_special_tokens: bool = True,706 padding: Union[bool, str, PaddingStrategy] = False,707 truncation: Union[bool, str, TruncationStrategy] = None,708 max_length: Optional[int] = None,709 stride: int = 0,710 is_split_into_words: bool = False,711 pad_to_multiple_of: Optional[int] = None,712 padding_side: Optional[str] = None,713 return_tensors: Optional[Union[str, TensorType]] = None,714 return_token_type_ids: Optional[bool] = None,715 return_attention_mask: Optional[bool] = None,716 return_overflowing_tokens: bool = False,717 return_special_tokens_mask: bool = False,718 return_offsets_mapping: bool = False,719 return_length: bool = False,720 verbose: bool = True,721 **kwargs,722 ) -> BatchEncoding:723 """724 Tokenize and prepare for the model a list of sequences or a list of pairs of sequences.725 726 Args:727 batch_text_or_text_pairs (`list[str]`, `list[tuple[str, str]]`, `list[list[str]]`, `list[tuple[list[str], list[str]]]`, and for not-fast tokenizers, also `list[list[int]]`, `list[tuple[list[int], list[int]]]`):728 Batch of sequences or pair of sequences to be encoded. This can be a list of729 string/string-sequences/int-sequences or a list of pair of string/string-sequences/int-sequence (see730 details in `encode_plus`).731 """732 733 # Backward compatibility for 'truncation_strategy', 'pad_to_max_length'734 padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(735 padding=padding,736 truncation=truncation,737 max_length=max_length,738 pad_to_multiple_of=pad_to_multiple_of,739 verbose=verbose,740 **kwargs,741 )742 743 return self._batch_encode_plus_boxes(744 batch_text_or_text_pairs=batch_text_or_text_pairs,745 is_pair=is_pair,746 boxes=boxes,747 word_labels=word_labels,748 add_special_tokens=add_special_tokens,749 padding_strategy=padding_strategy,750 truncation_strategy=truncation_strategy,751 max_length=max_length,752 stride=stride,753 is_split_into_words=is_split_into_words,754 pad_to_multiple_of=pad_to_multiple_of,755 padding_side=padding_side,756 return_tensors=return_tensors,757 return_token_type_ids=return_token_type_ids,758 return_attention_mask=return_attention_mask,759 return_overflowing_tokens=return_overflowing_tokens,760 return_special_tokens_mask=return_special_tokens_mask,761 return_offsets_mapping=return_offsets_mapping,762 return_length=return_length,763 verbose=verbose,764 **kwargs,765 )766 767 def encode_boxes(768 self,769 text: Union[TextInput, PreTokenizedInput, EncodedInput],770 text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None,771 boxes: Optional[list[list[int]]] = None,772 word_labels: Optional[list[list[int]]] = None,773 add_special_tokens: bool = True,774 padding: Union[bool, str, PaddingStrategy] = False,775 truncation: Union[bool, str, TruncationStrategy] = None,776 max_length: Optional[int] = None,777 stride: int = 0,778 return_tensors: Optional[Union[str, TensorType]] = None,779 **kwargs,780 ) -> list[int]:781 """782 Args:783 Converts a string to a sequence of ids (integer), using the tokenizer and vocabulary. Same as doing784 `self.convert_tokens_to_ids(self.tokenize(text))`.785 text (`str`, `list[str]` or `list[int]`):786 The first sequence to be encoded. This can be a string, a list of strings (tokenized string using the787 `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`788 method).789 text_pair (`str`, `list[str]` or `list[int]`, *optional*):790 Optional second sequence to be encoded. This can be a string, a list of strings (tokenized string using791 the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`792 method).793 """794 encoded_inputs = self.encode_plus_boxes(795 text,796 text_pair=text_pair,797 boxes=boxes,798 word_labels=word_labels,799 add_special_tokens=add_special_tokens,800 padding=padding,801 truncation=truncation,802 max_length=max_length,803 stride=stride,804 return_tensors=return_tensors,805 **kwargs,806 )807 808 return encoded_inputs["input_ids"]809 810 def encode_plus_boxes(811 self,812 text: Union[TextInput, PreTokenizedInput],813 text_pair: Optional[PreTokenizedInput] = None,814 boxes: Optional[list[list[int]]] = None,815 word_labels: Optional[list[list[int]]] = None,816 add_special_tokens: bool = True,817 padding: Union[bool, str, PaddingStrategy] = False,818 truncation: Union[bool, str, TruncationStrategy] = None,819 max_length: Optional[int] = None,820 stride: int = 0,821 is_split_into_words: bool = False,822 pad_to_multiple_of: Optional[int] = None,823 padding_side: Optional[str] = None,824 return_tensors: Optional[Union[str, TensorType]] = None,825 return_token_type_ids: Optional[bool] = None,826 return_attention_mask: Optional[bool] = None,827 return_overflowing_tokens: bool = False,828 return_special_tokens_mask: bool = False,829 return_offsets_mapping: bool = False,830 return_length: bool = False,831 verbose: bool = True,832 **kwargs,833 ) -> BatchEncoding:834 """835 Tokenize and prepare for the model a sequence or a pair of sequences.836 837 <Tip warning={true}>838 839 This method is deprecated, `__call__` should be used instead.840 841 </Tip>842 843 Args:844 text (`str`, `list[str]` or (for non-fast tokenizers) `list[int]`):845 The first sequence to be encoded. This can be a string, a list of strings (tokenized string using the846 `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`847 method).848 text_pair (`str`, `list[str]` or `list[int]`, *optional*):849 Optional second sequence to be encoded. This can be a string, a list of strings (tokenized string using850 the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`851 method).852 """853 854 # Backward compatibility for 'truncation_strategy', 'pad_to_max_length'855 padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(856 padding=padding,857 truncation=truncation,858 max_length=max_length,859 pad_to_multiple_of=pad_to_multiple_of,860 verbose=verbose,861 **kwargs,862 )863 864 return self._encode_plus_boxes(865 text=text,866 text_pair=text_pair,867 boxes=boxes,868 word_labels=word_labels,869 add_special_tokens=add_special_tokens,870 padding_strategy=padding_strategy,871 truncation_strategy=truncation_strategy,872 max_length=max_length,873 stride=stride,874 is_split_into_words=is_split_into_words,875 pad_to_multiple_of=pad_to_multiple_of,876 padding_side=padding_side,877 return_tensors=return_tensors,878 return_token_type_ids=return_token_type_ids,879 return_attention_mask=return_attention_mask,880 return_overflowing_tokens=return_overflowing_tokens,881 return_special_tokens_mask=return_special_tokens_mask,882 return_offsets_mapping=return_offsets_mapping,883 return_length=return_length,884 verbose=verbose,885 **kwargs,886 )887 888 def _batch_encode_plus_boxes(889 self,890 batch_text_or_text_pairs: Union[891 list[TextInput],892 list[TextInputPair],893 list[PreTokenizedInput],894 ],895 is_pair: Optional[bool] = None,896 boxes: Optional[list[list[list[int]]]] = None,897 word_labels: Optional[list[list[int]]] = None,898 add_special_tokens: bool = True,899 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,900 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,901 max_length: Optional[int] = None,902 stride: int = 0,903 pad_to_multiple_of: Optional[int] = None,904 padding_side: Optional[str] = None,905 return_tensors: Optional[Union[str, TensorType]] = None,906 return_token_type_ids: Optional[bool] = None,907 return_attention_mask: Optional[bool] = None,908 return_overflowing_tokens: bool = False,909 return_special_tokens_mask: bool = False,910 return_offsets_mapping: bool = False,911 return_length: bool = False,912 verbose: bool = True,913 **kwargs,914 ) -> BatchEncoding:915 if return_offsets_mapping:916 raise NotImplementedError(917 "return_offset_mapping is not available when using Python tokenizers. "918 "To use this feature, change your tokenizer to one deriving from "919 "transformers.PreTrainedTokenizerFast."920 )921 922 batch_outputs = self._batch_prepare_for_model_boxes(923 batch_text_or_text_pairs=batch_text_or_text_pairs,924 is_pair=is_pair,925 boxes=boxes,926 word_labels=word_labels,927 add_special_tokens=add_special_tokens,928 padding_strategy=padding_strategy,929 truncation_strategy=truncation_strategy,930 max_length=max_length,931 stride=stride,932 pad_to_multiple_of=pad_to_multiple_of,933 padding_side=padding_side,934 return_attention_mask=return_attention_mask,935 return_token_type_ids=return_token_type_ids,936 return_overflowing_tokens=return_overflowing_tokens,937 return_special_tokens_mask=return_special_tokens_mask,938 return_length=return_length,939 return_tensors=return_tensors,940 verbose=verbose,941 )942 943 return BatchEncoding(batch_outputs)944 945 @add_end_docstrings(UDOP_ENCODE_KWARGS_DOCSTRING)946 def _batch_prepare_for_model_boxes(947 self,948 batch_text_or_text_pairs,949 is_pair: Optional[bool] = None,950 boxes: Optional[list[list[int]]] = None,951 word_labels: Optional[list[list[int]]] = None,952 add_special_tokens: bool = True,953 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,954 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,955 max_length: Optional[int] = None,956 stride: int = 0,957 pad_to_multiple_of: Optional[int] = None,958 padding_side: Optional[str] = None,959 return_tensors: Optional[str] = None,960 return_token_type_ids: Optional[bool] = None,961 return_attention_mask: Optional[bool] = None,962 return_overflowing_tokens: bool = False,963 return_special_tokens_mask: bool = False,964 return_length: bool = False,965 verbose: bool = True,966 ) -> BatchEncoding:967 """968 Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It969 adds special tokens, truncates sequences if overflowing while taking into account the special tokens and970 manages a moving window (with user defined stride) for overflowing tokens971 972 Args:973 batch_ids_pairs: list of tokenized input ids or input ids pairs974 """975 976 batch_outputs = {}977 for idx, example in enumerate(zip(batch_text_or_text_pairs, boxes)):978 batch_text_or_text_pair, boxes_example = example979 outputs = self.prepare_for_model_boxes(980 batch_text_or_text_pair[0] if is_pair else batch_text_or_text_pair,981 batch_text_or_text_pair[1] if is_pair else None,982 boxes_example,983 word_labels=word_labels[idx] if word_labels is not None else None,984 add_special_tokens=add_special_tokens,985 padding=PaddingStrategy.DO_NOT_PAD.value, # we pad in batch afterward986 truncation=truncation_strategy.value,987 max_length=max_length,988 stride=stride,989 pad_to_multiple_of=None, # we pad in batch afterward990 padding_side=None, # we pad in batch afterward991 return_attention_mask=False, # we pad in batch afterward992 return_token_type_ids=return_token_type_ids,993 return_overflowing_tokens=return_overflowing_tokens,994 return_special_tokens_mask=return_special_tokens_mask,995 return_length=return_length,996 return_tensors=None, # We convert the whole batch to tensors at the end997 prepend_batch_axis=False,998 verbose=verbose,999 )1000 1001 for key, value in outputs.items():1002 if key not in batch_outputs:1003 batch_outputs[key] = []1004 batch_outputs[key].append(value)1005 1006 batch_outputs = self.pad(1007 batch_outputs,1008 padding=padding_strategy.value,1009 max_length=max_length,1010 pad_to_multiple_of=pad_to_multiple_of,1011 padding_side=padding_side,1012 return_attention_mask=return_attention_mask,1013 )1014 1015 batch_outputs = BatchEncoding(batch_outputs, tensor_type=return_tensors)1016 1017 return batch_outputs1018 1019 def _encode_plus_boxes(1020 self,1021 text: Union[TextInput, PreTokenizedInput],1022 text_pair: Optional[PreTokenizedInput] = None,1023 boxes: Optional[list[list[int]]] = None,1024 word_labels: Optional[list[int]] = None,1025 add_special_tokens: bool = True,1026 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,1027 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,1028 max_length: Optional[int] = None,1029 stride: int = 0,1030 pad_to_multiple_of: Optional[int] = None,1031 padding_side: Optional[str] = None,1032 return_tensors: Optional[Union[str, TensorType]] = None,1033 return_token_type_ids: Optional[bool] = None,1034 return_attention_mask: Optional[bool] = None,1035 return_overflowing_tokens: bool = False,1036 return_special_tokens_mask: bool = False,1037 return_offsets_mapping: bool = False,1038 return_length: bool = False,1039 verbose: bool = True,1040 **kwargs,1041 ) -> BatchEncoding:1042 if return_offsets_mapping:1043 raise NotImplementedError(1044 "return_offset_mapping is not available when using Python tokenizers. "1045 "To use this feature, change your tokenizer to one deriving from "1046 "transformers.PreTrainedTokenizerFast. "1047 "More information on available tokenizers at "1048 "https://github.com/huggingface/transformers/pull/2674"1049 )1050 1051 return self.prepare_for_model_boxes(1052 text=text,1053 text_pair=text_pair,1054 boxes=boxes,1055 word_labels=word_labels,1056 add_special_tokens=add_special_tokens,1057 padding=padding_strategy.value,1058 truncation=truncation_strategy.value,1059 max_length=max_length,1060 stride=stride,1061 pad_to_multiple_of=pad_to_multiple_of,1062 padding_side=padding_side,1063 return_tensors=return_tensors,1064 prepend_batch_axis=True,1065 return_attention_mask=return_attention_mask,1066 return_token_type_ids=return_token_type_ids,1067 return_overflowing_tokens=return_overflowing_tokens,1068 return_special_tokens_mask=return_special_tokens_mask,1069 return_length=return_length,1070 verbose=verbose,1071 )1072 1073 @add_end_docstrings(UDOP_ENCODE_KWARGS_DOCSTRING)1074 def prepare_for_model_boxes(1075 self,1076 text: Union[TextInput, PreTokenizedInput],1077 text_pair: Optional[PreTokenizedInput] = None,1078 boxes: Optional[list[list[int]]] = None,1079 word_labels: Optional[list[int]] = None,1080 add_special_tokens: bool = True,1081 padding: Union[bool, str, PaddingStrategy] = False,1082 truncation: Union[bool, str, TruncationStrategy] = None,1083 max_length: Optional[int] = None,1084 stride: int = 0,1085 pad_to_multiple_of: Optional[int] = None,1086 padding_side: Optional[str] = None,1087 return_tensors: Optional[Union[str, TensorType]] = None,1088 return_token_type_ids: Optional[bool] = None,1089 return_attention_mask: Optional[bool] = None,1090 return_overflowing_tokens: bool = False,1091 return_special_tokens_mask: bool = False,1092 return_offsets_mapping: bool = False,1093 return_length: bool = False,1094 verbose: bool = True,1095 prepend_batch_axis: bool = False,1096 **kwargs,1097 ) -> BatchEncoding:1098 """1099 Prepares a sequence or a pair of sequences so that it can be used by the model. It adds special tokens,1100 truncates sequences if overflowing while taking into account the special tokens and manages a moving window1101 (with user defined stride) for overflowing tokens.1102 1103 Word-level `boxes` are turned into token-level `bbox`. If provided, word-level `word_labels` are turned into1104 token-level `labels`. The word label is used for the first token of the word, while remaining tokens are1105 labeled with -100, such that they will be ignored by the loss function.1106 1107 Args:1108 text (`str`, `list[str]`, `list[list[str]]`):1109 The first sequence to be encoded. This can be a string, a list of strings or a list of list of strings.1110 text_pair (`list[str]` or `list[int]`, *optional*):1111 Optional second sequence to be encoded. This can be a list of strings (words of a single example) or a1112 list of list of strings (words of a batch of examples).1113 """1114 1115 # Backward compatibility for 'truncation_strategy', 'pad_to_max_length'1116 padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(1117 padding=padding,1118 truncation=truncation,1119 max_length=max_length,1120 pad_to_multiple_of=pad_to_multiple_of,1121 verbose=verbose,1122 **kwargs,1123 )1124 1125 tokens = []1126 pair_tokens = []1127 token_boxes = []1128 pair_token_boxes = []1129 labels = []1130 1131 if text_pair is None:1132 if word_labels is None:1133 # CASE 1: document image classification (training + inference) + CASE 2: token classification (inference)1134 for word, box in zip(text, boxes):1135 if len(word) < 1: # skip empty words1136 continue1137 word_tokens = self.tokenize(word)1138 tokens.extend(word_tokens)1139 token_boxes.extend([box] * len(word_tokens))1140 else:1141 # CASE 2: token classification (training)1142 for word, box, label in zip(text, boxes, word_labels):1143 if len(word) < 1: # skip empty words1144 continue1145 word_tokens = self.tokenize(word)1146 tokens.extend(word_tokens)1147 token_boxes.extend([box] * len(word_tokens))1148 if self.only_label_first_subword:1149 # Use the real label id for the first token of the word, and padding ids for the remaining tokens1150 labels.extend([label] + [self.pad_token_label] * (len(word_tokens) - 1))1151 else:1152 labels.extend([label] * len(word_tokens))1153 else:1154 # CASE 3: document visual question answering (inference)1155 # text = question1156 # text_pair = words1157 tokens = self.tokenize(text)1158 token_boxes = [self.pad_token_box for _ in range(len(tokens))]1159 1160 for word, box in zip(text_pair, boxes):1161 if len(word) < 1: # skip empty words1162 continue1163 word_tokens = self.tokenize(word)1164 pair_tokens.extend(word_tokens)1165 pair_token_boxes.extend([box] * len(word_tokens))1166 1167 # Create ids + pair_ids1168 ids = self.convert_tokens_to_ids(tokens)1169 pair_ids = self.convert_tokens_to_ids(pair_tokens) if pair_tokens else None1170 1171 # Compute the total size of the returned encodings1172 pair = bool(pair_ids is not None)1173 len_ids = len(ids)1174 len_pair_ids = len(pair_ids) if pair else 01175 total_len = len_ids + len_pair_ids + (self.num_special_tokens_to_add(pair=pair) if add_special_tokens else 0)1176 1177 # Truncation: Handle max sequence length1178 overflowing_tokens = []1179 overflowing_token_boxes = []1180 overflowing_labels = []1181 if truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE and max_length and total_len > max_length:1182 (1183 ids,1184 token_boxes,1185 pair_ids,1186 pair_token_boxes,1187 labels,1188 overflowing_tokens,1189 overflowing_token_boxes,1190 overflowing_labels,1191 ) = self.truncate_sequences(1192 ids,1193 token_boxes,1194 pair_ids=pair_ids,1195 pair_token_boxes=pair_token_boxes,1196 labels=labels,1197 num_tokens_to_remove=total_len - max_length,1198 truncation_strategy=truncation_strategy,1199 stride=stride,1200 )