chendl/compositional_test
1
1# coding=utf-82# Copyright 2019 HuggingFace Inc.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16 17import inspect18import itertools19import json20import os21import pickle22import re23import shutil24import sys25import tempfile26import unittest27import unittest.mock as mock28from collections import OrderedDict29from itertools import takewhile30from pathlib import Path31from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union32 33from huggingface_hub import HfFolder, delete_repo34from huggingface_hub.file_download import http_get35from parameterized import parameterized36from requests.exceptions import HTTPError37 38from transformers import (39 AlbertTokenizer,40 AlbertTokenizerFast,41 AutoTokenizer,42 BertTokenizer,43 BertTokenizerFast,44 GPT2TokenizerFast,45 PreTrainedTokenizer,46 PreTrainedTokenizerBase,47 PreTrainedTokenizerFast,48 SpecialTokensMixin,49 Trainer,50 TrainingArguments,51 is_flax_available,52 is_tf_available,53 is_tokenizers_available,54 is_torch_available,55 logging,56)57from transformers.testing_utils import (58 TOKEN,59 USER,60 check_json_file_has_correct_format,61 get_tests_dir,62 is_pt_tf_cross_test,63 is_staging_test,64 require_tf,65 require_tokenizers,66 require_torch,67 slow,68)69from transformers.tokenization_utils import AddedToken, Trie70 71 72if is_torch_available():73 import torch.nn as nn74 75 76if TYPE_CHECKING:77 from transformers import PretrainedConfig, PreTrainedModel, TFPreTrainedModel78 79 80sys.path.append(str(Path(__file__).parent.parent / "utils"))81 82from test_module.custom_tokenization import CustomTokenizer # noqa E40283 84 85if is_tokenizers_available():86 from test_module.custom_tokenization_fast import CustomTokenizerFast87 88 89logger = logging.get_logger(__name__)90 91NON_ENGLISH_TAGS = ["chinese", "dutch", "french", "finnish", "german", "multilingual"]92 93SMALL_TRAINING_CORPUS = [94 ["This is the first sentence.", "This is the second one."],95 ["This sentence (contains #) over symbols and numbers 12 3.", "But not this one."],96]97 98 99def filter_non_english(_, pretrained_name: str):100 """Filter all the model for non-english language"""101 return not any([lang in pretrained_name for lang in NON_ENGLISH_TAGS])102 103 104def filter_roberta_detectors(_, pretrained_name: str):105 return "detector" not in pretrained_name106 107 108def merge_model_tokenizer_mappings(109 model_mapping: Dict["PretrainedConfig", Union["PreTrainedModel", "TFPreTrainedModel"]],110 tokenizer_mapping: Dict["PretrainedConfig", Tuple["PreTrainedTokenizer", "PreTrainedTokenizerFast"]],111) -> Dict[112 Union["PreTrainedTokenizer", "PreTrainedTokenizerFast"],113 Tuple["PretrainedConfig", Union["PreTrainedModel", "TFPreTrainedModel"]],114]:115 configurations = list(model_mapping.keys())116 model_tokenizer_mapping = OrderedDict([])117 118 for configuration in configurations:119 if configuration in model_mapping and configuration in tokenizer_mapping:120 model = model_mapping[configuration]121 tokenizer = tokenizer_mapping[configuration][0]122 tokenizer_fast = tokenizer_mapping[configuration][1]123 124 if tokenizer is not None:125 if configuration.__name__.startswith(tokenizer.__name__.replace("Tokenizer", "")):126 model_tokenizer_mapping.update({tokenizer: (configuration, model)})127 if tokenizer_fast is not None:128 if configuration.__name__.startswith(tokenizer_fast.__name__.replace("TokenizerFast", "")):129 model_tokenizer_mapping.update({tokenizer_fast: (configuration, model)})130 131 return model_tokenizer_mapping132 133 134class TokenizerTesterMixin:135 tokenizer_class = None136 rust_tokenizer_class = None137 test_slow_tokenizer = True138 test_rust_tokenizer = True139 space_between_special_tokens = False140 from_pretrained_kwargs = None141 from_pretrained_filter = None142 from_pretrained_vocab_key = "vocab_file"143 test_seq2seq = True144 145 # set to True to test a sentencepiece tokenizer146 test_sentencepiece = False147 148 # set to True to ignore casing when testing a sentencepiece tokenizer149 # test_sentencepiece must also be set to True150 test_sentencepiece_ignore_case = False151 152 def setUp(self) -> None:153 # Tokenizer.filter makes it possible to filter which Tokenizer to case based on all the154 # information available in Tokenizer (name, rust class, python class, vocab key name)155 if self.test_rust_tokenizer:156 tokenizers_list = [157 (158 self.rust_tokenizer_class,159 pretrained_name,160 self.from_pretrained_kwargs if self.from_pretrained_kwargs is not None else {},161 )162 for pretrained_name in self.rust_tokenizer_class.pretrained_vocab_files_map[163 self.from_pretrained_vocab_key164 ].keys()165 if self.from_pretrained_filter is None166 or (self.from_pretrained_filter is not None and self.from_pretrained_filter(pretrained_name))167 ]168 self.tokenizers_list = tokenizers_list[:1] # Let's just test the first pretrained vocab for speed169 else:170 self.tokenizers_list = []171 with open(f"{get_tests_dir()}/fixtures/sample_text.txt", encoding="utf-8") as f_data:172 self._data = f_data.read().replace("\n\n", "\n").strip()173 174 self.tmpdirname = tempfile.mkdtemp()175 176 def tearDown(self):177 shutil.rmtree(self.tmpdirname)178 179 def get_input_output_texts(self, tokenizer):180 input_txt = self.get_clean_sequence(tokenizer)[0]181 return input_txt, input_txt182 183 def get_clean_sequence(self, tokenizer, with_prefix_space=False, max_length=20, min_length=5) -> Tuple[str, list]:184 toks = [(i, tokenizer.decode([i], clean_up_tokenization_spaces=False)) for i in range(len(tokenizer))]185 toks = list(filter(lambda t: re.match(r"^[ a-zA-Z]+$", t[1]), toks))186 toks = list(filter(lambda t: [t[0]] == tokenizer.encode(t[1], add_special_tokens=False), toks))187 if max_length is not None and len(toks) > max_length:188 toks = toks[:max_length]189 if min_length is not None and len(toks) < min_length and len(toks) > 0:190 while len(toks) < min_length:191 toks = toks + toks192 # toks_str = [t[1] for t in toks]193 toks_ids = [t[0] for t in toks]194 195 # Ensure consistency196 output_txt = tokenizer.decode(toks_ids, clean_up_tokenization_spaces=False)197 if " " not in output_txt and len(toks_ids) > 1:198 output_txt = (199 tokenizer.decode([toks_ids[0]], clean_up_tokenization_spaces=False)200 + " "201 + tokenizer.decode(toks_ids[1:], clean_up_tokenization_spaces=False)202 )203 if with_prefix_space:204 output_txt = " " + output_txt205 output_ids = tokenizer.encode(output_txt, add_special_tokens=False)206 return output_txt, output_ids207 208 def get_tokenizers(self, fast=True, **kwargs) -> List[PreTrainedTokenizerBase]:209 if fast and self.test_rust_tokenizer and self.test_slow_tokenizer:210 return [self.get_tokenizer(**kwargs), self.get_rust_tokenizer(**kwargs)]211 elif fast and self.test_rust_tokenizer:212 return [self.get_rust_tokenizer(**kwargs)]213 elif self.test_slow_tokenizer:214 return [self.get_tokenizer(**kwargs)]215 else:216 raise ValueError("This tokenizer class has no tokenizer to be tested.")217 218 def get_tokenizer(self, **kwargs) -> PreTrainedTokenizer:219 return self.tokenizer_class.from_pretrained(self.tmpdirname, **kwargs)220 221 def get_rust_tokenizer(self, **kwargs) -> PreTrainedTokenizerFast:222 return self.rust_tokenizer_class.from_pretrained(self.tmpdirname, **kwargs)223 224 def tokenizer_integration_test_util(225 self,226 expected_encoding: Dict,227 model_name: str,228 revision: str = None,229 sequences: List[str] = None,230 decode_kwargs: Dict[str, Any] = None,231 padding: bool = True,232 ):233 """234 Util for integration test.235 236 Text is tokenized and then reverted back to text. Both results are then checked.237 238 Args:239 expected_encoding:240 The expected result of the tokenizer output.241 model_name:242 The model name of the tokenizer to load and use.243 revision:244 The full git revision number of the model. This is to pin the245 tokenizer config and to avoid that tests start to fail if the246 config gets changed upstream.247 sequences:248 Can overwrite the texts that are used to check the tokenizer.249 This is useful if the tokenizer supports non english languages250 like france.251 decode_kwargs:252 Additional args for the ``decode`` function which reverts the253 tokenized text back to a string.254 padding:255 Activates and controls padding of the tokenizer.256 """257 decode_kwargs = {} if decode_kwargs is None else decode_kwargs258 259 if sequences is None:260 sequences = [261 "Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides "262 "general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural "263 "Language Understanding (NLU) and Natural Language Generation (NLG) with over 32+ pretrained "264 "models in 100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.",265 "BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly "266 "conditioning on both left and right context in all layers.",267 "The quick brown fox jumps over the lazy dog.",268 ]269 270 if self.test_sentencepiece_ignore_case:271 sequences = [sequence.lower() for sequence in sequences]272 273 tokenizer_classes = [self.tokenizer_class]274 if self.test_rust_tokenizer:275 tokenizer_classes.append(self.rust_tokenizer_class)276 277 for tokenizer_class in tokenizer_classes:278 tokenizer = tokenizer_class.from_pretrained(279 model_name,280 revision=revision, # to pin the tokenizer version281 )282 283 encoding = tokenizer(sequences, padding=padding)284 decoded_sequences = [285 tokenizer.decode(seq, skip_special_tokens=True, **decode_kwargs) for seq in encoding["input_ids"]286 ]287 288 encoding_data = encoding.data289 self.assertDictEqual(encoding_data, expected_encoding)290 291 for expected, decoded in zip(sequences, decoded_sequences):292 if self.test_sentencepiece_ignore_case:293 expected = expected.lower()294 self.assertEqual(expected, decoded)295 296 def assert_padded_input_match(self, input_r: list, input_p: list, max_length: int, pad_token_id: int):297 # Ensure we match max_length298 self.assertEqual(len(input_r), max_length)299 self.assertEqual(len(input_p), max_length)300 301 # Ensure the number of padded tokens is the same302 padded_tokens_r = list(takewhile(lambda i: i == pad_token_id, reversed(input_r)))303 padded_tokens_p = list(takewhile(lambda i: i == pad_token_id, reversed(input_p)))304 self.assertSequenceEqual(padded_tokens_r, padded_tokens_p)305 306 def assert_batch_padded_input_match(307 self,308 input_r: dict,309 input_p: dict,310 max_length: int,311 pad_token_id: int,312 model_main_input_name: str = "input_ids",313 ):314 for i_r in input_r.values():315 self.assertEqual(len(i_r), 2), self.assertEqual(len(i_r[0]), max_length), self.assertEqual(316 len(i_r[1]), max_length317 )318 self.assertEqual(len(i_r), 2), self.assertEqual(len(i_r[0]), max_length), self.assertEqual(319 len(i_r[1]), max_length320 )321 322 for i_r, i_p in zip(input_r[model_main_input_name], input_p[model_main_input_name]):323 self.assert_padded_input_match(i_r, i_p, max_length, pad_token_id)324 325 for i_r, i_p in zip(input_r["attention_mask"], input_p["attention_mask"]):326 self.assertSequenceEqual(i_r, i_p)327 328 @staticmethod329 def convert_batch_encode_plus_format_to_encode_plus(batch_encode_plus_sequences):330 # Switch from batch_encode_plus format: {'input_ids': [[...], [...]], ...}331 # to the list of examples/ encode_plus format: [{'input_ids': [...], ...}, {'input_ids': [...], ...}]332 return [333 {value: batch_encode_plus_sequences[value][i] for value in batch_encode_plus_sequences.keys()}334 for i in range(len(batch_encode_plus_sequences["input_ids"]))335 ]336 337 # TODO: this test can be combined with `test_sentencepiece_tokenize_and_convert_tokens_to_string` after the latter is extended to all tokenizers.338 def test_tokenize_special_tokens(self):339 """Test `tokenize` with special tokens."""340 tokenizers = self.get_tokenizers(fast=True, do_lower_case=True)341 for tokenizer in tokenizers:342 with self.subTest(f"{tokenizer.__class__.__name__}"):343 SPECIAL_TOKEN_1 = "[SPECIAL_TOKEN_1]"344 SPECIAL_TOKEN_2 = "[SPECIAL_TOKEN_2]"345 346 # TODO:347 # Can we combine `unique_no_split_tokens` and `all_special_tokens`(and properties related to it)348 # with one variable(property) for a better maintainability?349 350 # `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)351 tokenizer.add_tokens([SPECIAL_TOKEN_1], special_tokens=True)352 # `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,353 # which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)354 tokenizer.add_special_tokens({"additional_special_tokens": [SPECIAL_TOKEN_2]})355 356 token_1 = tokenizer.tokenize(SPECIAL_TOKEN_1)357 token_2 = tokenizer.tokenize(SPECIAL_TOKEN_2)358 359 self.assertEqual(len(token_1), 1)360 self.assertEqual(len(token_2), 1)361 self.assertEqual(token_1[0], SPECIAL_TOKEN_1)362 self.assertEqual(token_2[0], SPECIAL_TOKEN_2)363 364 # TODO: this test could be extended to all tokenizers - not just the sentencepiece365 def test_sentencepiece_tokenize_and_convert_tokens_to_string(self):366 """Test ``_tokenize`` and ``convert_tokens_to_string``."""367 if not self.test_sentencepiece:368 return369 370 tokenizer = self.get_tokenizer()371 text = "This is text to test the tokenizer."372 373 if self.test_sentencepiece_ignore_case:374 text = text.lower()375 376 tokens = tokenizer.tokenize(text)377 378 self.assertTrue(len(tokens) > 0)379 380 # check if converting back to original text works381 reverse_text = tokenizer.convert_tokens_to_string(tokens)382 383 if self.test_sentencepiece_ignore_case:384 reverse_text = reverse_text.lower()385 386 self.assertEqual(reverse_text, text)387 388 special_tokens = tokenizer.all_special_tokens389 special_tokens_string = tokenizer.convert_tokens_to_string(special_tokens)390 for special_token in special_tokens:391 self.assertIn(special_token, special_tokens_string)392 393 if self.test_rust_tokenizer:394 rust_tokenizer = self.get_rust_tokenizer()395 special_tokens_string_rust = rust_tokenizer.convert_tokens_to_string(special_tokens)396 self.assertEqual(special_tokens_string, special_tokens_string_rust)397 398 def test_sentencepiece_tokenize_and_decode(self):399 if not self.test_sentencepiece:400 return401 402 text = "This is text to test the tokenizer."403 if self.test_rust_tokenizer:404 tokenizer = self.get_tokenizer()405 rust_tokenizer = self.get_rust_tokenizer()406 407 slow_ids = tokenizer(text).input_ids408 fast_ids = rust_tokenizer(text).input_ids409 self.assertEqual(slow_ids, fast_ids)410 411 slow_decoded = tokenizer.decode(slow_ids)412 fast_decoded = rust_tokenizer.decode(slow_ids)413 self.assertEqual(slow_decoded, fast_decoded)414 415 def test_subword_regularization_tokenizer(self) -> None:416 if not self.test_sentencepiece:417 return418 419 # Subword regularization is only available for the slow tokenizer.420 sp_model_kwargs = {"enable_sampling": True, "alpha": 0.1, "nbest_size": -1}421 tokenizer = self.get_tokenizer(sp_model_kwargs=sp_model_kwargs)422 423 self.assertTrue(hasattr(tokenizer, "sp_model_kwargs"))424 self.assertIsNotNone(tokenizer.sp_model_kwargs)425 self.assertTrue(isinstance(tokenizer.sp_model_kwargs, dict))426 self.assertEqual(tokenizer.sp_model_kwargs, sp_model_kwargs)427 self.check_subword_sampling(tokenizer)428 429 def test_pickle_subword_regularization_tokenizer(self) -> None:430 if not self.test_sentencepiece:431 return432 433 """Google pickle __getstate__ __setstate__ if you are struggling with this."""434 # Subword regularization is only available for the slow tokenizer.435 sp_model_kwargs = {"enable_sampling": True, "alpha": 0.1, "nbest_size": -1}436 tokenizer = self.get_tokenizer(sp_model_kwargs=sp_model_kwargs)437 tokenizer_bin = pickle.dumps(tokenizer)438 del tokenizer439 tokenizer_new = pickle.loads(tokenizer_bin)440 441 self.assertTrue(hasattr(tokenizer_new, "sp_model_kwargs"))442 self.assertIsNotNone(tokenizer_new.sp_model_kwargs)443 self.assertTrue(isinstance(tokenizer_new.sp_model_kwargs, dict))444 self.assertEqual(tokenizer_new.sp_model_kwargs, sp_model_kwargs)445 self.check_subword_sampling(tokenizer_new)446 447 def test_save_sentencepiece_tokenizer(self) -> None:448 if not self.test_sentencepiece or not self.test_slow_tokenizer:449 return450 # We want to verify that we will be able to save the tokenizer even if the original files that were used to451 # build the tokenizer have been deleted in the meantime.452 text = "This is text to test the tokenizer."453 454 tokenizer_slow_1 = self.get_tokenizer()455 encoding_tokenizer_slow_1 = tokenizer_slow_1(text)456 457 tmpdirname_1 = tempfile.mkdtemp()458 tmpdirname_2 = tempfile.mkdtemp()459 460 tokenizer_slow_1.save_pretrained(tmpdirname_1)461 tokenizer_slow_2 = self.tokenizer_class.from_pretrained(tmpdirname_1)462 encoding_tokenizer_slow_2 = tokenizer_slow_2(text)463 464 shutil.rmtree(tmpdirname_1)465 tokenizer_slow_2.save_pretrained(tmpdirname_2)466 467 tokenizer_slow_3 = self.tokenizer_class.from_pretrained(tmpdirname_2)468 encoding_tokenizer_slow_3 = tokenizer_slow_3(text)469 shutil.rmtree(tmpdirname_2)470 471 self.assertEqual(encoding_tokenizer_slow_1, encoding_tokenizer_slow_2)472 self.assertEqual(encoding_tokenizer_slow_1, encoding_tokenizer_slow_3)473 474 def test_model_input_names_signature(self):475 accepted_model_main_input_names = [476 "input_ids", # nlp models477 "input_values", # speech models478 ]479 480 tokenizers = self.get_tokenizers()481 for tokenizer in tokenizers:482 # first name of model_input_names has to correspond to main model input name483 # to make sure `tokenizer.pad(...)` works correctly484 self.assertTrue(tokenizer.model_input_names[0] in accepted_model_main_input_names)485 486 def test_rust_tokenizer_signature(self):487 if not self.test_rust_tokenizer:488 return489 490 signature = inspect.signature(self.rust_tokenizer_class.__init__)491 492 self.assertIn("tokenizer_file", signature.parameters)493 self.assertIsNone(signature.parameters["tokenizer_file"].default)494 495 def test_tokenizer_slow_store_full_signature(self):496 if not self.test_slow_tokenizer:497 return498 499 signature = inspect.signature(self.tokenizer_class.__init__)500 tokenizer = self.get_tokenizer()501 502 for parameter_name, parameter in signature.parameters.items():503 if parameter.default != inspect.Parameter.empty:504 self.assertIn(parameter_name, tokenizer.init_kwargs)505 506 def test_tokenizer_fast_store_full_signature(self):507 if not self.test_rust_tokenizer:508 return509 510 signature = inspect.signature(self.rust_tokenizer_class.__init__)511 tokenizer = self.get_rust_tokenizer()512 513 for parameter_name, parameter in signature.parameters.items():514 if parameter.default != inspect.Parameter.empty and parameter_name not in [515 "vocab_file",516 "merges_file",517 "tokenizer_file",518 ]:519 self.assertIn(parameter_name, tokenizer.init_kwargs)520 521 def test_rust_and_python_full_tokenizers(self):522 if not self.test_rust_tokenizer:523 return524 525 if not self.test_slow_tokenizer:526 # as we don't have a slow version, we can't compare the outputs between slow and fast versions527 return528 529 tokenizer = self.get_tokenizer()530 rust_tokenizer = self.get_rust_tokenizer()531 532 sequence, _ = self.get_input_output_texts(tokenizer)533 534 # We don't have an exact equivalence on `tokenize()` between Rust and Slow535 # Slow tokenizer only split tokens, Rust tokenizers will replace with <unk>536 # tokens = tokenizer.tokenize(sequence)537 # rust_tokens = rust_tokenizer.tokenize(sequence)538 # self.assertListEqual(tokens, rust_tokens)539 540 ids = tokenizer.encode(sequence, add_special_tokens=False)541 rust_ids = rust_tokenizer.encode(sequence, add_special_tokens=False)542 self.assertListEqual(ids, rust_ids)543 544 ids = tokenizer.encode(sequence, add_special_tokens=True)545 rust_ids = rust_tokenizer.encode(sequence, add_special_tokens=True)546 self.assertListEqual(ids, rust_ids)547 548 def test_tokenizers_common_properties(self):549 tokenizers = self.get_tokenizers()550 for tokenizer in tokenizers:551 with self.subTest(f"{tokenizer.__class__.__name__}"):552 attributes_list = [553 "bos_token",554 "eos_token",555 "unk_token",556 "sep_token",557 "pad_token",558 "cls_token",559 "mask_token",560 ]561 for attr in attributes_list:562 self.assertTrue(hasattr(tokenizer, attr))563 self.assertTrue(hasattr(tokenizer, attr + "_id"))564 565 self.assertTrue(hasattr(tokenizer, "additional_special_tokens"))566 self.assertTrue(hasattr(tokenizer, "additional_special_tokens_ids"))567 568 attributes_list = [569 "model_max_length",570 "init_inputs",571 "init_kwargs",572 ]573 if not isinstance(tokenizer, PreTrainedTokenizerFast):574 attributes_list += [575 "added_tokens_encoder",576 "added_tokens_decoder",577 ]578 for attr in attributes_list:579 self.assertTrue(hasattr(tokenizer, attr))580 581 def test_tokenizers_common_ids_setters(self):582 tokenizers = self.get_tokenizers()583 for tokenizer in tokenizers:584 with self.subTest(f"{tokenizer.__class__.__name__}"):585 attributes_list = [586 "bos_token",587 "eos_token",588 "unk_token",589 "sep_token",590 "pad_token",591 "cls_token",592 "mask_token",593 ]594 595 vocab = tokenizer.get_vocab()596 token_id_to_test_setters = next(iter(vocab.values()))597 token_to_test_setters = tokenizer.convert_ids_to_tokens(598 token_id_to_test_setters, skip_special_tokens=False599 )600 601 for attr in attributes_list:602 setattr(tokenizer, attr + "_id", None)603 self.assertEqual(getattr(tokenizer, attr), None)604 self.assertEqual(getattr(tokenizer, attr + "_id"), None)605 606 setattr(tokenizer, attr + "_id", token_id_to_test_setters)607 self.assertEqual(getattr(tokenizer, attr), token_to_test_setters)608 self.assertEqual(getattr(tokenizer, attr + "_id"), token_id_to_test_setters)609 610 setattr(tokenizer, "additional_special_tokens_ids", [])611 self.assertListEqual(getattr(tokenizer, "additional_special_tokens"), [])612 self.assertListEqual(getattr(tokenizer, "additional_special_tokens_ids"), [])613 614 setattr(tokenizer, "additional_special_tokens_ids", [token_id_to_test_setters])615 self.assertListEqual(getattr(tokenizer, "additional_special_tokens"), [token_to_test_setters])616 self.assertListEqual(getattr(tokenizer, "additional_special_tokens_ids"), [token_id_to_test_setters])617 618 @parameterized.expand([(True,), (False,)])619 def test_tokenizers_special_tokens_properties_unset(self, verbose):620 tokenizers = self.get_tokenizers(verbose=verbose)621 for tokenizer in tokenizers:622 with self.subTest(f"{tokenizer.__class__.__name__}"):623 attributes_list = [624 "bos_token",625 "eos_token",626 "unk_token",627 "sep_token",628 "pad_token",629 "cls_token",630 "mask_token",631 "additional_special_tokens",632 ]633 for attr in attributes_list:634 setattr(tokenizer, attr, None)635 self.assertIsNone(getattr(tokenizer, attr))636 637 def test_save_and_load_tokenizer(self):638 # safety check on max_len default value so we are sure the test works639 tokenizers = self.get_tokenizers()640 for tokenizer in tokenizers:641 with self.subTest(f"{tokenizer.__class__.__name__}"):642 self.assertNotEqual(tokenizer.model_max_length, 42)643 644 # Now let's start the test645 tokenizers = self.get_tokenizers()646 for tokenizer in tokenizers:647 with self.subTest(f"{tokenizer.__class__.__name__}"):648 # Isolate this from the other tests because we save additional tokens/etc649 tmpdirname = tempfile.mkdtemp()650 651 sample_text = " He is very happy, UNwant\u00E9d,running"652 before_tokens = tokenizer.encode(sample_text, add_special_tokens=False)653 before_vocab = tokenizer.get_vocab()654 tokenizer.save_pretrained(tmpdirname)655 656 after_tokenizer = tokenizer.__class__.from_pretrained(tmpdirname)657 after_tokens = after_tokenizer.encode(sample_text, add_special_tokens=False)658 after_vocab = after_tokenizer.get_vocab()659 self.assertListEqual(before_tokens, after_tokens)660 self.assertDictEqual(before_vocab, after_vocab)661 662 shutil.rmtree(tmpdirname)663 664 tokenizers = self.get_tokenizers(model_max_length=42)665 for tokenizer in tokenizers:666 with self.subTest(f"{tokenizer.__class__.__name__}"):667 # Isolate this from the other tests because we save additional tokens/etc668 tmpdirname = tempfile.mkdtemp()669 670 sample_text = " He is very happy, UNwant\u00E9d,running"671 tokenizer.add_tokens(["bim", "bambam"])672 additional_special_tokens = tokenizer.additional_special_tokens673 additional_special_tokens.append("new_additional_special_token")674 tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})675 before_tokens = tokenizer.encode(sample_text, add_special_tokens=False)676 before_vocab = tokenizer.get_vocab()677 tokenizer.save_pretrained(tmpdirname)678 679 after_tokenizer = tokenizer.__class__.from_pretrained(tmpdirname)680 after_tokens = after_tokenizer.encode(sample_text, add_special_tokens=False)681 after_vocab = after_tokenizer.get_vocab()682 self.assertListEqual(before_tokens, after_tokens)683 self.assertDictEqual(before_vocab, after_vocab)684 self.assertIn("bim", after_vocab)685 self.assertIn("bambam", after_vocab)686 self.assertIn("new_additional_special_token", after_tokenizer.additional_special_tokens)687 self.assertEqual(after_tokenizer.model_max_length, 42)688 689 tokenizer = tokenizer.__class__.from_pretrained(tmpdirname, model_max_length=43)690 self.assertEqual(tokenizer.model_max_length, 43)691 692 shutil.rmtree(tmpdirname)693 694 # Test that we can also use the non-legacy saving format for fast tokenizers695 tokenizers = self.get_tokenizers(model_max_length=42)696 for tokenizer in tokenizers:697 if not tokenizer.is_fast:698 continue699 with self.subTest(f"{tokenizer.__class__.__name__}"):700 # Isolate this from the other tests because we save additional tokens/etc701 tmpdirname = tempfile.mkdtemp()702 703 sample_text = " He is very happy, UNwant\u00E9d,running"704 tokenizer.add_tokens(["bim", "bambam"])705 additional_special_tokens = tokenizer.additional_special_tokens706 additional_special_tokens.append("new_additional_special_token")707 tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})708 before_tokens = tokenizer.encode(sample_text, add_special_tokens=False)709 before_vocab = tokenizer.get_vocab()710 tokenizer.save_pretrained(tmpdirname)711 712 after_tokenizer = tokenizer.__class__.from_pretrained(tmpdirname)713 after_tokens = after_tokenizer.encode(sample_text, add_special_tokens=False)714 after_vocab = after_tokenizer.get_vocab()715 self.assertListEqual(before_tokens, after_tokens)716 self.assertDictEqual(before_vocab, after_vocab)717 self.assertIn("bim", after_vocab)718 self.assertIn("bambam", after_vocab)719 self.assertIn("new_additional_special_token", after_tokenizer.additional_special_tokens)720 self.assertEqual(after_tokenizer.model_max_length, 42)721 722 tokenizer = tokenizer.__class__.from_pretrained(tmpdirname, model_max_length=43)723 self.assertEqual(tokenizer.model_max_length, 43)724 725 shutil.rmtree(tmpdirname)726 727 def test_pickle_tokenizer(self):728 """Google pickle __getstate__ __setstate__ if you are struggling with this."""729 tokenizers = self.get_tokenizers()730 for tokenizer in tokenizers:731 with self.subTest(f"{tokenizer.__class__.__name__}"):732 self.assertIsNotNone(tokenizer)733 734 text = "Munich and Berlin are nice cities"735 subwords = tokenizer.tokenize(text)736 737 filename = os.path.join(self.tmpdirname, "tokenizer.bin")738 with open(filename, "wb") as handle:739 pickle.dump(tokenizer, handle)740 741 with open(filename, "rb") as handle:742 tokenizer_new = pickle.load(handle)743 744 subwords_loaded = tokenizer_new.tokenize(text)745 746 self.assertListEqual(subwords, subwords_loaded)747 748 @require_tokenizers749 def test_pickle_added_tokens(self):750 tok1 = AddedToken("<s>", rstrip=True, lstrip=True, normalized=False, single_word=True)751 tok2 = pickle.loads(pickle.dumps(tok1))752 753 self.assertEqual(tok1.__getstate__(), tok2.__getstate__())754 755 def test_added_tokens_do_lower_case(self):756 tokenizers = self.get_tokenizers(do_lower_case=True)757 for tokenizer in tokenizers:758 with self.subTest(f"{tokenizer.__class__.__name__}"):759 if not hasattr(tokenizer, "do_lower_case") or not tokenizer.do_lower_case:760 continue761 762 special_token = tokenizer.all_special_tokens[0]763 764 text = special_token + " aaaaa bbbbbb low cccccccccdddddddd l " + special_token765 text2 = special_token + " AAAAA BBBBBB low CCCCCCCCCDDDDDDDD l " + special_token766 767 toks_before_adding = tokenizer.tokenize(text) # toks before adding new_toks768 769 new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd", "AAAAA BBBBBB", "CCCCCCCCCDDDDDDDD"]770 added = tokenizer.add_tokens([AddedToken(tok, lstrip=True, rstrip=True) for tok in new_toks])771 772 toks_after_adding = tokenizer.tokenize(text)773 toks_after_adding2 = tokenizer.tokenize(text2)774 775 # Rust tokenizers dont't lowercase added tokens at the time calling `tokenizer.add_tokens`,776 # while python tokenizers do, so new_toks 0 and 2 would be treated as the same, so do new_toks 1 and 3.777 self.assertIn(added, [2, 4])778 779 self.assertListEqual(toks_after_adding, toks_after_adding2)780 self.assertTrue(781 len(toks_before_adding) > len(toks_after_adding), # toks_before_adding should be longer782 )783 784 # Check that none of the special tokens are lowercased785 sequence_with_special_tokens = "A " + " yEs ".join(tokenizer.all_special_tokens) + " B"786 # Convert the tokenized list to str as some special tokens are tokenized like normal tokens787 # which have a prefix spacee e.g. the mask token of Albert, and cannot match the original788 # special tokens exactly.789 tokenized_sequence = "".join(tokenizer.tokenize(sequence_with_special_tokens))790 791 for special_token in tokenizer.all_special_tokens:792 self.assertTrue(special_token in tokenized_sequence)793 794 tokenizers = self.get_tokenizers(do_lower_case=True)795 for tokenizer in tokenizers:796 with self.subTest(f"{tokenizer.__class__.__name__}"):797 if hasattr(tokenizer, "do_lower_case") and tokenizer.do_lower_case:798 continue799 800 special_token = tokenizer.all_special_tokens[0]801 802 text = special_token + " aaaaa bbbbbb low cccccccccdddddddd l " + special_token803 text2 = special_token + " AAAAA BBBBBB low CCCCCCCCCDDDDDDDD l " + special_token804 805 toks_before_adding = tokenizer.tokenize(text) # toks before adding new_toks806 807 new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd", "AAAAA BBBBBB", "CCCCCCCCCDDDDDDDD"]808 added = tokenizer.add_tokens([AddedToken(tok, lstrip=True, rstrip=True) for tok in new_toks])809 self.assertIn(added, [2, 4])810 811 toks_after_adding = tokenizer.tokenize(text)812 toks_after_adding2 = tokenizer.tokenize(text2)813 814 self.assertEqual(len(toks_after_adding), len(toks_after_adding2)) # Length should still be the same815 self.assertNotEqual(816 toks_after_adding[1], toks_after_adding2[1]817 ) # But at least the first non-special tokens should differ818 self.assertTrue(819 len(toks_before_adding) > len(toks_after_adding), # toks_before_adding should be longer820 )821 822 def test_add_tokens_tokenizer(self):823 tokenizers = self.get_tokenizers(do_lower_case=False)824 for tokenizer in tokenizers:825 with self.subTest(f"{tokenizer.__class__.__name__}"):826 vocab_size = tokenizer.vocab_size827 all_size = len(tokenizer)828 829 self.assertNotEqual(vocab_size, 0)830 831 # We usually have added tokens from the start in tests because our vocab fixtures are832 # smaller than the original vocabs - let's not assert this833 # self.assertEqual(vocab_size, all_size)834 835 new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd"]836 added_toks = tokenizer.add_tokens(new_toks)837 vocab_size_2 = tokenizer.vocab_size838 all_size_2 = len(tokenizer)839 840 self.assertNotEqual(vocab_size_2, 0)841 self.assertEqual(vocab_size, vocab_size_2)842 self.assertEqual(added_toks, len(new_toks))843 self.assertEqual(all_size_2, all_size + len(new_toks))844 845 tokens = tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l", add_special_tokens=False)846 847 self.assertGreaterEqual(len(tokens), 4)848 self.assertGreater(tokens[0], tokenizer.vocab_size - 1)849 self.assertGreater(tokens[-2], tokenizer.vocab_size - 1)850 851 new_toks_2 = {"eos_token": ">>>>|||<||<<|<<", "pad_token": "<<<<<|||>|>>>>|>"}852 added_toks_2 = tokenizer.add_special_tokens(new_toks_2)853 vocab_size_3 = tokenizer.vocab_size854 all_size_3 = len(tokenizer)855 856 self.assertNotEqual(vocab_size_3, 0)857 self.assertEqual(vocab_size, vocab_size_3)858 self.assertEqual(added_toks_2, len(new_toks_2))859 self.assertEqual(all_size_3, all_size_2 + len(new_toks_2))860 861 tokens = tokenizer.encode(862 ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l", add_special_tokens=False863 )864 865 self.assertGreaterEqual(len(tokens), 6)866 self.assertGreater(tokens[0], tokenizer.vocab_size - 1)867 self.assertGreater(tokens[0], tokens[1])868 self.assertGreater(tokens[-2], tokenizer.vocab_size - 1)869 self.assertGreater(tokens[-2], tokens[-3])870 self.assertEqual(tokens[0], tokenizer.eos_token_id)871 self.assertEqual(tokens[-2], tokenizer.pad_token_id)872 873 def test_add_special_tokens(self):874 tokenizers = self.get_tokenizers(do_lower_case=False)875 for tokenizer in tokenizers:876 with self.subTest(f"{tokenizer.__class__.__name__}"):877 input_text, ids = self.get_clean_sequence(tokenizer)878 879 special_token = "[SPECIAL_TOKEN]"880 881 tokenizer.add_special_tokens({"cls_token": special_token})882 encoded_special_token = tokenizer.encode(special_token, add_special_tokens=False)883 self.assertEqual(len(encoded_special_token), 1)884 885 text = tokenizer.decode(ids + encoded_special_token, clean_up_tokenization_spaces=False)886 encoded = tokenizer.encode(text, add_special_tokens=False)887 888 input_encoded = tokenizer.encode(input_text, add_special_tokens=False)889 special_token_id = tokenizer.encode(special_token, add_special_tokens=False)890 self.assertEqual(encoded, input_encoded + special_token_id)891 892 decoded = tokenizer.decode(encoded, skip_special_tokens=True)893 self.assertTrue(special_token not in decoded)894 895 def test_internal_consistency(self):896 tokenizers = self.get_tokenizers()897 for tokenizer in tokenizers:898 with self.subTest(f"{tokenizer.__class__.__name__}"):899 input_text, output_text = self.get_input_output_texts(tokenizer)900 901 tokens = tokenizer.tokenize(input_text)902 ids = tokenizer.convert_tokens_to_ids(tokens)903 ids_2 = tokenizer.encode(input_text, add_special_tokens=False)904 self.assertListEqual(ids, ids_2)905 906 tokens_2 = tokenizer.convert_ids_to_tokens(ids)907 self.assertNotEqual(len(tokens_2), 0)908 text_2 = tokenizer.decode(ids)909 self.assertIsInstance(text_2, str)910 911 self.assertEqual(text_2, output_text)912 913 @require_tokenizers914 def test_encode_decode_with_spaces(self):915 tokenizers = self.get_tokenizers(do_lower_case=False)916 for tokenizer in tokenizers:917 with self.subTest(f"{tokenizer.__class__.__name__}"):918 new_toks = [919 AddedToken("[ABC]", normalized=False),920 AddedToken("[DEF]", normalized=False),921 AddedToken("GHI IHG", normalized=False),922 ]923 tokenizer.add_tokens(new_toks)924 input = "[ABC][DEF][ABC]GHI IHG[DEF]"925 if self.space_between_special_tokens:926 output = "[ABC] [DEF] [ABC] GHI IHG [DEF]"927 else:928 output = input929 encoded = tokenizer.encode(input, add_special_tokens=False)930 decoded = tokenizer.decode(encoded, spaces_between_special_tokens=self.space_between_special_tokens)931 self.assertIn(decoded, [output, output.lower()])932 933 def test_pretrained_model_lists(self):934 # We should have at least one default checkpoint for each tokenizer935 # We should specify the max input length as well (used in some part to list the pretrained checkpoints)936 self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map), 1)937 self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0]), 1)938 self.assertEqual(939 len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0]),940 len(self.tokenizer_class.max_model_input_sizes),941 )942 943 weights_list = list(self.tokenizer_class.max_model_input_sizes.keys())944 weights_lists_2 = []945 for file_id, map_list in self.tokenizer_class.pretrained_vocab_files_map.items():946 weights_lists_2.append(list(map_list.keys()))947 948 for weights_list_2 in weights_lists_2:949 self.assertListEqual(weights_list, weights_list_2)950 951 def test_mask_output(self):952 tokenizers = self.get_tokenizers(do_lower_case=False)953 for tokenizer in tokenizers:954 with self.subTest(f"{tokenizer.__class__.__name__}"):955 if (956 tokenizer.build_inputs_with_special_tokens.__qualname__.split(".")[0] != "PreTrainedTokenizer"957 and "token_type_ids" in tokenizer.model_input_names958 ):959 seq_0 = "Test this method."960 seq_1 = "With these inputs."961 information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True)962 sequences, mask = information["input_ids"], information["token_type_ids"]963 self.assertEqual(len(sequences), len(mask))964 965 def test_token_type_ids(self):966 tokenizers = self.get_tokenizers()967 for tokenizer in tokenizers:968 with self.subTest(f"{tokenizer.__class__.__name__}"):969 seq_0 = "Test this method."970 971 # We want to have sequence 0 and sequence 1 are tagged972 # respectively with 0 and 1 token_ids973 # (regardless of whether the model use token type ids)974 # We use this assumption in the QA pipeline among other place975 output = tokenizer(seq_0, return_token_type_ids=True)976 self.assertIn(0, output["token_type_ids"])977 978 def test_sequence_ids(self):979 tokenizers = self.get_tokenizers()980 for tokenizer in tokenizers:981 if not tokenizer.is_fast:982 continue983 with self.subTest(f"{tokenizer.__class__.__name__}"):984 seq_0 = "Test this method."985 seq_1 = "With these inputs."986 987 # We want to have sequence 0 and sequence 1 are tagged988 # respectively with 0 and 1 token_ids989 # (regardless of whether the model use token type ids)990 # We use this assumption in the QA pipeline among other place991 output = tokenizer(seq_0)992 self.assertIn(0, output.sequence_ids())993 994 output = tokenizer(seq_0, seq_1)995 self.assertIn(0, output.sequence_ids())996 self.assertIn(1, output.sequence_ids())997 998 if tokenizer.num_special_tokens_to_add(pair=True):999 self.assertIn(None, output.sequence_ids())1000 1001 def test_number_of_added_tokens(self):1002 tokenizers = self.get_tokenizers(do_lower_case=False)1003 for tokenizer in tokenizers:1004 with self.subTest(f"{tokenizer.__class__.__name__}"):1005 seq_0 = "Test this method."1006 seq_1 = "With these inputs."1007 1008 sequences = tokenizer.encode(seq_0, seq_1, add_special_tokens=False)1009 attached_sequences = tokenizer.encode(seq_0, seq_1, add_special_tokens=True)1010 1011 # Method is implemented (e.g. not GPT-2)1012 if len(attached_sequences) != 2:1013 self.assertEqual(1014 tokenizer.num_special_tokens_to_add(pair=True), len(attached_sequences) - len(sequences)1015 )1016 1017 def test_maximum_encoding_length_single_input(self):1018 tokenizers = self.get_tokenizers(do_lower_case=False, model_max_length=100)1019 for tokenizer in tokenizers:1020 with self.subTest(f"{tokenizer.__class__.__name__}"):1021 seq_0, ids = self.get_clean_sequence(tokenizer, max_length=20)1022 1023 sequence = tokenizer.encode(seq_0, add_special_tokens=False)1024 total_length = len(sequence)1025 1026 self.assertGreater(1027 total_length, 4, "Issue with the testing sequence, please update it, it's too short"1028 )1029 1030 # Test with max model input length1031 model_max_length = tokenizer.model_max_length1032 self.assertEqual(model_max_length, 100)1033 seq_1 = seq_0 * model_max_length1034 1035 sequence1 = tokenizer(seq_1, add_special_tokens=False)1036 total_length1 = len(sequence1["input_ids"])1037 self.assertGreater(1038 total_length1,1039 model_max_length,1040 "Issue with the testing sequence, please update it, it's too short",1041 )1042 1043 # Simple1044 padding_strategies = (1045 [False, True, "longest"] if tokenizer.pad_token and tokenizer.pad_token_id >= 0 else [False]1046 )1047 for padding_state in padding_strategies:1048 with self.subTest(f"Padding: {padding_state}"):1049 for truncation_state in [True, "longest_first", "only_first"]:1050 with self.subTest(f"Truncation: {truncation_state}"):1051 output = tokenizer(seq_1, padding=padding_state, truncation=truncation_state)1052 self.assertEqual(len(output["input_ids"]), model_max_length)1053 1054 output = tokenizer([seq_1], padding=padding_state, truncation=truncation_state)1055 self.assertEqual(len(output["input_ids"][0]), model_max_length)1056 1057 # Simple with no truncation1058 # Reset warnings1059 tokenizer.deprecation_warnings = {}1060 with self.assertLogs("transformers", level="WARNING") as cm:1061 output = tokenizer(seq_1, padding=padding_state, truncation=False)1062 self.assertNotEqual(len(output["input_ids"]), model_max_length)1063 self.assertEqual(len(cm.records), 1)1064 self.assertTrue(1065 cm.records[0].message.startswith(1066 "Token indices sequence length is longer than the specified maximum sequence length"1067 " for this model"1068 )1069 )1070 1071 tokenizer.deprecation_warnings = {}1072 with self.assertLogs("transformers", level="WARNING") as cm:1073 output = tokenizer([seq_1], padding=padding_state, truncation=False)1074 self.assertNotEqual(len(output["input_ids"][0]), model_max_length)1075 self.assertEqual(len(cm.records), 1)1076 self.assertTrue(1077 cm.records[0].message.startswith(1078 "Token indices sequence length is longer than the specified maximum sequence length"1079 " for this model"1080 )1081 )1082 1083 # Overflowing tokens1084 stride = 21085 information = tokenizer(1086 seq_0,1087 max_length=total_length - 2,1088 add_special_tokens=False,1089 stride=stride,1090 truncation="longest_first",1091 return_overflowing_tokens=True,1092 # add_prefix_space=False,1093 )1094 1095 # Overflowing tokens are handled quite differently in slow and fast tokenizers1096 if isinstance(tokenizer, PreTrainedTokenizerFast):1097 truncated_sequence = information["input_ids"][0]1098 overflowing_tokens = information["input_ids"][1]1099 self.assertEqual(len(information["input_ids"]), 2)1100 1101 self.assertEqual(len(truncated_sequence), total_length - 2)1102 self.assertEqual(truncated_sequence, sequence[:-2])1103 1104 self.assertEqual(len(overflowing_tokens), 2 + stride)1105 self.assertEqual(overflowing_tokens, sequence[-(2 + stride) :])1106 else:1107 truncated_sequence = information["input_ids"]1108 overflowing_tokens = information["overflowing_tokens"]1109 1110 self.assertEqual(len(truncated_sequence), total_length - 2)1111 self.assertEqual(truncated_sequence, sequence[:-2])1112 1113 self.assertEqual(len(overflowing_tokens), 2 + stride)1114 self.assertEqual(overflowing_tokens, sequence[-(2 + stride) :])1115 1116 def test_maximum_encoding_length_pair_input(self):1117 tokenizers = self.get_tokenizers(do_lower_case=False, model_max_length=100)1118 for tokenizer in tokenizers:1119 with self.subTest(f"{tokenizer.__class__.__name__}"):1120 # Build a sequence from our model's vocabulary1121 stride = 21122 seq_0, ids = self.get_clean_sequence(tokenizer, max_length=20)1123 if len(ids) <= 2 + stride:1124 seq_0 = (seq_0 + " ") * (2 + stride)1125 ids = None1126 1127 seq0_tokens = tokenizer.encode(seq_0, add_special_tokens=False)1128 self.assertGreater(len(seq0_tokens), 2 + stride)1129 1130 seq_1 = "This is another sentence to be encoded."1131 seq1_tokens = tokenizer.encode(seq_1, add_special_tokens=False)1132 if abs(len(seq0_tokens) - len(seq1_tokens)) <= 2:1133 seq1_tokens = seq1_tokens + seq1_tokens1134 seq_1 = tokenizer.decode(seq1_tokens, clean_up_tokenization_spaces=False)1135 seq1_tokens = tokenizer.encode(seq_1, add_special_tokens=False)1136 1137 self.assertGreater(len(seq1_tokens), 2 + stride)1138 1139 smallest = seq1_tokens if len(seq0_tokens) > len(seq1_tokens) else seq0_tokens1140 1141 # We are not using the special tokens - a bit too hard to test all the tokenizers with this1142 # TODO try this again later1143 sequence = tokenizer.encode(seq_0, seq_1, add_special_tokens=False) # , add_prefix_space=False)1144 1145 # Test with max model input length1146 model_max_length = tokenizer.model_max_length1147 self.assertEqual(model_max_length, 100)1148 seq_2 = seq_0 * model_max_length1149 self.assertGreater(len(seq_2), model_max_length)1150 1151 sequence1 = tokenizer(seq_1, add_special_tokens=False)1152 total_length1 = len(sequence1["input_ids"])1153 sequence2 = tokenizer(seq_2, seq_1, add_special_tokens=False)1154 total_length2 = len(sequence2["input_ids"])1155 self.assertLess(1156 total_length1, model_max_length - 10, "Issue with the testing sequence, please update it."1157 )1158 self.assertGreater(1159 total_length2, model_max_length, "Issue with the testing sequence, please update it."1160 )1161 1162 # Simple1163 padding_strategies = (1164 [False, True, "longest"] if tokenizer.pad_token and tokenizer.pad_token_id >= 0 else [False]1165 )1166 for padding_state in padding_strategies:1167 with self.subTest(f"{tokenizer.__class__.__name__} Padding: {padding_state}"):1168 for truncation_state in [True, "longest_first", "only_first"]:1169 with self.subTest(f"{tokenizer.__class__.__name__} Truncation: {truncation_state}"):1170 output = tokenizer(seq_2, seq_1, padding=padding_state, truncation=truncation_state)1171 self.assertEqual(len(output["input_ids"]), model_max_length)1172 1173 output = tokenizer(1174 [seq_2], [seq_1], padding=padding_state, truncation=truncation_state1175 )1176 self.assertEqual(len(output["input_ids"][0]), model_max_length)1177 1178 # Simple1179 output = tokenizer(seq_1, seq_2, padding=padding_state, truncation="only_second")1180 self.assertEqual(len(output["input_ids"]), model_max_length)1181 1182 output = tokenizer([seq_1], [seq_2], padding=padding_state, truncation="only_second")1183 self.assertEqual(len(output["input_ids"][0]), model_max_length)1184 1185 # Simple with no truncation1186 # Reset warnings1187 tokenizer.deprecation_warnings = {}1188 with self.assertLogs("transformers", level="WARNING") as cm:1189 output = tokenizer(seq_1, seq_2, padding=padding_state, truncation=False)1190 self.assertNotEqual(len(output["input_ids"]), model_max_length)1191 self.assertEqual(len(cm.records), 1)1192 self.assertTrue(1193 cm.records[0].message.startswith(1194 "Token indices sequence length is longer than the specified maximum sequence length"1195 " for this model"1196 )1197 )1198 1199 tokenizer.deprecation_warnings = {}1200 with self.assertLogs("transformers", level="WARNING") as cm: