Aluode/PerceptionLabPortable
0
1# Copyright 2020 The HuggingFace Inc. team.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""15Tokenization classes for python tokenizers. For fast tokenizers (provided by HuggingFace's tokenizers library) see16tokenization_utils_fast.py17"""18 19import bisect20import itertools21import re22import unicodedata23from collections import OrderedDict24from typing import Any, Optional, Union, overload25 26from .tokenization_utils_base import (27 ENCODE_KWARGS_DOCSTRING,28 ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING,29 INIT_TOKENIZER_DOCSTRING,30 AddedToken,31 BatchEncoding,32 EncodedInput,33 EncodedInputPair,34 PreTokenizedInput,35 PreTokenizedInputPair,36 PreTrainedTokenizerBase,37 TextInput,38 TextInputPair,39 TruncationStrategy,40)41from .utils import PaddingStrategy, TensorType, add_end_docstrings, logging42 43 44logger = logging.get_logger(__name__)45 46# Slow tokenizers are saved in a vocabulary plus three separated files47SPECIAL_TOKENS_MAP_FILE = "special_tokens_map.json"48ADDED_TOKENS_FILE = "added_tokens.json"49TOKENIZER_CONFIG_FILE = "tokenizer_config.json"50 51 52class Trie:53 """54 Trie in Python. Creates a Trie out of a list of words. The trie is used to split on `added_tokens` in one pass55 Loose reference https://en.wikipedia.org/wiki/Trie56 """57 58 def __init__(self, *args):59 self.data = {}60 self._tokens = set()61 self._termination_char = ""62 self.update(*args)63 64 def update(self, *args):65 """66 Updates the Trie with new tokens provided as arguments.67 68 Args:69 *args: Variable number of words to be added to the Trie.70 """71 for token in tuple(*args):72 self.add(token)73 74 def add(self, word: str):75 """76 Passes over every char (utf-8 char) on word and recursively adds it to the internal `data` trie representation.77 The special key `""` in `self._termination_char` is used to represent termination.78 79 This function is idempotent, adding twice the same word will leave the trie unchanged80 81 Example:82 83 ```python84 >>> trie = Trie()85 >>> trie.add("Hello 友達")86 >>> trie.data87 {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}88 89 >>> trie.add("Hello")90 >>> trie.data91 {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}92 ```93 """94 if not word:95 # Prevent empty string96 return97 98 self._tokens.add(word)99 ref = self.data100 for char in word:101 ref[char] = ref.setdefault(char, {})102 ref = ref[char]103 ref[self._termination_char] = 1104 105 def split(self, text: str) -> list[str]:106 """107 Will look for the words added to the trie within `text`. Output is the original string split along the108 boundaries of the words found.109 110 This trie will match the longest possible word first !111 112 Example:113 114 ```python115 >>> trie = Trie()116 >>> trie.split("[CLS] This is a extra_id_100")117 ["[CLS] This is a extra_id_100"]118 119 >>> trie.add("[CLS]")120 >>> trie.add("extra_id_1")121 >>> trie.add("extra_id_100")122 >>> trie.split("[CLS] This is a extra_id_100")123 ["[CLS]", " This is a ", "extra_id_100"]124 ```125 """126 # indexes are counted left of the chars index.127 # "hello", index 0, is left of h, index 1 is between h and e.128 # index 5 is right of the "o".129 130 # States are going to capture every possible start (indexes as above)131 # as keys, and have as values, a pointer to the position in the trie132 # where we're at. This is a partial match for now.133 # This enables to keep track of multiple matches while we're iterating134 # the string135 # If the trie contains, "blowing", and "lower" and we encounter the136 # string "blower", we need to split into ["b", "lower"].137 # This is where we need to keep track of multiple possible starts.138 states = OrderedDict()139 140 # This will contain every indices where we need141 # to cut.142 # We force to cut at offset 0 and len(text) (added later)143 offsets = [0]144 145 # This is used by the lookahead which needs to skip over146 # some text where the full match exceeded the place in the initial147 # for loop148 skip = 0149 # Main loop, Giving this algorithm O(n) complexity150 for current, current_char in enumerate(text):151 if skip and current < skip:152 # Prevents the lookahead for matching twice153 # like extra_id_100 and id_100154 continue155 156 # This will track every state157 # that stop matching, we need to stop tracking them.158 # If we look at "lowball", we're going to match "l" (add it to states), "o", "w", then159 # fail on "b", we need to remove 0 from the valid states.160 to_remove = set()161 # Whenever we found a match, we need to drop everything162 # this is a greedy algorithm, it will match on the first found token163 reset = False164 165 # In this case, we already have partial matches (But unfinished)166 for start, trie_pointer in states.items():167 if "" in trie_pointer:168 # This is a final match, we need to reset and169 # store the results in `offsets`.170 171 # Lookahead to match longest first172 # Important in case of extra_id_1 vs extra_id_100173 # Here we are also actively looking for other earlier partial174 # matches175 # "[CLS]", "L", we need to match CLS even if L is special176 for lookstart, looktrie_pointer in states.items():177 if lookstart > start:178 # This partial match is later, we can stop looking179 break180 elif lookstart < start:181 # This partial match is earlier, the trie pointer182 # was already updated, so index is + 1183 lookahead_index = current + 1184 end = current + 1185 else:186 # Here lookstart == start and187 # looktrie_pointer == trie_pointer188 # It wasn't updated yet so indices are current ones189 lookahead_index = current190 end = current191 next_char = text[lookahead_index] if lookahead_index < len(text) else None192 if "" in looktrie_pointer:193 start = lookstart194 end = lookahead_index195 skip = lookahead_index196 197 while next_char in looktrie_pointer:198 looktrie_pointer = looktrie_pointer[next_char]199 lookahead_index += 1200 if "" in looktrie_pointer:201 start = lookstart202 end = lookahead_index203 skip = lookahead_index204 205 if lookahead_index == len(text):206 # End of string207 break208 next_char = text[lookahead_index]209 # End lookahead210 211 # Storing and resetting212 offsets.append(start)213 offsets.append(end)214 reset = True215 break216 elif current_char in trie_pointer:217 # The current character being looked at has a match within the trie218 # update the pointer (it will be stored back into states later).219 trie_pointer = trie_pointer[current_char]220 221 # Storing back the new pointer into the states.222 # Partial matches got longer by one.223 states[start] = trie_pointer224 else:225 # The new character has not match in the trie, we need226 # to stop keeping track of this partial match.227 # We can't do it directly within the loop because of how228 # python iteration works229 to_remove.add(start)230 231 # Either clearing the full start (we found a real match)232 # Or clearing only the partial matches that didn't work.233 if reset:234 states = {}235 else:236 for start in to_remove:237 del states[start]238 239 # If this character is a starting character within the trie240 # start keeping track of this partial match.241 if current >= skip and current_char in self.data:242 states[current] = self.data[current_char]243 244 # We have a cut at the end with states.245 for start, trie_pointer in states.items():246 if "" in trie_pointer:247 # This is a final match, we need to reset and248 # store the results in `offsets`.249 end = len(text)250 offsets.append(start)251 offsets.append(end)252 # Longest cut is always the one with lower start so the first253 # item so we need to break.254 break255 256 return self.cut_text(text, offsets)257 258 def cut_text(self, text, offsets):259 # We have all the offsets now, we just need to do the actual splitting.260 # We need to eventually add the first part of the string and the eventual261 # last part.262 offsets.append(len(text))263 tokens = []264 start = 0265 for end in offsets:266 if start > end:267 logger.error(268 "There was a bug in Trie algorithm in tokenization. Attempting to recover. Please report it"269 " anyway."270 )271 continue272 elif start == end:273 # This might happen if there's a match at index 0274 # we're also preventing zero-width cuts in case of two275 # consecutive matches276 continue277 tokens.append(text[start:end])278 start = end279 280 return tokens281 282 283class ExtensionsTrie(Trie):284 def __init__(self, *args):285 super().__init__(*args)286 287 def extensions(self, prefix: str):288 """289 Generates all extensions of a given prefix token in the Trie.290 291 Example:292 293 ```python294 >>> trie = Trie()295 >>> trie.add("apple")296 >>> trie.add("app")297 >>> trie.add("application")298 >>> trie.extensions("app")299 ['app', 'apple', 'application']300 ```301 """302 prefix_node = self._get_node(prefix)303 ret = self._collect_tokens(prefix_node)304 return [prefix + token for token in ret]305 306 def _get_node(self, token: str) -> dict:307 """308 Retrieves the node corresponding to the given token in the Trie.309 310 Args:311 token (str): The token for which the corresponding node needs to be retrieved.312 313 Returns:314 dict: The node in the Trie corresponding to the given token.315 """316 node = self.data317 for char in token:318 if char not in node:319 break320 321 node = node[char]322 return node323 324 def _collect_tokens(self, node: dict) -> list:325 """326 Generates all tokens in the Trie starting from a given node.327 328 Args:329 node (dict): The node in the Trie from which tokens need to be generated.330 331 Returns:332 list: List of tokens generated from the given node.333 """334 tokens = [self._termination_char] if self._termination_char in node else []335 for token, subtrie_head in node.items():336 if token != self._termination_char:337 subtokens = self._collect_tokens(subtrie_head)338 tokens.extend([token + subtoken for subtoken in subtokens])339 return tokens340 341 342def _is_whitespace(char):343 """Checks whether `char` is a whitespace character."""344 # \t, \n, and \r are technically control characters but we treat them345 # as whitespace since they are generally considered as such.346 if char == " " or char == "\t" or char == "\n" or char == "\r":347 return True348 cat = unicodedata.category(char)349 if cat == "Zs":350 return True351 return False352 353 354def _is_control(char):355 """Checks whether `char` is a control character."""356 # These are technically control characters but we count them as whitespace357 # characters.358 if char == "\t" or char == "\n" or char == "\r":359 return False360 cat = unicodedata.category(char)361 if cat.startswith("C"):362 return True363 return False364 365 366def _is_punctuation(char):367 """Checks whether `char` is a punctuation character."""368 cp = ord(char)369 # We treat all non-letter/number ASCII as punctuation.370 # Characters such as "^", "$", and "`" are not in the Unicode371 # Punctuation class but we treat them as punctuation anyways, for372 # consistency.373 if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):374 return True375 cat = unicodedata.category(char)376 if cat.startswith("P"):377 return True378 return False379 380 381def _is_end_of_word(text):382 """Checks whether the last character in text is one of a punctuation, control or whitespace character."""383 last_char = text[-1]384 return bool(_is_control(last_char) | _is_punctuation(last_char) | _is_whitespace(last_char))385 386 387def _is_start_of_word(text):388 """Checks whether the first character in text is one of a punctuation, control or whitespace character."""389 first_char = text[0]390 return bool(_is_control(first_char) | _is_punctuation(first_char) | _is_whitespace(first_char))391 392 393def _insert_one_token_to_ordered_list(token_list: list[str], new_token: str):394 """395 Inserts one token to an ordered list if it does not already exist. Note: token_list must be sorted.396 """397 insertion_idx = bisect.bisect_left(token_list, new_token)398 # Checks if new_token is already in the ordered token_list399 if insertion_idx < len(token_list) and token_list[insertion_idx] == new_token:400 # new_token is in token_list, don't add401 return402 else:403 token_list.insert(insertion_idx, new_token)404 405 406@add_end_docstrings(INIT_TOKENIZER_DOCSTRING)407class PreTrainedTokenizer(PreTrainedTokenizerBase):408 """409 Base class for all slow tokenizers.410 411 Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`].412 413 Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading414 pretrained tokenizers as well as adding tokens to the vocabulary.415 416 This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the417 specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).418 """419 420 def __init__(self, **kwargs):421 # 1. Init the parent class422 423 self.tokens_trie = Trie()424 425 # 2. init `_added_tokens_decoder` if child class did not426 if not hasattr(self, "_added_tokens_decoder"):427 self._added_tokens_decoder: dict[int, AddedToken] = {}428 429 # 3. if a `added_tokens_decoder` is passed, we are loading from a saved tokenizer, we overwrite430 self._added_tokens_decoder.update(kwargs.pop("added_tokens_decoder", {}))431 self._added_tokens_encoder: dict[str, int] = {k.content: v for v, k in self._added_tokens_decoder.items()}432 433 # 4 init the parent class434 super().__init__(**kwargs)435 436 # 4. If some of the special tokens are not part of the vocab, we add them, at the end.437 # the order of addition is the same as self.SPECIAL_TOKENS_ATTRIBUTES following `tokenizers`438 self._add_tokens(439 [token for token in self.all_special_tokens_extended if token not in self._added_tokens_encoder],440 special_tokens=True,441 )442 443 self._decode_use_source_tokenizer = False444 445 @property446 def is_fast(self) -> bool:447 return False448 449 @property450 def vocab_size(self) -> int:451 """452 `int`: Size of the base vocabulary (without the added tokens).453 """454 raise NotImplementedError455 456 @property457 def added_tokens_encoder(self) -> dict[str, int]:458 """459 Returns the sorted mapping from string to index. The added tokens encoder is cached for performance460 optimisation in `self._added_tokens_encoder` for the slow tokenizers.461 """462 return {k.content: v for v, k in sorted(self._added_tokens_decoder.items(), key=lambda item: item[0])}463 464 @property465 def added_tokens_decoder(self) -> dict[int, AddedToken]:466 """467 Returns the added tokens in the vocabulary as a dictionary of index to AddedToken.468 469 Returns:470 `dict[str, int]`: The added tokens.471 """472 return dict(sorted(self._added_tokens_decoder.items(), key=lambda item: item[0]))473 474 @added_tokens_decoder.setter475 def added_tokens_decoder(self, value: dict[int, Union[AddedToken, str]]) -> dict[int, AddedToken]:476 # Always raise an error if string because users should define the behavior477 for index, token in value.items():478 if not isinstance(token, (str, AddedToken)) or not isinstance(index, int):479 raise TypeError(480 f"The provided `added_tokens_decoder` has an element of type {index.__class__, token.__class__}, should be a dict of {int, Union[AddedToken, str]}"481 )482 483 self._added_tokens_decoder[index] = AddedToken(token) if isinstance(token, str) else token484 self._added_tokens_encoder[str(token)] = index485 self._update_total_vocab_size()486 487 def get_added_vocab(self) -> dict[str, int]:488 """489 Returns the added tokens in the vocabulary as a dictionary of token to index. Results might be different from490 the fast call because for now we always add the tokens even if they are already in the vocabulary. This is491 something we should change.492 493 Returns:494 `dict[str, int]`: The added tokens.495 """496 return self._added_tokens_encoder497 498 def __len__(self):499 """500 Size of the full vocabulary with the added tokens.501 """502 return self.total_vocab_size503 504 def _update_total_vocab_size(self):505 """506 Update the size of the full vocabulary with the added tokens. Counts the `keys` and not the `values` because507 otherwise if there is a hole in the vocab, we will add tokenizers at a wrong index. This operation is slow and508 is only updated when adding tokens.509 """510 self.total_vocab_size = len(self.get_vocab())511 512 def _add_tokens(self, new_tokens: Union[list[str], list[AddedToken]], special_tokens: bool = False) -> int:513 """514 Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to515 it with indices starting from length of the current vocabulary. Special tokens are sometimes already in the516 vocab which is why they have to be handled specifically.517 518 Args:519 new_tokens (`list[str]`or `list[tokenizers.AddedToken]`):520 Token(s) to add in vocabulary. A token is counted as added if it's not already in the vocabulary521 (tested by checking if the tokenizer assign the index of the `unk_token` to them). If a token is part522 of the vocabulary then we simply mark this token as an `AddedToken` which allows to control the523 stripping and normalization of this token. This is NOT possible in `tokenizers`.524 special_tokens (`bool`, *optional*, defaults to `False`):525 Whether or not the tokens should be added as special tokens.526 527 Returns:528 `int`: The number of tokens actually added to the vocabulary.529 530 Examples:531 532 ```python533 # Let's see how to increase the vocabulary of Bert model and tokenizer534 tokenizer = BertTokenizer.from_pretrained("google-bert/bert-base-uncased")535 model = BertModel.from_pretrained("google-bert/bert-base-uncased")536 537 num_added_toks = tokenizer.add_tokens(["new_tok1", "my_new-tok2"])538 print("We have added", num_added_toks, "tokens")539 # Note: resize_token_embeddings expects to receive the full size of the new vocabulary, i.e. the length of the tokenizer.540 model.resize_token_embeddings(len(tokenizer))541 ```"""542 added_tokens = 0543 if new_tokens is None:544 return added_tokens545 # TODO this is fairly slow to improve!546 current_vocab = self.get_vocab().copy()547 new_idx = len(current_vocab) # only call this once, len gives the last index + 1548 for token in new_tokens:549 if not isinstance(token, (str, AddedToken)):550 raise TypeError(f"Token {token} is not a string but a {type(token)}.")551 if str(token) == "":552 continue553 if isinstance(token, str):554 if token in self._added_tokens_encoder:555 continue556 else:557 # very important for fast and slow equivalence!558 is_special = token in self.all_special_tokens or special_tokens559 token = AddedToken(560 token, rstrip=False, lstrip=False, normalized=not is_special, special=is_special561 )562 elif special_tokens:563 # doing token.special=True changes the normalization! will fix in rust564 # this is important and the only reason why the AddedTokens in each class are normalized by default565 token.__setstate__({"special": True, "normalized": token.normalized})566 if token in self._added_tokens_decoder:567 continue568 if not token.special and token.normalized and getattr(self, "do_lower_case", False):569 # Normalize if requested570 token.content = token.content.lower()571 if token.content not in current_vocab:572 token_index = new_idx + added_tokens573 current_vocab[token.content] = token_index574 added_tokens += 1575 else:576 token_index = current_vocab[token.content]577 578 if token.special and str(token) not in self.all_special_tokens:579 self._special_tokens_map["additional_special_tokens"].append(token)580 # the setter automatically updates the reverse map581 self._added_tokens_decoder[token_index] = token582 self._added_tokens_encoder[token.content] = token_index583 if self.verbose:584 logger.info(f"Adding {token} to the vocabulary")585 586 self._update_trie()587 self._update_total_vocab_size()588 return added_tokens589 590 def _update_trie(self, unique_no_split_tokens: Optional[list[str]] = None):591 for token in self._added_tokens_decoder.values():592 if token.content not in self.tokens_trie._tokens:593 self.tokens_trie.add(token.content)594 for token in unique_no_split_tokens or []:595 if token not in self.tokens_trie._tokens:596 self.tokens_trie.add(token)597 598 def num_special_tokens_to_add(self, pair: bool = False) -> int:599 """600 Returns the number of added tokens when encoding a sequence with special tokens.601 602 <Tip>603 604 This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put605 this inside your training loop.606 607 </Tip>608 609 Args:610 pair (`bool`, *optional*, defaults to `False`):611 Whether the number of added tokens should be computed in the case of a sequence pair or a single612 sequence.613 614 Returns:615 `int`: Number of special tokens added to sequences.616 """617 token_ids_0 = []618 token_ids_1 = []619 return len(self.build_inputs_with_special_tokens(token_ids_0, token_ids_1 if pair else None))620 621 def tokenize(self, text: TextInput, **kwargs) -> list[str]:622 """623 Converts a string into a sequence of tokens, using the tokenizer.624 625 Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies626 (BPE/SentencePieces/WordPieces). Takes care of added tokens.627 628 Args:629 text (`str`):630 The sequence to be encoded.631 **kwargs (additional keyword arguments):632 Passed along to the model-specific `prepare_for_tokenization` preprocessing method.633 634 Returns:635 `list[str]`: The list of tokens.636 """637 split_special_tokens = kwargs.pop("split_special_tokens", self.split_special_tokens)638 639 text, kwargs = self.prepare_for_tokenization(text, **kwargs)640 641 if kwargs:642 logger.warning(f"Keyword arguments {kwargs} not recognized.")643 644 if hasattr(self, "do_lower_case") and self.do_lower_case:645 # convert non-special tokens to lowercase. Might be super slow as well?646 escaped_special_toks = [re.escape(s_tok) for s_tok in (self.all_special_tokens)]647 escaped_special_toks += [648 re.escape(s_tok.content)649 for s_tok in (self._added_tokens_decoder.values())650 if not s_tok.special and s_tok.normalized651 ]652 pattern = r"(" + r"|".join(escaped_special_toks) + r")|" + r"(.+?)"653 text = re.sub(pattern, lambda m: m.groups()[0] or m.groups()[1].lower(), text)654 655 if split_special_tokens:656 no_split_token = []657 tokens = [text]658 else:659 no_split_token = self._added_tokens_encoder.keys() # don't split on any of the added tokens660 # "This is something<special_token_1> else"661 tokens = self.tokens_trie.split(text)662 663 # ["This is something", "<special_token_1>", " else"]664 for i, token in enumerate(tokens):665 if token in no_split_token:666 tok_extended = self._added_tokens_decoder.get(self._added_tokens_encoder[token], None)667 left = tokens[i - 1] if i > 0 else None668 right = tokens[i + 1] if i < len(tokens) - 1 else None669 if isinstance(tok_extended, AddedToken):670 if tok_extended.rstrip and right:671 # A bit counter-intuitive but we strip the left of the string672 # since tok_extended.rstrip means the special token is eating all white spaces on its right673 tokens[i + 1] = right.lstrip()674 # Strip white spaces on the left675 if tok_extended.lstrip and left:676 tokens[i - 1] = left.rstrip() # Opposite here677 if tok_extended.single_word and left and left[-1] != " ":678 tokens[i - 1] += token679 tokens[i] = ""680 elif tok_extended.single_word and right and right[0] != " ":681 tokens[i + 1] = token + tokens[i + 1]682 tokens[i] = ""683 else:684 raise ValueError(685 f"{tok_extended} cannot be tokenized because it was not properly added"686 f" to the tokenizer. This means that it is not an `AddedToken` but a {type(tok_extended)}"687 )688 # ["This is something", "<special_token_1>", "else"]689 tokenized_text = []690 for token in tokens:691 # Need to skip eventual empty (fully stripped) tokens692 if not token:693 continue694 if token in no_split_token:695 tokenized_text.append(token)696 else:697 tokenized_text.extend(self._tokenize(token))698 # ["This", " is", " something", "<special_token_1>", "else"]699 return tokenized_text700 701 def _tokenize(self, text, **kwargs):702 """703 Converts a string into a sequence of tokens (string), using the tokenizer. Split in words for word-based704 vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).705 706 Do NOT take care of added tokens.707 """708 raise NotImplementedError709 710 def convert_tokens_to_ids(self, tokens: Union[str, list[str]]) -> Union[int, list[int]]:711 """712 Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the713 vocabulary.714 715 Args:716 tokens (`str` or `list[str]`): One or several token(s) to convert to token id(s).717 718 Returns:719 `int` or `list[int]`: The token id or list of token ids.720 """721 if tokens is None:722 return None723 724 if isinstance(tokens, str):725 return self._convert_token_to_id_with_added_voc(tokens)726 727 ids = []728 for token in tokens:729 ids.append(self._convert_token_to_id_with_added_voc(token))730 return ids731 732 def _convert_token_to_id_with_added_voc(self, token):733 if token is None:734 return None735 736 if token in self._added_tokens_encoder:737 return self._added_tokens_encoder[token]738 return self._convert_token_to_id(token)739 740 def _convert_token_to_id(self, token):741 raise NotImplementedError742 743 def _encode_plus(744 self,745 text: Union[TextInput, PreTokenizedInput, EncodedInput],746 text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None,747 add_special_tokens: bool = True,748 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,749 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,750 max_length: Optional[int] = None,751 stride: int = 0,752 is_split_into_words: bool = False,753 pad_to_multiple_of: Optional[int] = None,754 padding_side: Optional[str] = None,755 return_tensors: Optional[Union[str, TensorType]] = None,756 return_token_type_ids: Optional[bool] = None,757 return_attention_mask: Optional[bool] = None,758 return_overflowing_tokens: bool = False,759 return_special_tokens_mask: bool = False,760 return_offsets_mapping: bool = False,761 return_length: bool = False,762 verbose: bool = True,763 **kwargs,764 ) -> BatchEncoding:765 def get_input_ids(text):766 if isinstance(text, str):767 tokens = self.tokenize(text, **kwargs)768 return self.convert_tokens_to_ids(tokens)769 elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], str):770 if is_split_into_words:771 tokens = list(772 itertools.chain(*(self.tokenize(t, is_split_into_words=True, **kwargs) for t in text))773 )774 return self.convert_tokens_to_ids(tokens)775 else:776 return self.convert_tokens_to_ids(text)777 elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):778 return text779 else:780 if is_split_into_words:781 raise ValueError(782 f"Input {text} is not valid. Should be a string or a list/tuple of strings when"783 " `is_split_into_words=True`."784 )785 else:786 raise ValueError(787 f"Input {text} is not valid. Should be a string, a list/tuple of strings or a list/tuple of"788 " integers."789 )790 791 if return_offsets_mapping:792 raise NotImplementedError(793 "return_offset_mapping is not available when using Python tokenizers. "794 "To use this feature, change your tokenizer to one deriving from "795 "transformers.PreTrainedTokenizerFast. "796 "More information on available tokenizers at "797 "https://github.com/huggingface/transformers/pull/2674"798 )799 800 first_ids = get_input_ids(text)801 second_ids = get_input_ids(text_pair) if text_pair is not None else None802 803 return self.prepare_for_model(804 first_ids,805 pair_ids=second_ids,806 add_special_tokens=add_special_tokens,807 padding=padding_strategy.value,808 truncation=truncation_strategy.value,809 max_length=max_length,810 stride=stride,811 pad_to_multiple_of=pad_to_multiple_of,812 padding_side=padding_side,813 return_tensors=return_tensors,814 prepend_batch_axis=True,815 return_attention_mask=return_attention_mask,816 return_token_type_ids=return_token_type_ids,817 return_overflowing_tokens=return_overflowing_tokens,818 return_special_tokens_mask=return_special_tokens_mask,819 return_length=return_length,820 verbose=verbose,821 )822 823 def _batch_encode_plus(824 self,825 batch_text_or_text_pairs: Union[826 list[TextInput],827 list[TextInputPair],828 list[PreTokenizedInput],829 list[PreTokenizedInputPair],830 list[EncodedInput],831 list[EncodedInputPair],832 ],833 add_special_tokens: bool = True,834 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,835 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,836 max_length: Optional[int] = None,837 stride: int = 0,838 is_split_into_words: bool = False,839 pad_to_multiple_of: Optional[int] = None,840 padding_side: Optional[str] = None,841 return_tensors: Optional[Union[str, TensorType]] = None,842 return_token_type_ids: Optional[bool] = None,843 return_attention_mask: Optional[bool] = None,844 return_overflowing_tokens: bool = False,845 return_special_tokens_mask: bool = False,846 return_offsets_mapping: bool = False,847 return_length: bool = False,848 verbose: bool = True,849 split_special_tokens: bool = False,850 **kwargs,851 ) -> BatchEncoding:852 def get_input_ids(text):853 if isinstance(text, str):854 tokens = self.tokenize(text, **kwargs)855 return self.convert_tokens_to_ids(tokens)856 elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], str):857 if is_split_into_words:858 tokens = list(859 itertools.chain(*(self.tokenize(t, is_split_into_words=True, **kwargs) for t in text))860 )861 return self.convert_tokens_to_ids(tokens)862 else:863 return self.convert_tokens_to_ids(text)864 elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):865 return text866 else:867 raise ValueError(868 "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."869 )870 871 if return_offsets_mapping:872 raise NotImplementedError(873 "return_offset_mapping is not available when using Python tokenizers. "874 "To use this feature, change your tokenizer to one deriving from "875 "transformers.PreTrainedTokenizerFast."876 )877 878 input_ids = []879 for ids_or_pair_ids in batch_text_or_text_pairs:880 if (881 not isinstance(ids_or_pair_ids, (list, tuple))882 or is_split_into_words883 and not isinstance(ids_or_pair_ids[0], (list, tuple))884 ):885 ids, pair_ids = ids_or_pair_ids, None886 else:887 ids, pair_ids = ids_or_pair_ids888 889 first_ids = get_input_ids(ids)890 second_ids = get_input_ids(pair_ids) if pair_ids is not None else None891 input_ids.append((first_ids, second_ids))892 893 batch_outputs = self._batch_prepare_for_model(894 input_ids,895 add_special_tokens=add_special_tokens,896 padding_strategy=padding_strategy,897 truncation_strategy=truncation_strategy,898 max_length=max_length,899 stride=stride,900 pad_to_multiple_of=pad_to_multiple_of,901 padding_side=padding_side,902 return_attention_mask=return_attention_mask,903 return_token_type_ids=return_token_type_ids,904 return_overflowing_tokens=return_overflowing_tokens,905 return_special_tokens_mask=return_special_tokens_mask,906 return_length=return_length,907 return_tensors=return_tensors,908 verbose=verbose,909 split_special_tokens=split_special_tokens,910 )911 912 return BatchEncoding(batch_outputs)913 914 @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)915 def _batch_prepare_for_model(916 self,917 batch_ids_pairs: list[Union[PreTokenizedInputPair, tuple[list[int], None]]],918 add_special_tokens: bool = True,919 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,920 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,921 max_length: Optional[int] = None,922 stride: int = 0,923 pad_to_multiple_of: Optional[int] = None,924 padding_side: Optional[str] = None,925 return_tensors: Optional[str] = None,926 return_token_type_ids: Optional[bool] = None,927 return_attention_mask: Optional[bool] = None,928 return_overflowing_tokens: bool = False,929 return_special_tokens_mask: bool = False,930 return_length: bool = False,931 verbose: bool = True,932 split_special_tokens: bool = False,933 ) -> BatchEncoding:934 """935 Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It936 adds special tokens, truncates sequences if overflowing while taking into account the special tokens and937 manages a moving window (with user defined stride) for overflowing tokens938 939 Args:940 batch_ids_pairs: list of tokenized input ids or input ids pairs941 """942 943 batch_outputs = {}944 for first_ids, second_ids in batch_ids_pairs:945 outputs = self.prepare_for_model(946 first_ids,947 second_ids,948 add_special_tokens=add_special_tokens,949 padding=PaddingStrategy.DO_NOT_PAD.value, # we pad in batch afterward950 truncation=truncation_strategy.value,951 max_length=max_length,952 stride=stride,953 pad_to_multiple_of=None, # we pad in batch afterward954 padding_side=None, # we pad in batch afterward955 return_attention_mask=False, # we pad in batch afterward956 return_token_type_ids=return_token_type_ids,957 return_overflowing_tokens=return_overflowing_tokens,958 return_special_tokens_mask=return_special_tokens_mask,959 return_length=return_length,960 return_tensors=None, # We convert the whole batch to tensors at the end961 prepend_batch_axis=False,962 verbose=verbose,963 split_special_tokens=split_special_tokens,964 )965 966 for key, value in outputs.items():967 if key not in batch_outputs:968 batch_outputs[key] = []969 batch_outputs[key].append(value)970 971 batch_outputs = self.pad(972 batch_outputs,973 padding=padding_strategy.value,974 max_length=max_length,975 pad_to_multiple_of=pad_to_multiple_of,976 padding_side=padding_side,977 return_attention_mask=return_attention_mask,978 )979 980 batch_outputs = BatchEncoding(batch_outputs, tensor_type=return_tensors)981 982 return batch_outputs983 984 def prepare_for_tokenization(985 self, text: str, is_split_into_words: bool = False, **kwargs986 ) -> tuple[str, dict[str, Any]]:987 """988 Performs any necessary transformations before tokenization.989 990 This method should pop the arguments from kwargs and return the remaining `kwargs` as well. We test the991 `kwargs` at the end of the encoding process to be sure all the arguments have been used.992 993 Args:994 text (`str`):995 The text to prepare.996 is_split_into_words (`bool`, *optional*, defaults to `False`):997 Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the998 tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace)999 which it will tokenize. This is useful for NER or token classification.1000 kwargs (`dict[str, Any]`, *optional*):1001 Keyword arguments to use for the tokenization.1002 1003 Returns:1004 `tuple[str, dict[str, Any]]`: The prepared text and the unused kwargs.1005 """1006 return (text, kwargs)1007 1008 def get_special_tokens_mask(1009 self, token_ids_0: list, token_ids_1: Optional[list] = None, already_has_special_tokens: bool = False1010 ) -> list[int]:1011 """1012 Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding1013 special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.1014 1015 Args:1016 token_ids_0 (`list[int]`):1017 List of ids of the first sequence.1018 token_ids_1 (`list[int]`, *optional*):1019 List of ids of the second sequence.1020 already_has_special_tokens (`bool`, *optional*, defaults to `False`):1021 Whether or not the token list is already formatted with special tokens for the model.1022 1023 Returns:1024 A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.1025 """1026 if already_has_special_tokens:1027 if token_ids_1 is not None:1028 raise ValueError(1029 "You should not supply a second sequence if the provided sequence of "1030 "ids is already formatted with special tokens for the model."1031 )1032 1033 return super().get_special_tokens_mask(1034 token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True1035 )1036 return [0] * ((len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0))1037 1038 @overload1039 def convert_ids_to_tokens(self, ids: int, skip_special_tokens: bool = False) -> str: ...1040 1041 @overload1042 def convert_ids_to_tokens(self, ids: list[int], skip_special_tokens: bool = False) -> list[str]: ...1043 1044 def convert_ids_to_tokens(1045 self, ids: Union[int, list[int]], skip_special_tokens: bool = False1046 ) -> Union[str, list[str]]:1047 """1048 Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and1049 added tokens.1050 1051 Args:1052 ids (`int` or `list[int]`):1053 The token id (or token ids) to convert to tokens.1054 skip_special_tokens (`bool`, *optional*, defaults to `False`):1055 Whether or not to remove special tokens in the decoding.1056 1057 Returns:1058 `str` or `list[str]`: The decoded token(s).1059 """1060 if isinstance(ids, int):1061 if ids in self._added_tokens_decoder:1062 return self._added_tokens_decoder[ids].content1063 else:1064 return self._convert_id_to_token(ids)1065 tokens = []1066 for index in ids:1067 index = int(index)1068 if skip_special_tokens and index in self.all_special_ids:1069 continue1070 if index in self._added_tokens_decoder:1071 tokens.append(self._added_tokens_decoder[index].content)1072 else:1073 tokens.append(self._convert_id_to_token(index))1074 return tokens1075 1076 def _convert_id_to_token(self, index: int) -> str:1077 raise NotImplementedError1078 1079 def convert_tokens_to_string(self, tokens: list[str]) -> str:1080 return " ".join(tokens)1081 1082 def _decode(1083 self,1084 token_ids: Union[int, list[int]],1085 skip_special_tokens: bool = False,1086 clean_up_tokenization_spaces: Optional[bool] = None,1087 spaces_between_special_tokens: bool = True,1088 **kwargs,1089 ) -> str:1090 self._decode_use_source_tokenizer = kwargs.pop("use_source_tokenizer", False)1091 1092 filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)1093 # If given is a single id, prevents splitting the string in upcoming loop1094 if isinstance(filtered_tokens, str):1095 filtered_tokens = [filtered_tokens]1096 1097 legacy_added_tokens = set(self._added_tokens_encoder.keys()) - set(self.all_special_tokens) | {1098 token for token in self.additional_special_tokens if self.convert_tokens_to_ids(token) >= self.vocab_size1099 }1100 # To avoid mixing byte-level and unicode for byte-level BPT1101 # we need to build string separately for added tokens and byte-level tokens1102 # cf. https://github.com/huggingface/transformers/issues/11331103 sub_texts = []1104 current_sub_text = []1105 # TODO @ArthurZ in version 5, special tokens should be handled in convert_tokens_to_string, while _convert_tokens_to_string1106 for token in filtered_tokens:1107 if skip_special_tokens and token in self.all_special_tokens:1108 continue1109 if token in legacy_added_tokens:1110 if current_sub_text:1111 string = self.convert_tokens_to_string(current_sub_text)1112 if len(string) > 0:1113 sub_texts.append(string)1114 current_sub_text = []1115 sub_texts.append(token)1116 else:1117 current_sub_text.append(token)1118 if current_sub_text:1119 sub_texts.append(self.convert_tokens_to_string(current_sub_text))1120 1121 if spaces_between_special_tokens:1122 text = " ".join(sub_texts)1123 else:1124 text = "".join(sub_texts)1125 1126 clean_up_tokenization_spaces = (1127 clean_up_tokenization_spaces1128 if clean_up_tokenization_spaces is not None1129 else self.clean_up_tokenization_spaces1130 )1131 if clean_up_tokenization_spaces:1132 clean_text = self.clean_up_tokenization(text)1133 return clean_text1134 else:1135 return text1136