DoruC/Grounded-Segment-Anything
0
1# coding=utf-82# Copyright 2020 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""16 Tokenization classes for python tokenizers. For fast tokenizers (provided by HuggingFace's tokenizers library) see17 tokenization_utils_fast.py18"""19import bisect20import itertools21import re22import unicodedata23from collections import OrderedDict24from typing import Any, Dict, List, Optional, Tuple, Union, overload25 26from .tokenization_utils_base import (27 ENCODE_KWARGS_DOCSTRING,28 ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING,29 INIT_TOKENIZER_DOCSTRING,30 AddedToken,31 BatchEncoding,32 EncodedInput,33 EncodedInputPair,34 PreTokenizedInput,35 PreTokenizedInputPair,36 PreTrainedTokenizerBase,37 TextInput,38 TextInputPair,39 TruncationStrategy,40)41from .utils import PaddingStrategy, TensorType, add_end_docstrings, logging42 43 44logger = logging.get_logger(__name__)45 46# Slow tokenizers are saved in a vocabulary plus three separated files47SPECIAL_TOKENS_MAP_FILE = "special_tokens_map.json"48ADDED_TOKENS_FILE = "added_tokens.json"49TOKENIZER_CONFIG_FILE = "tokenizer_config.json"50 51 52class Trie:53 """54 Trie in Python. Creates a Trie out of a list of words. The trie is used to split on `added_tokens` in one pass55 Loose reference https://en.wikipedia.org/wiki/Trie56 """57 58 def __init__(self):59 self.data = {}60 self._tokens = set()61 62 def add(self, word: str):63 """64 Passes over every char (utf-8 char) on word and recursively adds it to the internal `data` trie representation.65 The special key `""` is used to represent termination.66 67 This function is idempotent, adding twice the same word will leave the trie unchanged68 69 Example:70 71 ```python72 >>> trie = Trie()73 >>> trie.add("Hello 友達")74 >>> trie.data75 {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}76 77 >>> trie.add("Hello")78 >>> trie.data79 {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}80 ```81 """82 if not word:83 # Prevent empty string84 return85 86 self._tokens.add(word)87 ref = self.data88 for char in word:89 ref[char] = char in ref and ref[char] or {}90 ref = ref[char]91 ref[""] = 192 93 def split(self, text: str) -> List[str]:94 """95 Will look for the words added to the trie within `text`. Output is the original string splitted along the96 boundaries of the words found.97 98 This trie will match the longest possible word first !99 100 Example:101 102 ```python103 >>> trie = Trie()104 >>> trie.split("[CLS] This is a extra_id_100")105 ["[CLS] This is a extra_id_100"]106 107 >>> trie.add("[CLS]")108 >>> trie.add("extra_id_1")109 >>> trie.add("extra_id_100")110 >>> trie.split("[CLS] This is a extra_id_100")111 ["[CLS]", " This is a ", "extra_id_100"]112 ```113 """114 # indexes are counted left of the chars index.115 # "hello", index 0, is left of h, index 1 is between h and e.116 # index 5 is right of the "o".117 118 # States are going to capture every possible start (indexes as above)119 # as keys, and have as values, a pointer to the position in the trie120 # where we're at. This is a partial match for now.121 # This enables to keep track of multiple matches while we're iterating122 # the string123 # If the trie contains, "blowing", and "lower" and we encounter the124 # string "blower", we need to split into ["b", "lower"].125 # This is where we need to keep track of multiple possible starts.126 states = OrderedDict()127 128 # This will contain every indices where we need129 # to cut.130 # We force to cut at offset 0 and len(text) (added later)131 offsets = [0]132 133 # This is used by the lookahead which needs to skip over134 # some text where the full match exceeded the place in the initial135 # for loop136 skip = 0137 # Main loop, Giving this algorithm O(n) complexity138 for current, current_char in enumerate(text):139 if skip and current < skip:140 # Prevents the lookahead for matching twice141 # like extra_id_100 and id_100142 continue143 144 # This will track every state145 # that stop matching, we need to stop tracking them.146 # If we look at "lowball", we're going to match "l" (add it to states), "o", "w", then147 # fail on "b", we need to remove 0 from the valid states.148 to_remove = set()149 # Whenever we found a match, we need to drop everything150 # this is a greedy algorithm, it will match on the first found token151 reset = False152 153 # In this case, we already have partial matches (But unfinished)154 for start, trie_pointer in states.items():155 if "" in trie_pointer:156 # This is a final match, we need to reset and157 # store the results in `offsets`.158 159 # Lookahead to match longest first160 # Important in case of extra_id_1 vs extra_id_100161 # Here we are also actively looking for other earlier partial162 # matches163 # "[CLS]", "L", we need to match CLS even if L is special164 for lookstart, looktrie_pointer in states.items():165 if lookstart > start:166 # This partial match is later, we can stop looking167 break168 elif lookstart < start:169 # This partial match is earlier, the trie pointer170 # was already updated, so index is + 1171 lookahead_index = current + 1172 end = current + 1173 else:174 # Here lookstart == start and175 # looktrie_pointer == trie_pointer176 # It wasn't updated yet so indices are current ones177 lookahead_index = current178 end = current179 next_char = text[lookahead_index] if lookahead_index < len(text) else None180 if "" in looktrie_pointer:181 start = lookstart182 end = lookahead_index183 skip = lookahead_index184 185 while next_char in looktrie_pointer:186 looktrie_pointer = looktrie_pointer[next_char]187 lookahead_index += 1188 if "" in looktrie_pointer:189 start = lookstart190 end = lookahead_index191 skip = lookahead_index192 193 if lookahead_index == len(text):194 # End of string195 break196 next_char = text[lookahead_index]197 # End lookahead198 199 # Storing and resetting200 offsets.append(start)201 offsets.append(end)202 reset = True203 break204 elif current_char in trie_pointer:205 # The current character being looked at has a match within the trie206 # update the pointer (it will be stored back into states later).207 trie_pointer = trie_pointer[current_char]208 209 # Storing back the new pointer into the states.210 # Partial matches got longer by one.211 states[start] = trie_pointer212 else:213 # The new character has not match in the trie, we need214 # to stop keeping track of this partial match.215 # We can't do it directly within the loop because of how216 # python iteration works217 to_remove.add(start)218 219 # Either clearing the full start (we found a real match)220 # Or clearing only the partial matches that didn't work.221 if reset:222 states = {}223 else:224 for start in to_remove:225 del states[start]226 227 # If this character is a starting character within the trie228 # start keeping track of this partial match.229 if current >= skip and current_char in self.data:230 states[current] = self.data[current_char]231 232 # We have a cut at the end with states.233 for start, trie_pointer in states.items():234 if "" in trie_pointer:235 # This is a final match, we need to reset and236 # store the results in `offsets`.237 end = len(text)238 offsets.append(start)239 offsets.append(end)240 # Longest cut is always the one with lower start so the first241 # item so we need to break.242 break243 244 return self.cut_text(text, offsets)245 246 def cut_text(self, text, offsets):247 # We have all the offsets now, we just need to do the actual splitting.248 # We need to eventually add the first part of the string and the eventual249 # last part.250 offsets.append(len(text))251 tokens = []252 start = 0253 for end in offsets:254 if start > end:255 logger.error(256 "There was a bug in Trie algorithm in tokenization. Attempting to recover. Please report it"257 " anyway."258 )259 continue260 elif start == end:261 # This might happen if there's a match at index 0262 # we're also preventing zero-width cuts in case of two263 # consecutive matches264 continue265 tokens.append(text[start:end])266 start = end267 268 return tokens269 270 271def _is_whitespace(char):272 """Checks whether `char` is a whitespace character."""273 # \t, \n, and \r are technically control characters but we treat them274 # as whitespace since they are generally considered as such.275 if char == " " or char == "\t" or char == "\n" or char == "\r":276 return True277 cat = unicodedata.category(char)278 if cat == "Zs":279 return True280 return False281 282 283def _is_control(char):284 """Checks whether `char` is a control character."""285 # These are technically control characters but we count them as whitespace286 # characters.287 if char == "\t" or char == "\n" or char == "\r":288 return False289 cat = unicodedata.category(char)290 if cat.startswith("C"):291 return True292 return False293 294 295def _is_punctuation(char):296 """Checks whether `char` is a punctuation character."""297 cp = ord(char)298 # We treat all non-letter/number ASCII as punctuation.299 # Characters such as "^", "$", and "`" are not in the Unicode300 # Punctuation class but we treat them as punctuation anyways, for301 # consistency.302 if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):303 return True304 cat = unicodedata.category(char)305 if cat.startswith("P"):306 return True307 return False308 309 310def _is_end_of_word(text):311 """Checks whether the last character in text is one of a punctuation, control or whitespace character."""312 last_char = text[-1]313 return bool(_is_control(last_char) | _is_punctuation(last_char) | _is_whitespace(last_char))314 315 316def _is_start_of_word(text):317 """Checks whether the first character in text is one of a punctuation, control or whitespace character."""318 first_char = text[0]319 return bool(_is_control(first_char) | _is_punctuation(first_char) | _is_whitespace(first_char))320 321 322def _insert_one_token_to_ordered_list(token_list: List[str], new_token: str):323 """324 Inserts one token to an ordered list if it does not already exist. Note: token_list must be sorted.325 """326 insertion_idx = bisect.bisect_left(token_list, new_token)327 # Checks if new_token is already in the ordered token_list328 if insertion_idx < len(token_list) and token_list[insertion_idx] == new_token:329 # new_token is in token_list, don't add330 return331 else:332 token_list.insert(insertion_idx, new_token)333 334 335@add_end_docstrings(INIT_TOKENIZER_DOCSTRING)336class PreTrainedTokenizer(PreTrainedTokenizerBase):337 """338 Base class for all slow tokenizers.339 340 Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`].341 342 Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading343 pretrained tokenizers as well as adding tokens to the vocabulary.344 345 This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the346 specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).347 """348 349 def __init__(self, **kwargs):350 # 1. Init the parent class351 super().__init__(**kwargs)352 self.tokens_trie = Trie()353 354 # 2. init `_added_tokens_decoder` if child class did not355 if not hasattr(self, "_added_tokens_decoder"):356 self._added_tokens_decoder: Dict[int, AddedToken] = {}357 # 3. if a `added_tokens_decoder` is passed, we are loading from a saved tokenizer, we overwrite358 if "added_tokens_decoder" in kwargs:359 # overwriting the class's added_tokens_decoder. This is the source of truth!360 self._added_tokens_decoder.update(kwargs.get("added_tokens_decoder"))361 362 self._added_tokens_encoder: Dict[str, int] = {k.content: v for v, k in self._added_tokens_decoder.items()}363 364 # 4. If some of the special tokens are not part of the vocab, we add them, at the end.365 # the order of addition is the same as self.SPECIAL_TOKENS_ATTRIBUTES following `tokenizers`366 self._add_tokens(self.all_special_tokens_extended, special_tokens=True)367 368 self._decode_use_source_tokenizer = False369 370 @property371 def is_fast(self) -> bool:372 return False373 374 @property375 def vocab_size(self) -> int:376 """377 `int`: Size of the base vocabulary (without the added tokens).378 """379 raise NotImplementedError380 381 @property382 def added_tokens_encoder(self) -> Dict[str, int]:383 """384 Returns the sorted mapping from string to index. The added tokens encoder is cached for performance385 optimisation in `self._added_tokens_encoder` for the slow tokenizers.386 """387 return {k.content: v for v, k in sorted(self._added_tokens_decoder.items(), key=lambda item: item[0])}388 389 @property390 def added_tokens_decoder(self) -> Dict[int, AddedToken]:391 """392 Returns the added tokens in the vocabulary as a dictionary of index to AddedToken.393 394 Returns:395 `Dict[str, int]`: The added tokens.396 """397 return dict(sorted(self._added_tokens_decoder.items(), key=lambda item: item[0]))398 399 @added_tokens_decoder.setter400 def added_tokens_decoder(self, value: Dict[int, Union[AddedToken, str]]) -> Dict[int, AddedToken]:401 # Always raise an error if string because users should define the behavior402 for index, token in value.items():403 if not isinstance(token, (str, AddedToken)) or not isinstance(index, int):404 raise ValueError(405 f"The provided `added_tokens_decoder` has an element of type {index.__class__, token.__class__}, should be a dict of {int, Union[AddedToken, str]}"406 )407 408 self._added_tokens_decoder[index] = AddedToken(token) if isinstance(token, str) else token409 self._added_tokens_encoder[str(token)] = index410 411 def get_added_vocab(self) -> Dict[str, int]:412 """413 Returns the added tokens in the vocabulary as a dictionary of token to index. Results might be different from414 the fast call because for now we always add the tokens even if they are already in the vocabulary. This is415 something we should change.416 417 Returns:418 `Dict[str, int]`: The added tokens.419 """420 return self._added_tokens_encoder421 422 def __len__(self):423 """424 Size of the full vocabulary with the added tokens. Counts the `keys` and not the `values` because otherwise if425 there is a hole in the vocab, we will add tokenizers at a wrong index.426 """427 return len(set(self.get_vocab().keys()))428 429 def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:430 """431 Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to432 it with indices starting from length of the current vocabulary. Special tokens are sometimes already in the433 vocab which is why they have to be handled specifically.434 435 Args:436 new_tokens (`List[str]`or `List[tokenizers.AddedToken]`):437 Token(s) to add in vocabulary. A token is counted as added if it's not already in the vocabulary438 (tested by checking if the tokenizer assign the index of the `unk_token` to them). If a token is part439 of the vocabulary then we simply mark this token as an `AddedToken` which allows to control the440 stripping and normalization of this token. This is NOT possible in `tokenizers`.441 special_tokens (`bool`, *optional*, defaults to `False`):442 Whether or not the tokens should be added as special tokens.443 444 Returns:445 `int`: The number of tokens actually added to the vocabulary.446 447 Examples:448 449 ```python450 # Let's see how to increase the vocabulary of Bert model and tokenizer451 tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")452 model = BertModel.from_pretrained("bert-base-uncased")453 454 num_added_toks = tokenizer.add_tokens(["new_tok1", "my_new-tok2"])455 print("We have added", num_added_toks, "tokens")456 # Note: resize_token_embeddings expects to receive the full size of the new vocabulary, i.e. the length of the tokenizer.457 model.resize_token_embeddings(len(tokenizer))458 ```"""459 added_tokens = 0460 if new_tokens is None:461 return added_tokens462 current_vocab = self.get_vocab().copy()463 new_idx = len(current_vocab) # only call this once, len gives the last index + 1464 for token in new_tokens:465 if not isinstance(token, (str, AddedToken)):466 raise TypeError(f"Token {token} is not a string but a {type(token)}.")467 if str(token) == "":468 continue469 if isinstance(token, str):470 # for legacy AddedTokens strip left and right by default471 # TODO this will be remove to have the same default behavior as rust472 token = AddedToken(token, normalized=not special_tokens, rstrip=True, lstrip=True)473 if special_tokens:474 token.special = True475 if token in self._added_tokens_decoder:476 continue477 if not token.special and token.normalized and hasattr(self, "do_lower_case") and self.do_lower_case:478 # Normalize if requested479 token.content = token.content.lower()480 if token.content not in current_vocab:481 token_index = new_idx + added_tokens482 current_vocab[token.content] = token_index483 added_tokens += 1484 else:485 token_index = current_vocab[token.content]486 487 if token.special and str(token) not in self.all_special_tokens:488 self._additional_special_tokens.append(token)489 # the setter automatically updates the reverse map490 self._added_tokens_decoder[token_index] = token491 self._added_tokens_encoder[token.content] = token_index492 if self.verbose:493 logger.info(f"Adding {token} to the vocabulary")494 495 self._update_trie()496 return added_tokens497 498 def _update_trie(self, unique_no_split_tokens: Optional[str] = []):499 for token in self._added_tokens_decoder.values():500 if token not in self.tokens_trie._tokens:501 self.tokens_trie.add(token.content)502 for token in unique_no_split_tokens:503 if token not in self.tokens_trie._tokens:504 self.tokens_trie.add(token)505 506 def num_special_tokens_to_add(self, pair: bool = False) -> int:507 """508 Returns the number of added tokens when encoding a sequence with special tokens.509 510 <Tip>511 512 This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put513 this inside your training loop.514 515 </Tip>516 517 Args:518 pair (`bool`, *optional*, defaults to `False`):519 Whether the number of added tokens should be computed in the case of a sequence pair or a single520 sequence.521 522 Returns:523 `int`: Number of special tokens added to sequences.524 """525 token_ids_0 = []526 token_ids_1 = []527 return len(self.build_inputs_with_special_tokens(token_ids_0, token_ids_1 if pair else None))528 529 def tokenize(self, text: TextInput, **kwargs) -> List[str]:530 """531 Converts a string in a sequence of tokens, using the tokenizer.532 533 Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies534 (BPE/SentencePieces/WordPieces). Takes care of added tokens.535 536 Args:537 text (`str`):538 The sequence to be encoded.539 **kwargs (additional keyword arguments):540 Passed along to the model-specific `prepare_for_tokenization` preprocessing method.541 542 Returns:543 `List[str]`: The list of tokens.544 """545 split_special_tokens = kwargs.pop("split_special_tokens", self.split_special_tokens)546 547 text, kwargs = self.prepare_for_tokenization(text, **kwargs)548 549 if kwargs:550 logger.warning(f"Keyword arguments {kwargs} not recognized.")551 552 if hasattr(self, "do_lower_case") and self.do_lower_case:553 # convert non-special tokens to lowercase554 escaped_special_toks = [re.escape(s_tok) for s_tok in (self.all_special_tokens)]555 escaped_special_toks += [556 re.escape(s_tok.content)557 for s_tok in (self._added_tokens_decoder.values())558 if not s_tok.special and s_tok.normalized559 ]560 pattern = r"(" + r"|".join(escaped_special_toks) + r")|" + r"(.+?)"561 text = re.sub(pattern, lambda m: m.groups()[0] or m.groups()[1].lower(), text)562 563 if split_special_tokens:564 no_split_token = []565 tokens = [text]566 else:567 no_split_token = set(self._added_tokens_encoder.keys()) # don't split on any of the added tokens568 # "This is something<special_token_1> else"569 tokens = self.tokens_trie.split(text)570 571 # ["This is something", "<special_token_1>", " else"]572 for i, token in enumerate(tokens):573 if token in no_split_token:574 tok_extended = self._added_tokens_decoder.get(self._added_tokens_encoder[token], None)575 left = tokens[i - 1] if i > 0 else None576 right = tokens[i + 1] if i < len(tokens) - 1 else None577 if isinstance(tok_extended, AddedToken):578 if tok_extended.rstrip and right:579 # A bit counter-intuitive but we strip the left of the string580 # since tok_extended.rstrip means the special token is eating all white spaces on its right581 tokens[i + 1] = right.lstrip()582 # Strip white spaces on the left583 if tok_extended.lstrip and left:584 tokens[i - 1] = left.rstrip() # Opposite here585 if tok_extended.single_word and left and left[-1] != " ":586 tokens[i - 1] += token587 tokens[i] = ""588 elif tok_extended.single_word and right and right[0] != " ":589 tokens[i + 1] = token + tokens[i + 1]590 tokens[i] = ""591 592 else:593 raise ValueError(594 f"{tok_extended} cannot be tokenized because it was not properly added"595 f" to the tokenizer. This means that it is not an `AddedToken` but a {type(tok_extended)}"596 )597 # ["This is something", "<special_token_1>", "else"]598 tokenized_text = []599 for token in tokens:600 # Need to skip eventual empty (fully stripped) tokens601 if not token:602 continue603 if token in no_split_token:604 tokenized_text.append(token)605 else:606 tokenized_text.extend(self._tokenize(token))607 # ["This", " is", " something", "<special_token_1>", "else"]608 return tokenized_text609 610 def _tokenize(self, text, **kwargs):611 """612 Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based613 vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).614 615 Do NOT take care of added tokens.616 """617 raise NotImplementedError618 619 def convert_tokens_to_ids(self, tokens: Union[str, List[str]]) -> Union[int, List[int]]:620 """621 Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the622 vocabulary.623 624 Args:625 tokens (`str` or `List[str]`): One or several token(s) to convert to token id(s).626 627 Returns:628 `int` or `List[int]`: The token id or list of token ids.629 """630 if tokens is None:631 return None632 633 if isinstance(tokens, str):634 return self._convert_token_to_id_with_added_voc(tokens)635 636 ids = []637 for token in tokens:638 ids.append(self._convert_token_to_id_with_added_voc(token))639 return ids640 641 def _convert_token_to_id_with_added_voc(self, token):642 if token is None:643 return None644 645 if token in self._added_tokens_encoder:646 return self._added_tokens_encoder[token]647 return self._convert_token_to_id(token)648 649 def _convert_token_to_id(self, token):650 raise NotImplementedError651 652 def _encode_plus(653 self,654 text: Union[TextInput, PreTokenizedInput, EncodedInput],655 text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None,656 add_special_tokens: bool = True,657 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,658 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,659 max_length: Optional[int] = None,660 stride: int = 0,661 is_split_into_words: bool = False,662 pad_to_multiple_of: Optional[int] = None,663 return_tensors: Optional[Union[str, TensorType]] = None,664 return_token_type_ids: Optional[bool] = None,665 return_attention_mask: Optional[bool] = None,666 return_overflowing_tokens: bool = False,667 return_special_tokens_mask: bool = False,668 return_offsets_mapping: bool = False,669 return_length: bool = False,670 verbose: bool = True,671 **kwargs,672 ) -> BatchEncoding:673 def get_input_ids(text):674 if isinstance(text, str):675 tokens = self.tokenize(text, **kwargs)676 return self.convert_tokens_to_ids(tokens)677 elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], str):678 if is_split_into_words:679 tokens = list(680 itertools.chain(*(self.tokenize(t, is_split_into_words=True, **kwargs) for t in text))681 )682 return self.convert_tokens_to_ids(tokens)683 else:684 return self.convert_tokens_to_ids(text)685 elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):686 return text687 else:688 if is_split_into_words:689 raise ValueError(690 f"Input {text} is not valid. Should be a string or a list/tuple of strings when"691 " `is_split_into_words=True`."692 )693 else:694 raise ValueError(695 f"Input {text} is not valid. Should be a string, a list/tuple of strings or a list/tuple of"696 " integers."697 )698 699 if return_offsets_mapping:700 raise NotImplementedError(701 "return_offset_mapping is not available when using Python tokenizers. "702 "To use this feature, change your tokenizer to one deriving from "703 "transformers.PreTrainedTokenizerFast. "704 "More information on available tokenizers at "705 "https://github.com/huggingface/transformers/pull/2674"706 )707 708 first_ids = get_input_ids(text)709 second_ids = get_input_ids(text_pair) if text_pair is not None else None710 711 return self.prepare_for_model(712 first_ids,713 pair_ids=second_ids,714 add_special_tokens=add_special_tokens,715 padding=padding_strategy.value,716 truncation=truncation_strategy.value,717 max_length=max_length,718 stride=stride,719 pad_to_multiple_of=pad_to_multiple_of,720 return_tensors=return_tensors,721 prepend_batch_axis=True,722 return_attention_mask=return_attention_mask,723 return_token_type_ids=return_token_type_ids,724 return_overflowing_tokens=return_overflowing_tokens,725 return_special_tokens_mask=return_special_tokens_mask,726 return_length=return_length,727 verbose=verbose,728 )729 730 def _batch_encode_plus(731 self,732 batch_text_or_text_pairs: Union[733 List[TextInput],734 List[TextInputPair],735 List[PreTokenizedInput],736 List[PreTokenizedInputPair],737 List[EncodedInput],738 List[EncodedInputPair],739 ],740 add_special_tokens: bool = True,741 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,742 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,743 max_length: Optional[int] = None,744 stride: int = 0,745 is_split_into_words: bool = False,746 pad_to_multiple_of: Optional[int] = None,747 return_tensors: Optional[Union[str, TensorType]] = None,748 return_token_type_ids: Optional[bool] = None,749 return_attention_mask: Optional[bool] = None,750 return_overflowing_tokens: bool = False,751 return_special_tokens_mask: bool = False,752 return_offsets_mapping: bool = False,753 return_length: bool = False,754 verbose: bool = True,755 **kwargs,756 ) -> BatchEncoding:757 def get_input_ids(text):758 if isinstance(text, str):759 tokens = self.tokenize(text, **kwargs)760 return self.convert_tokens_to_ids(tokens)761 elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], str):762 if is_split_into_words:763 tokens = list(764 itertools.chain(*(self.tokenize(t, is_split_into_words=True, **kwargs) for t in text))765 )766 return self.convert_tokens_to_ids(tokens)767 else:768 return self.convert_tokens_to_ids(text)769 elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):770 return text771 else:772 raise ValueError(773 "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."774 )775 776 if return_offsets_mapping:777 raise NotImplementedError(778 "return_offset_mapping is not available when using Python tokenizers. "779 "To use this feature, change your tokenizer to one deriving from "780 "transformers.PreTrainedTokenizerFast."781 )782 783 input_ids = []784 for ids_or_pair_ids in batch_text_or_text_pairs:785 if not isinstance(ids_or_pair_ids, (list, tuple)):786 ids, pair_ids = ids_or_pair_ids, None787 elif is_split_into_words and not isinstance(ids_or_pair_ids[0], (list, tuple)):788 ids, pair_ids = ids_or_pair_ids, None789 else:790 ids, pair_ids = ids_or_pair_ids791 792 first_ids = get_input_ids(ids)793 second_ids = get_input_ids(pair_ids) if pair_ids is not None else None794 input_ids.append((first_ids, second_ids))795 796 batch_outputs = self._batch_prepare_for_model(797 input_ids,798 add_special_tokens=add_special_tokens,799 padding_strategy=padding_strategy,800 truncation_strategy=truncation_strategy,801 max_length=max_length,802 stride=stride,803 pad_to_multiple_of=pad_to_multiple_of,804 return_attention_mask=return_attention_mask,805 return_token_type_ids=return_token_type_ids,806 return_overflowing_tokens=return_overflowing_tokens,807 return_special_tokens_mask=return_special_tokens_mask,808 return_length=return_length,809 return_tensors=return_tensors,810 verbose=verbose,811 )812 813 return BatchEncoding(batch_outputs)814 815 @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)816 def _batch_prepare_for_model(817 self,818 batch_ids_pairs: List[Union[PreTokenizedInputPair, Tuple[List[int], None]]],819 add_special_tokens: bool = True,820 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,821 truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,822 max_length: Optional[int] = None,823 stride: int = 0,824 pad_to_multiple_of: Optional[int] = None,825 return_tensors: Optional[str] = None,826 return_token_type_ids: Optional[bool] = None,827 return_attention_mask: Optional[bool] = None,828 return_overflowing_tokens: bool = False,829 return_special_tokens_mask: bool = False,830 return_length: bool = False,831 verbose: bool = True,832 ) -> BatchEncoding:833 """834 Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It835 adds special tokens, truncates sequences if overflowing while taking into account the special tokens and836 manages a moving window (with user defined stride) for overflowing tokens837 838 Args:839 batch_ids_pairs: list of tokenized input ids or input ids pairs840 """841 842 batch_outputs = {}843 for first_ids, second_ids in batch_ids_pairs:844 outputs = self.prepare_for_model(845 first_ids,846 second_ids,847 add_special_tokens=add_special_tokens,848 padding=PaddingStrategy.DO_NOT_PAD.value, # we pad in batch afterward849 truncation=truncation_strategy.value,850 max_length=max_length,851 stride=stride,852 pad_to_multiple_of=None, # we pad in batch afterward853 return_attention_mask=False, # we pad in batch afterward854 return_token_type_ids=return_token_type_ids,855 return_overflowing_tokens=return_overflowing_tokens,856 return_special_tokens_mask=return_special_tokens_mask,857 return_length=return_length,858 return_tensors=None, # We convert the whole batch to tensors at the end859 prepend_batch_axis=False,860 verbose=verbose,861 )862 863 for key, value in outputs.items():864 if key not in batch_outputs:865 batch_outputs[key] = []866 batch_outputs[key].append(value)867 868 batch_outputs = self.pad(869 batch_outputs,870 padding=padding_strategy.value,871 max_length=max_length,872 pad_to_multiple_of=pad_to_multiple_of,873 return_attention_mask=return_attention_mask,874 )875 876 batch_outputs = BatchEncoding(batch_outputs, tensor_type=return_tensors)877 878 return batch_outputs879 880 def prepare_for_tokenization(881 self, text: str, is_split_into_words: bool = False, **kwargs882 ) -> Tuple[str, Dict[str, Any]]:883 """884 Performs any necessary transformations before tokenization.885 886 This method should pop the arguments from kwargs and return the remaining `kwargs` as well. We test the887 `kwargs` at the end of the encoding process to be sure all the arguments have been used.888 889 Args:890 text (`str`):891 The text to prepare.892 is_split_into_words (`bool`, *optional*, defaults to `False`):893 Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the894 tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace)895 which it will tokenize. This is useful for NER or token classification.896 kwargs (`Dict[str, Any]`, *optional*):897 Keyword arguments to use for the tokenization.898 899 Returns:900 `Tuple[str, Dict[str, Any]]`: The prepared text and the unused kwargs.901 """902 return (text, kwargs)903 904 def get_special_tokens_mask(905 self, token_ids_0: List, token_ids_1: Optional[List] = None, already_has_special_tokens: bool = False906 ) -> List[int]:907 """908 Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding909 special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.910 911 Args:912 token_ids_0 (`List[int]`):913 List of ids of the first sequence.914 token_ids_1 (`List[int]`, *optional*):915 List of ids of the second sequence.916 already_has_special_tokens (`bool`, *optional*, defaults to `False`):917 Whether or not the token list is already formatted with special tokens for the model.918 919 Returns:920 A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.921 """922 if already_has_special_tokens:923 if token_ids_1 is not None:924 raise ValueError(925 "You should not supply a second sequence if the provided sequence of "926 "ids is already formatted with special tokens for the model."927 )928 929 return super().get_special_tokens_mask(930 token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True931 )932 return [0] * ((len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0))933 934 @overload935 def convert_ids_to_tokens(self, ids: int, skip_special_tokens: bool = False) -> str:936 ...937 938 @overload939 def convert_ids_to_tokens(self, ids: List[int], skip_special_tokens: bool = False) -> List[str]:940 ...941 942 def convert_ids_to_tokens(943 self, ids: Union[int, List[int]], skip_special_tokens: bool = False944 ) -> Union[str, List[str]]:945 """946 Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and947 added tokens.948 949 Args:950 ids (`int` or `List[int]`):951 The token id (or token ids) to convert to tokens.952 skip_special_tokens (`bool`, *optional*, defaults to `False`):953 Whether or not to remove special tokens in the decoding.954 955 Returns:956 `str` or `List[str]`: The decoded token(s).957 """958 if isinstance(ids, int):959 if ids in self._added_tokens_decoder:960 return self._added_tokens_decoder[ids].content961 else:962 return self._convert_id_to_token(ids)963 tokens = []964 for index in ids:965 index = int(index)966 if skip_special_tokens and index in self.all_special_ids:967 continue968 if index in self._added_tokens_decoder:969 tokens.append(self._added_tokens_decoder[index].content)970 else:971 tokens.append(self._convert_id_to_token(index))972 return tokens973 974 def _convert_id_to_token(self, index: int) -> str:975 raise NotImplementedError976 977 def convert_tokens_to_string(self, tokens: List[str]) -> str:978 return " ".join(tokens)979 980 def _decode(981 self,982 token_ids: List[int],983 skip_special_tokens: bool = False,984 clean_up_tokenization_spaces: bool = None,985 spaces_between_special_tokens: bool = True,986 **kwargs,987 ) -> str:988 self._decode_use_source_tokenizer = kwargs.pop("use_source_tokenizer", False)989 990 filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)991 legacy_added_tokens = set(self._added_tokens_encoder.keys()) - set(self.all_special_tokens) | {992 token for token in self.additional_special_tokens if self.convert_tokens_to_ids(token) >= self.vocab_size993 }994 # To avoid mixing byte-level and unicode for byte-level BPT995 # we need to build string separately for added tokens and byte-level tokens996 # cf. https://github.com/huggingface/transformers/issues/1133997 sub_texts = []998 current_sub_text = []999 # TODO @ArthurZ in version 5, special tokens should be handled in convert_tokens_to_string, while _convert_tokens_to_string1000 for token in filtered_tokens:1001 if skip_special_tokens and token in self.all_special_ids:1002 continue1003 if token in legacy_added_tokens:1004 if current_sub_text:1005 string = self.convert_tokens_to_string(current_sub_text)1006 if len(string) > 0:1007 sub_texts.append(string)1008 current_sub_text = []1009 sub_texts.append(token)1010 else:1011 current_sub_text.append(token)1012 if current_sub_text:1013 sub_texts.append(self.convert_tokens_to_string(current_sub_text))1014 1015 if spaces_between_special_tokens:1016 text = " ".join(sub_texts)1017 else:1018 text = "".join(sub_texts)1019 1020 clean_up_tokenization_spaces = (1021 clean_up_tokenization_spaces1022 if clean_up_tokenization_spaces is not None1023 else self.clean_up_tokenization_spaces1024 )1025 if clean_up_tokenization_spaces:1026 clean_text = self.clean_up_tokenization(text)1027 return clean_text1028 else:1029 return text1030