DoruC/Grounded-Segment-Anything
0
1# coding=utf-82# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Tokenization classes for Bert."""16 17 18import collections19import os20import unicodedata21from typing import List, Optional, Tuple22 23from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace24from ...utils import logging25 26 27logger = logging.get_logger(__name__)28 29VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}30 31PRETRAINED_VOCAB_FILES_MAP = {32 "vocab_file": {33 "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",34 "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",35 "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",36 "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",37 "bert-base-multilingual-uncased": (38 "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"39 ),40 "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",41 "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",42 "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",43 "bert-large-uncased-whole-word-masking": (44 "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"45 ),46 "bert-large-cased-whole-word-masking": (47 "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"48 ),49 "bert-large-uncased-whole-word-masking-finetuned-squad": (50 "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"51 ),52 "bert-large-cased-whole-word-masking-finetuned-squad": (53 "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"54 ),55 "bert-base-cased-finetuned-mrpc": (56 "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"57 ),58 "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",59 "bert-base-german-dbmdz-uncased": (60 "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"61 ),62 "TurkuNLP/bert-base-finnish-cased-v1": (63 "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"64 ),65 "TurkuNLP/bert-base-finnish-uncased-v1": (66 "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"67 ),68 "wietsedv/bert-base-dutch-cased": (69 "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"70 ),71 }72}73 74PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {75 "bert-base-uncased": 512,76 "bert-large-uncased": 512,77 "bert-base-cased": 512,78 "bert-large-cased": 512,79 "bert-base-multilingual-uncased": 512,80 "bert-base-multilingual-cased": 512,81 "bert-base-chinese": 512,82 "bert-base-german-cased": 512,83 "bert-large-uncased-whole-word-masking": 512,84 "bert-large-cased-whole-word-masking": 512,85 "bert-large-uncased-whole-word-masking-finetuned-squad": 512,86 "bert-large-cased-whole-word-masking-finetuned-squad": 512,87 "bert-base-cased-finetuned-mrpc": 512,88 "bert-base-german-dbmdz-cased": 512,89 "bert-base-german-dbmdz-uncased": 512,90 "TurkuNLP/bert-base-finnish-cased-v1": 512,91 "TurkuNLP/bert-base-finnish-uncased-v1": 512,92 "wietsedv/bert-base-dutch-cased": 512,93}94 95PRETRAINED_INIT_CONFIGURATION = {96 "bert-base-uncased": {"do_lower_case": True},97 "bert-large-uncased": {"do_lower_case": True},98 "bert-base-cased": {"do_lower_case": False},99 "bert-large-cased": {"do_lower_case": False},100 "bert-base-multilingual-uncased": {"do_lower_case": True},101 "bert-base-multilingual-cased": {"do_lower_case": False},102 "bert-base-chinese": {"do_lower_case": False},103 "bert-base-german-cased": {"do_lower_case": False},104 "bert-large-uncased-whole-word-masking": {"do_lower_case": True},105 "bert-large-cased-whole-word-masking": {"do_lower_case": False},106 "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},107 "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},108 "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},109 "bert-base-german-dbmdz-cased": {"do_lower_case": False},110 "bert-base-german-dbmdz-uncased": {"do_lower_case": True},111 "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},112 "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},113 "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},114}115 116 117def load_vocab(vocab_file):118 """Loads a vocabulary file into a dictionary."""119 vocab = collections.OrderedDict()120 with open(vocab_file, "r", encoding="utf-8") as reader:121 tokens = reader.readlines()122 for index, token in enumerate(tokens):123 token = token.rstrip("\n")124 vocab[token] = index125 return vocab126 127 128def whitespace_tokenize(text):129 """Runs basic whitespace cleaning and splitting on a piece of text."""130 text = text.strip()131 if not text:132 return []133 tokens = text.split()134 return tokens135 136 137class BertTokenizer(PreTrainedTokenizer):138 r"""139 Construct a BERT tokenizer. Based on WordPiece.140 141 This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to142 this superclass for more information regarding those methods.143 144 Args:145 vocab_file (`str`):146 File containing the vocabulary.147 do_lower_case (`bool`, *optional*, defaults to `True`):148 Whether or not to lowercase the input when tokenizing.149 do_basic_tokenize (`bool`, *optional*, defaults to `True`):150 Whether or not to do basic tokenization before WordPiece.151 never_split (`Iterable`, *optional*):152 Collection of tokens which will never be split during tokenization. Only has an effect when153 `do_basic_tokenize=True`154 unk_token (`str`, *optional*, defaults to `"[UNK]"`):155 The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this156 token instead.157 sep_token (`str`, *optional*, defaults to `"[SEP]"`):158 The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for159 sequence classification or for a text and a question for question answering. It is also used as the last160 token of a sequence built with special tokens.161 pad_token (`str`, *optional*, defaults to `"[PAD]"`):162 The token used for padding, for example when batching sequences of different lengths.163 cls_token (`str`, *optional*, defaults to `"[CLS]"`):164 The classifier token which is used when doing sequence classification (classification of the whole sequence165 instead of per-token classification). It is the first token of the sequence when built with special tokens.166 mask_token (`str`, *optional*, defaults to `"[MASK]"`):167 The token used for masking values. This is the token used when training this model with masked language168 modeling. This is the token which the model will try to predict.169 tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):170 Whether or not to tokenize Chinese characters.171 172 This should likely be deactivated for Japanese (see this173 [issue](https://github.com/huggingface/transformers/issues/328)).174 strip_accents (`bool`, *optional*):175 Whether or not to strip all accents. If this option is not specified, then it will be determined by the176 value for `lowercase` (as in the original BERT).177 """178 179 vocab_files_names = VOCAB_FILES_NAMES180 pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP181 pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION182 max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES183 184 def __init__(185 self,186 vocab_file,187 do_lower_case=True,188 do_basic_tokenize=True,189 never_split=None,190 unk_token="[UNK]",191 sep_token="[SEP]",192 pad_token="[PAD]",193 cls_token="[CLS]",194 mask_token="[MASK]",195 tokenize_chinese_chars=True,196 strip_accents=None,197 **kwargs,198 ):199 if not os.path.isfile(vocab_file):200 raise ValueError(201 f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"202 " model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"203 )204 self.vocab = load_vocab(vocab_file)205 self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])206 self.do_basic_tokenize = do_basic_tokenize207 if do_basic_tokenize:208 self.basic_tokenizer = BasicTokenizer(209 do_lower_case=do_lower_case,210 never_split=never_split,211 tokenize_chinese_chars=tokenize_chinese_chars,212 strip_accents=strip_accents,213 )214 215 self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=str(unk_token))216 217 super().__init__(218 do_lower_case=do_lower_case,219 do_basic_tokenize=do_basic_tokenize,220 never_split=never_split,221 unk_token=unk_token,222 sep_token=sep_token,223 pad_token=pad_token,224 cls_token=cls_token,225 mask_token=mask_token,226 tokenize_chinese_chars=tokenize_chinese_chars,227 strip_accents=strip_accents,228 **kwargs,229 )230 231 @property232 def do_lower_case(self):233 return self.basic_tokenizer.do_lower_case234 235 @property236 def vocab_size(self):237 return len(self.vocab)238 239 def get_vocab(self):240 return dict(self.vocab, **self.added_tokens_encoder)241 242 def _tokenize(self, text, split_special_tokens=False):243 split_tokens = []244 if self.do_basic_tokenize:245 for token in self.basic_tokenizer.tokenize(246 text, never_split=self.all_special_tokens if not split_special_tokens else None247 ):248 # If the token is part of the never_split set249 if token in self.basic_tokenizer.never_split:250 split_tokens.append(token)251 else:252 split_tokens += self.wordpiece_tokenizer.tokenize(token)253 else:254 split_tokens = self.wordpiece_tokenizer.tokenize(text)255 return split_tokens256 257 def _convert_token_to_id(self, token):258 """Converts a token (str) in an id using the vocab."""259 return self.vocab.get(token, self.vocab.get(self.unk_token))260 261 def _convert_id_to_token(self, index):262 """Converts an index (integer) in a token (str) using the vocab."""263 return self.ids_to_tokens.get(index, self.unk_token)264 265 def convert_tokens_to_string(self, tokens):266 """Converts a sequence of tokens (string) in a single string."""267 out_string = " ".join(tokens).replace(" ##", "").strip()268 return out_string269 270 def build_inputs_with_special_tokens(271 self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None272 ) -> List[int]:273 """274 Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and275 adding special tokens. A BERT sequence has the following format:276 277 - single sequence: `[CLS] X [SEP]`278 - pair of sequences: `[CLS] A [SEP] B [SEP]`279 280 Args:281 token_ids_0 (`List[int]`):282 List of IDs to which the special tokens will be added.283 token_ids_1 (`List[int]`, *optional*):284 Optional second list of IDs for sequence pairs.285 286 Returns:287 `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.288 """289 if token_ids_1 is None:290 return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]291 cls = [self.cls_token_id]292 sep = [self.sep_token_id]293 return cls + token_ids_0 + sep + token_ids_1 + sep294 295 def get_special_tokens_mask(296 self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False297 ) -> List[int]:298 """299 Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding300 special tokens using the tokenizer `prepare_for_model` method.301 302 Args:303 token_ids_0 (`List[int]`):304 List of IDs.305 token_ids_1 (`List[int]`, *optional*):306 Optional second list of IDs for sequence pairs.307 already_has_special_tokens (`bool`, *optional*, defaults to `False`):308 Whether or not the token list is already formatted with special tokens for the model.309 310 Returns:311 `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.312 """313 314 if already_has_special_tokens:315 return super().get_special_tokens_mask(316 token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True317 )318 319 if token_ids_1 is not None:320 return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]321 return [1] + ([0] * len(token_ids_0)) + [1]322 323 def create_token_type_ids_from_sequences(324 self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None325 ) -> List[int]:326 """327 Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence328 pair mask has the following format:329 330 ```331 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1332 | first sequence | second sequence |333 ```334 335 If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).336 337 Args:338 token_ids_0 (`List[int]`):339 List of IDs.340 token_ids_1 (`List[int]`, *optional*):341 Optional second list of IDs for sequence pairs.342 343 Returns:344 `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).345 """346 sep = [self.sep_token_id]347 cls = [self.cls_token_id]348 if token_ids_1 is None:349 return len(cls + token_ids_0 + sep) * [0]350 return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]351 352 def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:353 index = 0354 if os.path.isdir(save_directory):355 vocab_file = os.path.join(356 save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]357 )358 else:359 vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory360 with open(vocab_file, "w", encoding="utf-8") as writer:361 for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):362 if index != token_index:363 logger.warning(364 f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."365 " Please check that the vocabulary is not corrupted!"366 )367 index = token_index368 writer.write(token + "\n")369 index += 1370 return (vocab_file,)371 372 373class BasicTokenizer(object):374 """375 Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).376 377 Args:378 do_lower_case (`bool`, *optional*, defaults to `True`):379 Whether or not to lowercase the input when tokenizing.380 never_split (`Iterable`, *optional*):381 Collection of tokens which will never be split during tokenization. Only has an effect when382 `do_basic_tokenize=True`383 tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):384 Whether or not to tokenize Chinese characters.385 386 This should likely be deactivated for Japanese (see this387 [issue](https://github.com/huggingface/transformers/issues/328)).388 strip_accents (`bool`, *optional*):389 Whether or not to strip all accents. If this option is not specified, then it will be determined by the390 value for `lowercase` (as in the original BERT).391 do_split_on_punc (`bool`, *optional*, defaults to `True`):392 In some instances we want to skip the basic punctuation splitting so that later tokenization can capture393 the full context of the words, such as contractions.394 """395 396 def __init__(397 self,398 do_lower_case=True,399 never_split=None,400 tokenize_chinese_chars=True,401 strip_accents=None,402 do_split_on_punc=True,403 ):404 if never_split is None:405 never_split = []406 self.do_lower_case = do_lower_case407 self.never_split = set(never_split)408 self.tokenize_chinese_chars = tokenize_chinese_chars409 self.strip_accents = strip_accents410 self.do_split_on_punc = do_split_on_punc411 412 def tokenize(self, text, never_split=None):413 """414 Basic Tokenization of a piece of text. For sub-word tokenization, see WordPieceTokenizer.415 416 Args:417 never_split (`List[str]`, *optional*)418 Kept for backward compatibility purposes. Now implemented directly at the base class level (see419 [`PreTrainedTokenizer.tokenize`]) List of token not to split.420 """421 # union() returns a new set by concatenating the two sets.422 never_split = self.never_split.union(set(never_split)) if never_split else self.never_split423 text = self._clean_text(text)424 425 # This was added on November 1st, 2018 for the multilingual and Chinese426 # models. This is also applied to the English models now, but it doesn't427 # matter since the English models were not trained on any Chinese data428 # and generally don't have any Chinese data in them (there are Chinese429 # characters in the vocabulary because Wikipedia does have some Chinese430 # words in the English Wikipedia.).431 if self.tokenize_chinese_chars:432 text = self._tokenize_chinese_chars(text)433 # prevents treating the same character with different unicode codepoints as different characters434 unicode_normalized_text = unicodedata.normalize("NFC", text)435 orig_tokens = whitespace_tokenize(unicode_normalized_text)436 split_tokens = []437 for token in orig_tokens:438 if token not in never_split:439 if self.do_lower_case:440 token = token.lower()441 if self.strip_accents is not False:442 token = self._run_strip_accents(token)443 elif self.strip_accents:444 token = self._run_strip_accents(token)445 split_tokens.extend(self._run_split_on_punc(token, never_split))446 447 output_tokens = whitespace_tokenize(" ".join(split_tokens))448 return output_tokens449 450 def _run_strip_accents(self, text):451 """Strips accents from a piece of text."""452 text = unicodedata.normalize("NFD", text)453 output = []454 for char in text:455 cat = unicodedata.category(char)456 if cat == "Mn":457 continue458 output.append(char)459 return "".join(output)460 461 def _run_split_on_punc(self, text, never_split=None):462 """Splits punctuation on a piece of text."""463 if not self.do_split_on_punc or (never_split is not None and text in never_split):464 return [text]465 chars = list(text)466 i = 0467 start_new_word = True468 output = []469 while i < len(chars):470 char = chars[i]471 if _is_punctuation(char):472 output.append([char])473 start_new_word = True474 else:475 if start_new_word:476 output.append([])477 start_new_word = False478 output[-1].append(char)479 i += 1480 481 return ["".join(x) for x in output]482 483 def _tokenize_chinese_chars(self, text):484 """Adds whitespace around any CJK character."""485 output = []486 for char in text:487 cp = ord(char)488 if self._is_chinese_char(cp):489 output.append(" ")490 output.append(char)491 output.append(" ")492 else:493 output.append(char)494 return "".join(output)495 496 def _is_chinese_char(self, cp):497 """Checks whether CP is the codepoint of a CJK character."""498 # This defines a "chinese character" as anything in the CJK Unicode block:499 # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)500 #501 # Note that the CJK Unicode block is NOT all Japanese and Korean characters,502 # despite its name. The modern Korean Hangul alphabet is a different block,503 # as is Japanese Hiragana and Katakana. Those alphabets are used to write504 # space-separated words, so they are not treated specially and handled505 # like the all of the other languages.506 if (507 (cp >= 0x4E00 and cp <= 0x9FFF)508 or (cp >= 0x3400 and cp <= 0x4DBF) #509 or (cp >= 0x20000 and cp <= 0x2A6DF) #510 or (cp >= 0x2A700 and cp <= 0x2B73F) #511 or (cp >= 0x2B740 and cp <= 0x2B81F) #512 or (cp >= 0x2B820 and cp <= 0x2CEAF) #513 or (cp >= 0xF900 and cp <= 0xFAFF)514 or (cp >= 0x2F800 and cp <= 0x2FA1F) #515 ): #516 return True517 518 return False519 520 def _clean_text(self, text):521 """Performs invalid character removal and whitespace cleanup on text."""522 output = []523 for char in text:524 cp = ord(char)525 if cp == 0 or cp == 0xFFFD or _is_control(char):526 continue527 if _is_whitespace(char):528 output.append(" ")529 else:530 output.append(char)531 return "".join(output)532 533 534class WordpieceTokenizer(object):535 """Runs WordPiece tokenization."""536 537 def __init__(self, vocab, unk_token, max_input_chars_per_word=100):538 self.vocab = vocab539 self.unk_token = unk_token540 self.max_input_chars_per_word = max_input_chars_per_word541 542 def tokenize(self, text):543 """544 Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform545 tokenization using the given vocabulary.546 547 For example, `input = "unaffable"` wil return as output `["un", "##aff", "##able"]`.548 549 Args:550 text: A single token or whitespace separated tokens. This should have551 already been passed through *BasicTokenizer*.552 553 Returns:554 A list of wordpiece tokens.555 """556 557 output_tokens = []558 for token in whitespace_tokenize(text):559 chars = list(token)560 if len(chars) > self.max_input_chars_per_word:561 output_tokens.append(self.unk_token)562 continue563 564 is_bad = False565 start = 0566 sub_tokens = []567 while start < len(chars):568 end = len(chars)569 cur_substr = None570 while start < end:571 substr = "".join(chars[start:end])572 if start > 0:573 substr = "##" + substr574 if substr in self.vocab:575 cur_substr = substr576 break577 end -= 1578 if cur_substr is None:579 is_bad = True580 break581 sub_tokens.append(cur_substr)582 start = end583 584 if is_bad:585 output_tokens.append(self.unk_token)586 else:587 output_tokens.extend(sub_tokens)588 return output_tokens589