Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2018 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Tokenization classes for ConvBERT."""16 17import collections18import os19import unicodedata20from typing import Optional21 22from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace23from ...utils import logging24 25 26logger = logging.get_logger(__name__)27 28VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}29 30 31# Copied from transformers.models.bert.tokenization_bert.load_vocab32def load_vocab(vocab_file):33 """Loads a vocabulary file into a dictionary."""34 vocab = collections.OrderedDict()35 with open(vocab_file, "r", encoding="utf-8") as reader:36 tokens = reader.readlines()37 for index, token in enumerate(tokens):38 token = token.rstrip("\n")39 vocab[token] = index40 return vocab41 42 43# Copied from transformers.models.bert.tokenization_bert.whitespace_tokenize44def whitespace_tokenize(text):45 """Runs basic whitespace cleaning and splitting on a piece of text."""46 text = text.strip()47 if not text:48 return []49 tokens = text.split()50 return tokens51 52 53# Copied from transformers.models.bert.tokenization_bert.BertTokenizer with bert-base-cased->YituTech/conv-bert-base, ConvBertTokenizer->BertTokenizer, BERT->ConvBERT54class ConvBertTokenizer(PreTrainedTokenizer):55 r"""56 Construct a ConvBERT tokenizer. Based on WordPiece.57 58 This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to59 this superclass for more information regarding those methods.60 61 Args:62 vocab_file (`str`):63 File containing the vocabulary.64 do_lower_case (`bool`, *optional*, defaults to `True`):65 Whether or not to lowercase the input when tokenizing.66 do_basic_tokenize (`bool`, *optional*, defaults to `True`):67 Whether or not to do basic tokenization before WordPiece.68 never_split (`Iterable`, *optional*):69 Collection of tokens which will never be split during tokenization. Only has an effect when70 `do_basic_tokenize=True`71 unk_token (`str`, *optional*, defaults to `"[UNK]"`):72 The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this73 token instead.74 sep_token (`str`, *optional*, defaults to `"[SEP]"`):75 The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for76 sequence classification or for a text and a question for question answering. It is also used as the last77 token of a sequence built with special tokens.78 pad_token (`str`, *optional*, defaults to `"[PAD]"`):79 The token used for padding, for example when batching sequences of different lengths.80 cls_token (`str`, *optional*, defaults to `"[CLS]"`):81 The classifier token which is used when doing sequence classification (classification of the whole sequence82 instead of per-token classification). It is the first token of the sequence when built with special tokens.83 mask_token (`str`, *optional*, defaults to `"[MASK]"`):84 The token used for masking values. This is the token used when training this model with masked language85 modeling. This is the token which the model will try to predict.86 tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):87 Whether or not to tokenize Chinese characters.88 89 This should likely be deactivated for Japanese (see this90 [issue](https://github.com/huggingface/transformers/issues/328)).91 strip_accents (`bool`, *optional*):92 Whether or not to strip all accents. If this option is not specified, then it will be determined by the93 value for `lowercase` (as in the original ConvBERT).94 clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`):95 Whether or not to cleanup spaces after decoding, cleanup consists in removing potential artifacts like96 extra spaces.97 """98 99 vocab_files_names = VOCAB_FILES_NAMES100 101 def __init__(102 self,103 vocab_file,104 do_lower_case=True,105 do_basic_tokenize=True,106 never_split=None,107 unk_token="[UNK]",108 sep_token="[SEP]",109 pad_token="[PAD]",110 cls_token="[CLS]",111 mask_token="[MASK]",112 tokenize_chinese_chars=True,113 strip_accents=None,114 clean_up_tokenization_spaces=True,115 **kwargs,116 ):117 if not os.path.isfile(vocab_file):118 raise ValueError(119 f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"120 " model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"121 )122 self.vocab = load_vocab(vocab_file)123 self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])124 self.do_basic_tokenize = do_basic_tokenize125 if do_basic_tokenize:126 self.basic_tokenizer = BasicTokenizer(127 do_lower_case=do_lower_case,128 never_split=never_split,129 tokenize_chinese_chars=tokenize_chinese_chars,130 strip_accents=strip_accents,131 )132 133 self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=str(unk_token))134 135 super().__init__(136 do_lower_case=do_lower_case,137 do_basic_tokenize=do_basic_tokenize,138 never_split=never_split,139 unk_token=unk_token,140 sep_token=sep_token,141 pad_token=pad_token,142 cls_token=cls_token,143 mask_token=mask_token,144 tokenize_chinese_chars=tokenize_chinese_chars,145 strip_accents=strip_accents,146 clean_up_tokenization_spaces=clean_up_tokenization_spaces,147 **kwargs,148 )149 150 @property151 def do_lower_case(self):152 return self.basic_tokenizer.do_lower_case153 154 @property155 def vocab_size(self):156 return len(self.vocab)157 158 def get_vocab(self):159 return dict(self.vocab, **self.added_tokens_encoder)160 161 def _tokenize(self, text, split_special_tokens=False):162 split_tokens = []163 if self.do_basic_tokenize:164 for token in self.basic_tokenizer.tokenize(165 text, never_split=self.all_special_tokens if not split_special_tokens else None166 ):167 # If the token is part of the never_split set168 if token in self.basic_tokenizer.never_split:169 split_tokens.append(token)170 else:171 split_tokens += self.wordpiece_tokenizer.tokenize(token)172 else:173 split_tokens = self.wordpiece_tokenizer.tokenize(text)174 return split_tokens175 176 def _convert_token_to_id(self, token):177 """Converts a token (str) in an id using the vocab."""178 return self.vocab.get(token, self.vocab.get(self.unk_token))179 180 def _convert_id_to_token(self, index):181 """Converts an index (integer) in a token (str) using the vocab."""182 return self.ids_to_tokens.get(index, self.unk_token)183 184 def convert_tokens_to_string(self, tokens):185 """Converts a sequence of tokens (string) in a single string."""186 out_string = " ".join(tokens).replace(" ##", "").strip()187 return out_string188 189 def build_inputs_with_special_tokens(190 self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None191 ) -> list[int]:192 """193 Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and194 adding special tokens. A ConvBERT sequence has the following format:195 196 - single sequence: `[CLS] X [SEP]`197 - pair of sequences: `[CLS] A [SEP] B [SEP]`198 199 Args:200 token_ids_0 (`List[int]`):201 List of IDs to which the special tokens will be added.202 token_ids_1 (`List[int]`, *optional*):203 Optional second list of IDs for sequence pairs.204 205 Returns:206 `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.207 """208 if token_ids_1 is None:209 return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]210 cls = [self.cls_token_id]211 sep = [self.sep_token_id]212 return cls + token_ids_0 + sep + token_ids_1 + sep213 214 def get_special_tokens_mask(215 self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None, already_has_special_tokens: bool = False216 ) -> list[int]:217 """218 Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding219 special tokens using the tokenizer `prepare_for_model` method.220 221 Args:222 token_ids_0 (`List[int]`):223 List of IDs.224 token_ids_1 (`List[int]`, *optional*):225 Optional second list of IDs for sequence pairs.226 already_has_special_tokens (`bool`, *optional*, defaults to `False`):227 Whether or not the token list is already formatted with special tokens for the model.228 229 Returns:230 `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.231 """232 233 if already_has_special_tokens:234 return super().get_special_tokens_mask(235 token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True236 )237 238 if token_ids_1 is not None:239 return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]240 return [1] + ([0] * len(token_ids_0)) + [1]241 242 def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:243 index = 0244 if os.path.isdir(save_directory):245 vocab_file = os.path.join(246 save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]247 )248 else:249 vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory250 with open(vocab_file, "w", encoding="utf-8") as writer:251 for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):252 if index != token_index:253 logger.warning(254 f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."255 " Please check that the vocabulary is not corrupted!"256 )257 index = token_index258 writer.write(token + "\n")259 index += 1260 return (vocab_file,)261 262 263# Copied from transformers.models.bert.tokenization_bert.BasicTokenizer264class BasicTokenizer:265 """266 Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).267 268 Args:269 do_lower_case (`bool`, *optional*, defaults to `True`):270 Whether or not to lowercase the input when tokenizing.271 never_split (`Iterable`, *optional*):272 Collection of tokens which will never be split during tokenization. Only has an effect when273 `do_basic_tokenize=True`274 tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):275 Whether or not to tokenize Chinese characters.276 277 This should likely be deactivated for Japanese (see this278 [issue](https://github.com/huggingface/transformers/issues/328)).279 strip_accents (`bool`, *optional*):280 Whether or not to strip all accents. If this option is not specified, then it will be determined by the281 value for `lowercase` (as in the original BERT).282 do_split_on_punc (`bool`, *optional*, defaults to `True`):283 In some instances we want to skip the basic punctuation splitting so that later tokenization can capture284 the full context of the words, such as contractions.285 """286 287 def __init__(288 self,289 do_lower_case=True,290 never_split=None,291 tokenize_chinese_chars=True,292 strip_accents=None,293 do_split_on_punc=True,294 ):295 if never_split is None:296 never_split = []297 self.do_lower_case = do_lower_case298 self.never_split = set(never_split)299 self.tokenize_chinese_chars = tokenize_chinese_chars300 self.strip_accents = strip_accents301 self.do_split_on_punc = do_split_on_punc302 303 def tokenize(self, text, never_split=None):304 """305 Basic Tokenization of a piece of text. For sub-word tokenization, see WordPieceTokenizer.306 307 Args:308 never_split (`List[str]`, *optional*)309 Kept for backward compatibility purposes. Now implemented directly at the base class level (see310 [`PreTrainedTokenizer.tokenize`]) List of token not to split.311 """312 # union() returns a new set by concatenating the two sets.313 never_split = self.never_split.union(set(never_split)) if never_split else self.never_split314 text = self._clean_text(text)315 316 # This was added on November 1st, 2018 for the multilingual and Chinese317 # models. This is also applied to the English models now, but it doesn't318 # matter since the English models were not trained on any Chinese data319 # and generally don't have any Chinese data in them (there are Chinese320 # characters in the vocabulary because Wikipedia does have some Chinese321 # words in the English Wikipedia.).322 if self.tokenize_chinese_chars:323 text = self._tokenize_chinese_chars(text)324 # prevents treating the same character with different unicode codepoints as different characters325 unicode_normalized_text = unicodedata.normalize("NFC", text)326 orig_tokens = whitespace_tokenize(unicode_normalized_text)327 split_tokens = []328 for token in orig_tokens:329 if token not in never_split:330 if self.do_lower_case:331 token = token.lower()332 if self.strip_accents is not False:333 token = self._run_strip_accents(token)334 elif self.strip_accents:335 token = self._run_strip_accents(token)336 split_tokens.extend(self._run_split_on_punc(token, never_split))337 338 output_tokens = whitespace_tokenize(" ".join(split_tokens))339 return output_tokens340 341 def _run_strip_accents(self, text):342 """Strips accents from a piece of text."""343 text = unicodedata.normalize("NFD", text)344 output = []345 for char in text:346 cat = unicodedata.category(char)347 if cat == "Mn":348 continue349 output.append(char)350 return "".join(output)351 352 def _run_split_on_punc(self, text, never_split=None):353 """Splits punctuation on a piece of text."""354 if not self.do_split_on_punc or (never_split is not None and text in never_split):355 return [text]356 chars = list(text)357 i = 0358 start_new_word = True359 output = []360 while i < len(chars):361 char = chars[i]362 if _is_punctuation(char):363 output.append([char])364 start_new_word = True365 else:366 if start_new_word:367 output.append([])368 start_new_word = False369 output[-1].append(char)370 i += 1371 372 return ["".join(x) for x in output]373 374 def _tokenize_chinese_chars(self, text):375 """Adds whitespace around any CJK character."""376 output = []377 for char in text:378 cp = ord(char)379 if self._is_chinese_char(cp):380 output.append(" ")381 output.append(char)382 output.append(" ")383 else:384 output.append(char)385 return "".join(output)386 387 def _is_chinese_char(self, cp):388 """Checks whether CP is the codepoint of a CJK character."""389 # This defines a "chinese character" as anything in the CJK Unicode block:390 # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)391 #392 # Note that the CJK Unicode block is NOT all Japanese and Korean characters,393 # despite its name. The modern Korean Hangul alphabet is a different block,394 # as is Japanese Hiragana and Katakana. Those alphabets are used to write395 # space-separated words, so they are not treated specially and handled396 # like the all of the other languages.397 if (398 (cp >= 0x4E00 and cp <= 0x9FFF)399 or (cp >= 0x3400 and cp <= 0x4DBF)400 or (cp >= 0x20000 and cp <= 0x2A6DF)401 or (cp >= 0x2A700 and cp <= 0x2B73F)402 or (cp >= 0x2B740 and cp <= 0x2B81F)403 or (cp >= 0x2B820 and cp <= 0x2CEAF)404 or (cp >= 0xF900 and cp <= 0xFAFF)405 or (cp >= 0x2F800 and cp <= 0x2FA1F)406 ):407 return True408 409 return False410 411 def _clean_text(self, text):412 """Performs invalid character removal and whitespace cleanup on text."""413 output = []414 for char in text:415 cp = ord(char)416 if cp == 0 or cp == 0xFFFD or _is_control(char):417 continue418 if _is_whitespace(char):419 output.append(" ")420 else:421 output.append(char)422 return "".join(output)423 424 425# Copied from transformers.models.bert.tokenization_bert.WordpieceTokenizer426class WordpieceTokenizer:427 """Runs WordPiece tokenization."""428 429 def __init__(self, vocab, unk_token, max_input_chars_per_word=100):430 self.vocab = vocab431 self.unk_token = unk_token432 self.max_input_chars_per_word = max_input_chars_per_word433 434 def tokenize(self, text):435 """436 Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform437 tokenization using the given vocabulary.438 439 For example, `input = "unaffable"` will return as output `["un", "##aff", "##able"]`.440 441 Args:442 text: A single token or whitespace separated tokens. This should have443 already been passed through *BasicTokenizer*.444 445 Returns:446 A list of wordpiece tokens.447 """448 449 output_tokens = []450 for token in whitespace_tokenize(text):451 chars = list(token)452 if len(chars) > self.max_input_chars_per_word:453 output_tokens.append(self.unk_token)454 continue455 456 is_bad = False457 start = 0458 sub_tokens = []459 while start < len(chars):460 end = len(chars)461 cur_substr = None462 while start < end:463 substr = "".join(chars[start:end])464 if start > 0:465 substr = "##" + substr466 if substr in self.vocab:467 cur_substr = substr468 break469 end -= 1470 if cur_substr is None:471 is_bad = True472 break473 sub_tokens.append(cur_substr)474 start = end475 476 if is_bad:477 output_tokens.append(self.unk_token)478 else:479 output_tokens.extend(sub_tokens)480 return output_tokens481 482 483__all__ = ["ConvBertTokenizer"]484