Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2021 The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Tokenization classes for RoFormer."""16 17import collections18import os19import unicodedata20from typing import Optional21 22from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace23from ...utils import logging24 25 26logger = logging.get_logger(__name__)27 28VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}29 30 31# Copied from transformers.models.bert.tokenization_bert.load_vocab32def load_vocab(vocab_file):33 """Loads a vocabulary file into a dictionary."""34 vocab = collections.OrderedDict()35 with open(vocab_file, "r", encoding="utf-8") as reader:36 tokens = reader.readlines()37 for index, token in enumerate(tokens):38 token = token.rstrip("\n")39 vocab[token] = index40 return vocab41 42 43# Copied from transformers.models.bert.tokenization_bert.whitespace_tokenize44def whitespace_tokenize(text):45 """Runs basic whitespace cleaning and splitting on a piece of text."""46 text = text.strip()47 if not text:48 return []49 tokens = text.split()50 return tokens51 52 53# Copied from transformers.models.bert.tokenization_bert.BasicTokenizer54class BasicTokenizer:55 """56 Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).57 58 Args:59 do_lower_case (`bool`, *optional*, defaults to `True`):60 Whether or not to lowercase the input when tokenizing.61 never_split (`Iterable`, *optional*):62 Collection of tokens which will never be split during tokenization. Only has an effect when63 `do_basic_tokenize=True`64 tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):65 Whether or not to tokenize Chinese characters.66 67 This should likely be deactivated for Japanese (see this68 [issue](https://github.com/huggingface/transformers/issues/328)).69 strip_accents (`bool`, *optional*):70 Whether or not to strip all accents. If this option is not specified, then it will be determined by the71 value for `lowercase` (as in the original BERT).72 do_split_on_punc (`bool`, *optional*, defaults to `True`):73 In some instances we want to skip the basic punctuation splitting so that later tokenization can capture74 the full context of the words, such as contractions.75 """76 77 def __init__(78 self,79 do_lower_case=True,80 never_split=None,81 tokenize_chinese_chars=True,82 strip_accents=None,83 do_split_on_punc=True,84 ):85 if never_split is None:86 never_split = []87 self.do_lower_case = do_lower_case88 self.never_split = set(never_split)89 self.tokenize_chinese_chars = tokenize_chinese_chars90 self.strip_accents = strip_accents91 self.do_split_on_punc = do_split_on_punc92 93 def tokenize(self, text, never_split=None):94 """95 Basic Tokenization of a piece of text. For sub-word tokenization, see WordPieceTokenizer.96 97 Args:98 never_split (`List[str]`, *optional*)99 Kept for backward compatibility purposes. Now implemented directly at the base class level (see100 [`PreTrainedTokenizer.tokenize`]) List of token not to split.101 """102 # union() returns a new set by concatenating the two sets.103 never_split = self.never_split.union(set(never_split)) if never_split else self.never_split104 text = self._clean_text(text)105 106 # This was added on November 1st, 2018 for the multilingual and Chinese107 # models. This is also applied to the English models now, but it doesn't108 # matter since the English models were not trained on any Chinese data109 # and generally don't have any Chinese data in them (there are Chinese110 # characters in the vocabulary because Wikipedia does have some Chinese111 # words in the English Wikipedia.).112 if self.tokenize_chinese_chars:113 text = self._tokenize_chinese_chars(text)114 # prevents treating the same character with different unicode codepoints as different characters115 unicode_normalized_text = unicodedata.normalize("NFC", text)116 orig_tokens = whitespace_tokenize(unicode_normalized_text)117 split_tokens = []118 for token in orig_tokens:119 if token not in never_split:120 if self.do_lower_case:121 token = token.lower()122 if self.strip_accents is not False:123 token = self._run_strip_accents(token)124 elif self.strip_accents:125 token = self._run_strip_accents(token)126 split_tokens.extend(self._run_split_on_punc(token, never_split))127 128 output_tokens = whitespace_tokenize(" ".join(split_tokens))129 return output_tokens130 131 def _run_strip_accents(self, text):132 """Strips accents from a piece of text."""133 text = unicodedata.normalize("NFD", text)134 output = []135 for char in text:136 cat = unicodedata.category(char)137 if cat == "Mn":138 continue139 output.append(char)140 return "".join(output)141 142 def _run_split_on_punc(self, text, never_split=None):143 """Splits punctuation on a piece of text."""144 if not self.do_split_on_punc or (never_split is not None and text in never_split):145 return [text]146 chars = list(text)147 i = 0148 start_new_word = True149 output = []150 while i < len(chars):151 char = chars[i]152 if _is_punctuation(char):153 output.append([char])154 start_new_word = True155 else:156 if start_new_word:157 output.append([])158 start_new_word = False159 output[-1].append(char)160 i += 1161 162 return ["".join(x) for x in output]163 164 def _tokenize_chinese_chars(self, text):165 """Adds whitespace around any CJK character."""166 output = []167 for char in text:168 cp = ord(char)169 if self._is_chinese_char(cp):170 output.append(" ")171 output.append(char)172 output.append(" ")173 else:174 output.append(char)175 return "".join(output)176 177 def _is_chinese_char(self, cp):178 """Checks whether CP is the codepoint of a CJK character."""179 # This defines a "chinese character" as anything in the CJK Unicode block:180 # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)181 #182 # Note that the CJK Unicode block is NOT all Japanese and Korean characters,183 # despite its name. The modern Korean Hangul alphabet is a different block,184 # as is Japanese Hiragana and Katakana. Those alphabets are used to write185 # space-separated words, so they are not treated specially and handled186 # like the all of the other languages.187 if (188 (cp >= 0x4E00 and cp <= 0x9FFF)189 or (cp >= 0x3400 and cp <= 0x4DBF)190 or (cp >= 0x20000 and cp <= 0x2A6DF)191 or (cp >= 0x2A700 and cp <= 0x2B73F)192 or (cp >= 0x2B740 and cp <= 0x2B81F)193 or (cp >= 0x2B820 and cp <= 0x2CEAF)194 or (cp >= 0xF900 and cp <= 0xFAFF)195 or (cp >= 0x2F800 and cp <= 0x2FA1F)196 ):197 return True198 199 return False200 201 def _clean_text(self, text):202 """Performs invalid character removal and whitespace cleanup on text."""203 output = []204 for char in text:205 cp = ord(char)206 if cp == 0 or cp == 0xFFFD or _is_control(char):207 continue208 if _is_whitespace(char):209 output.append(" ")210 else:211 output.append(char)212 return "".join(output)213 214 215# Copied from transformers.models.bert.tokenization_bert.WordpieceTokenizer216class WordpieceTokenizer:217 """Runs WordPiece tokenization."""218 219 def __init__(self, vocab, unk_token, max_input_chars_per_word=100):220 self.vocab = vocab221 self.unk_token = unk_token222 self.max_input_chars_per_word = max_input_chars_per_word223 224 def tokenize(self, text):225 """226 Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform227 tokenization using the given vocabulary.228 229 For example, `input = "unaffable"` will return as output `["un", "##aff", "##able"]`.230 231 Args:232 text: A single token or whitespace separated tokens. This should have233 already been passed through *BasicTokenizer*.234 235 Returns:236 A list of wordpiece tokens.237 """238 239 output_tokens = []240 for token in whitespace_tokenize(text):241 chars = list(token)242 if len(chars) > self.max_input_chars_per_word:243 output_tokens.append(self.unk_token)244 continue245 246 is_bad = False247 start = 0248 sub_tokens = []249 while start < len(chars):250 end = len(chars)251 cur_substr = None252 while start < end:253 substr = "".join(chars[start:end])254 if start > 0:255 substr = "##" + substr256 if substr in self.vocab:257 cur_substr = substr258 break259 end -= 1260 if cur_substr is None:261 is_bad = True262 break263 sub_tokens.append(cur_substr)264 start = end265 266 if is_bad:267 output_tokens.append(self.unk_token)268 else:269 output_tokens.extend(sub_tokens)270 return output_tokens271 272 273class RoFormerTokenizer(PreTrainedTokenizer):274 r"""275 Construct a RoFormer tokenizer. Based on [Rust Jieba](https://pypi.org/project/rjieba/).276 277 This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to278 this superclass for more information regarding those methods.279 280 Args:281 vocab_file (`str`):282 File containing the vocabulary.283 do_lower_case (`bool`, *optional*, defaults to `True`):284 Whether or not to lowercase the input when tokenizing.285 do_basic_tokenize (`bool`, *optional*, defaults to `True`):286 Whether or not to do basic tokenization before WordPiece.287 never_split (`Iterable`, *optional*):288 Collection of tokens which will never be split during tokenization. Only has an effect when289 `do_basic_tokenize=True`290 unk_token (`str`, *optional*, defaults to `"[UNK]"`):291 The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this292 token instead.293 sep_token (`str`, *optional*, defaults to `"[SEP]"`):294 The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for295 sequence classification or for a text and a question for question answering. It is also used as the last296 token of a sequence built with special tokens.297 pad_token (`str`, *optional*, defaults to `"[PAD]"`):298 The token used for padding, for example when batching sequences of different lengths.299 cls_token (`str`, *optional*, defaults to `"[CLS]"`):300 The classifier token which is used when doing sequence classification (classification of the whole sequence301 instead of per-token classification). It is the first token of the sequence when built with special tokens.302 mask_token (`str`, *optional*, defaults to `"[MASK]"`):303 The token used for masking values. This is the token used when training this model with masked language304 modeling. This is the token which the model will try to predict.305 tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):306 Whether or not to tokenize Chinese characters.307 308 This should likely be deactivated for Japanese (see this309 [issue](https://github.com/huggingface/transformers/issues/328)).310 strip_accents (`bool`, *optional*):311 Whether or not to strip all accents. If this option is not specified, then it will be determined by the312 value for `lowercase` (as in the original BERT).313 314 Example:315 316 ```python317 >>> from transformers import RoFormerTokenizer318 319 >>> tokenizer = RoFormerTokenizer.from_pretrained("junnyu/roformer_chinese_base")320 >>> tokenizer.tokenize("今天天气非常好。")321 ['今', '天', '天', '气', '非常', '好', '。']322 ```"""323 324 vocab_files_names = VOCAB_FILES_NAMES325 326 def __init__(327 self,328 vocab_file,329 do_lower_case=True,330 do_basic_tokenize=True,331 never_split=None,332 unk_token="[UNK]",333 sep_token="[SEP]",334 pad_token="[PAD]",335 cls_token="[CLS]",336 mask_token="[MASK]",337 tokenize_chinese_chars=True,338 strip_accents=None,339 **kwargs,340 ):341 if not os.path.isfile(vocab_file):342 raise ValueError(343 f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"344 " model use `tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"345 )346 self.vocab = load_vocab(vocab_file)347 self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])348 self.do_basic_tokenize = do_basic_tokenize349 if do_basic_tokenize:350 self.basic_tokenizer = BasicTokenizer(351 do_lower_case=do_lower_case,352 never_split=never_split,353 tokenize_chinese_chars=tokenize_chinese_chars,354 strip_accents=strip_accents,355 )356 self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=str(unk_token))357 try:358 import rjieba359 except ImportError:360 raise ImportError(361 "You need to install rjieba to use RoFormerTokenizer. "362 "See https://pypi.org/project/rjieba/ for installation."363 )364 self.jieba = rjieba365 366 super().__init__(367 do_lower_case=do_lower_case,368 do_basic_tokenize=do_basic_tokenize,369 never_split=never_split,370 unk_token=unk_token,371 sep_token=sep_token,372 pad_token=pad_token,373 cls_token=cls_token,374 mask_token=mask_token,375 tokenize_chinese_chars=tokenize_chinese_chars,376 strip_accents=strip_accents,377 **kwargs,378 )379 380 @property381 def do_lower_case(self):382 return self.basic_tokenizer.do_lower_case383 384 @property385 def vocab_size(self):386 return len(self.vocab)387 388 def __getstate__(self):389 state = self.__dict__.copy()390 state["jieba"] = None391 return state392 393 def __setstate__(self, d):394 self.__dict__ = d395 import rjieba396 397 self.jieba = rjieba398 399 def get_vocab(self):400 return dict(self.vocab, **self.added_tokens_encoder)401 402 def _tokenize(self, text, use_jieba=True):403 split_tokens = []404 if use_jieba:405 for wholword in self.jieba.cut(text, False):406 if wholword in self.vocab:407 split_tokens.append(wholword)408 else:409 # use bert tokenizer to _tokenize410 char_list = self._tokenize(wholword, use_jieba=False)411 split_tokens.extend(char_list)412 else:413 if self.do_basic_tokenize:414 for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens):415 # If the token is part of the never_split set416 if token in self.basic_tokenizer.never_split:417 split_tokens.append(token)418 else:419 split_tokens += self.wordpiece_tokenizer.tokenize(token)420 else:421 split_tokens = self.wordpiece_tokenizer.tokenize(text)422 return split_tokens423 424 def _convert_token_to_id(self, token):425 """Converts a token (str) in an id using the vocab."""426 return self.vocab.get(token, self.vocab.get(self.unk_token))427 428 def _convert_id_to_token(self, index):429 """Converts an index (integer) in a token (str) using the vocab."""430 return self.ids_to_tokens.get(index, self.unk_token)431 432 def convert_tokens_to_string(self, tokens):433 """Converts a sequence of tokens (string) in a single string."""434 out_string = " ".join(tokens).replace(" ##", "").strip()435 return out_string436 437 def build_inputs_with_special_tokens(438 self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None439 ) -> list[int]:440 """441 Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and442 adding special tokens. A RoFormer sequence has the following format:443 444 - single sequence: `[CLS] X [SEP]`445 - pair of sequences: `[CLS] A [SEP] B [SEP]`446 447 Args:448 token_ids_0 (`List[int]`):449 List of IDs to which the special tokens will be added.450 token_ids_1 (`List[int]`, *optional*):451 Optional second list of IDs for sequence pairs.452 453 Returns:454 `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.455 """456 if token_ids_1 is None:457 return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]458 cls = [self.cls_token_id]459 sep = [self.sep_token_id]460 return cls + token_ids_0 + sep + token_ids_1 + sep461 462 def get_special_tokens_mask(463 self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None, already_has_special_tokens: bool = False464 ) -> list[int]:465 """466 Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding467 special tokens using the tokenizer `prepare_for_model` method.468 469 Args:470 token_ids_0 (`List[int]`):471 List of IDs.472 token_ids_1 (`List[int]`, *optional*):473 Optional second list of IDs for sequence pairs.474 already_has_special_tokens (`bool`, *optional*, defaults to `False`):475 Whether or not the token list is already formatted with special tokens for the model.476 477 Returns:478 `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.479 """480 481 if already_has_special_tokens:482 return super().get_special_tokens_mask(483 token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True484 )485 486 if token_ids_1 is not None:487 return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]488 return [1] + ([0] * len(token_ids_0)) + [1]489 490 def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:491 index = 0492 if os.path.isdir(save_directory):493 vocab_file = os.path.join(494 save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]495 )496 else:497 vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory498 with open(vocab_file, "w", encoding="utf-8") as writer:499 for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):500 if index != token_index:501 logger.warning(502 f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."503 " Please check that the vocabulary is not corrupted!"504 )505 index = token_index506 writer.write(token + "\n")507 index += 1508 return (vocab_file,)509 510 511__all__ = ["RoFormerTokenizer"]512