Aluode/PerceptionLabPortable
0
1# coding=utf-82# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Tokenization classes for ALBERT model."""16 17import os18import unicodedata19from shutil import copyfile20from typing import Any, Optional21 22import sentencepiece as spm23 24from ...tokenization_utils import AddedToken, PreTrainedTokenizer25from ...utils import logging26from ...utils.import_utils import requires27 28 29logger = logging.get_logger(__name__)30VOCAB_FILES_NAMES = {"vocab_file": "spiece.model"}31 32 33SPIECE_UNDERLINE = "▁"34 35 36@requires(backends=("sentencepiece",))37class AlbertTokenizer(PreTrainedTokenizer):38 """39 Construct an ALBERT tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).40 41 This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to42 this superclass for more information regarding those methods.43 44 Args:45 vocab_file (`str`):46 [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that47 contains the vocabulary necessary to instantiate a tokenizer.48 do_lower_case (`bool`, *optional*, defaults to `True`):49 Whether or not to lowercase the input when tokenizing.50 remove_space (`bool`, *optional*, defaults to `True`):51 Whether or not to strip the text when tokenizing (removing excess spaces before and after the string).52 keep_accents (`bool`, *optional*, defaults to `False`):53 Whether or not to keep accents when tokenizing.54 bos_token (`str`, *optional*, defaults to `"[CLS]"`):55 The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.56 57 <Tip>58 59 When building a sequence using special tokens, this is not the token that is used for the beginning of60 sequence. The token used is the `cls_token`.61 62 </Tip>63 64 eos_token (`str`, *optional*, defaults to `"[SEP]"`):65 The end of sequence token.66 67 <Tip>68 69 When building a sequence using special tokens, this is not the token that is used for the end of sequence.70 The token used is the `sep_token`.71 72 </Tip>73 74 unk_token (`str`, *optional*, defaults to `"<unk>"`):75 The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this76 token instead.77 sep_token (`str`, *optional*, defaults to `"[SEP]"`):78 The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for79 sequence classification or for a text and a question for question answering. It is also used as the last80 token of a sequence built with special tokens.81 pad_token (`str`, *optional*, defaults to `"<pad>"`):82 The token used for padding, for example when batching sequences of different lengths.83 cls_token (`str`, *optional*, defaults to `"[CLS]"`):84 The classifier token which is used when doing sequence classification (classification of the whole sequence85 instead of per-token classification). It is the first token of the sequence when built with special tokens.86 mask_token (`str`, *optional*, defaults to `"[MASK]"`):87 The token used for masking values. This is the token used when training this model with masked language88 modeling. This is the token which the model will try to predict.89 sp_model_kwargs (`dict`, *optional*):90 Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for91 SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,92 to set:93 94 - `enable_sampling`: Enable subword regularization.95 - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.96 97 - `nbest_size = {0,1}`: No sampling is performed.98 - `nbest_size > 1`: samples from the nbest_size results.99 - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)100 using forward-filtering-and-backward-sampling algorithm.101 102 - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for103 BPE-dropout.104 105 Attributes:106 sp_model (`SentencePieceProcessor`):107 The *SentencePiece* processor that is used for every conversion (string, tokens and IDs).108 """109 110 vocab_files_names = VOCAB_FILES_NAMES111 112 def __init__(113 self,114 vocab_file,115 do_lower_case=True,116 remove_space=True,117 keep_accents=False,118 bos_token="[CLS]",119 eos_token="[SEP]",120 unk_token="<unk>",121 sep_token="[SEP]",122 pad_token="<pad>",123 cls_token="[CLS]",124 mask_token="[MASK]",125 sp_model_kwargs: Optional[dict[str, Any]] = None,126 **kwargs,127 ) -> None:128 # Mask token behave like a normal word, i.e. include the space before it and129 # is included in the raw text, there should be a match in a non-normalized sentence.130 mask_token = (131 AddedToken(mask_token, lstrip=True, rstrip=False, normalized=False)132 if isinstance(mask_token, str)133 else mask_token134 )135 136 self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs137 138 self.do_lower_case = do_lower_case139 self.remove_space = remove_space140 self.keep_accents = keep_accents141 self.vocab_file = vocab_file142 143 self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)144 self.sp_model.Load(vocab_file)145 146 super().__init__(147 do_lower_case=do_lower_case,148 remove_space=remove_space,149 keep_accents=keep_accents,150 bos_token=bos_token,151 eos_token=eos_token,152 unk_token=unk_token,153 sep_token=sep_token,154 pad_token=pad_token,155 cls_token=cls_token,156 mask_token=mask_token,157 sp_model_kwargs=self.sp_model_kwargs,158 **kwargs,159 )160 161 @property162 def vocab_size(self) -> int:163 return len(self.sp_model)164 165 def get_vocab(self) -> dict[str, int]:166 vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}167 vocab.update(self.added_tokens_encoder)168 return vocab169 170 def __getstate__(self):171 state = self.__dict__.copy()172 state["sp_model"] = None173 return state174 175 def __setstate__(self, d):176 self.__dict__ = d177 178 # for backward compatibility179 if not hasattr(self, "sp_model_kwargs"):180 self.sp_model_kwargs = {}181 182 self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)183 self.sp_model.Load(self.vocab_file)184 185 def preprocess_text(self, inputs):186 if self.remove_space:187 outputs = " ".join(inputs.strip().split())188 else:189 outputs = inputs190 outputs = outputs.replace("``", '"').replace("''", '"')191 192 if not self.keep_accents:193 outputs = unicodedata.normalize("NFKD", outputs)194 outputs = "".join([c for c in outputs if not unicodedata.combining(c)])195 if self.do_lower_case:196 outputs = outputs.lower()197 198 return outputs199 200 def _tokenize(self, text: str) -> list[str]:201 """Tokenize a string."""202 text = self.preprocess_text(text)203 pieces = self.sp_model.encode(text, out_type=str)204 new_pieces = []205 for piece in pieces:206 if len(piece) > 1 and piece[-1] == "," and piece[-2].isdigit():207 # Logic to handle special cases see https://github.com/google-research/bert/blob/master/README.md#tokenization208 # `9,9` -> ['▁9', ',', '9'] instead of [`_9,`, '9']209 cur_pieces = self.sp_model.EncodeAsPieces(piece[:-1].replace(SPIECE_UNDERLINE, ""))210 if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:211 if len(cur_pieces[0]) == 1:212 cur_pieces = cur_pieces[1:]213 else:214 cur_pieces[0] = cur_pieces[0][1:]215 cur_pieces.append(piece[-1])216 new_pieces.extend(cur_pieces)217 else:218 new_pieces.append(piece)219 220 return new_pieces221 222 def _convert_token_to_id(self, token):223 """Converts a token (str) in an id using the vocab."""224 return self.sp_model.PieceToId(token)225 226 def _convert_id_to_token(self, index):227 """Converts an index (integer) in a token (str) using the vocab."""228 return self.sp_model.IdToPiece(index)229 230 def convert_tokens_to_string(self, tokens):231 """Converts a sequence of tokens (string) in a single string."""232 current_sub_tokens = []233 out_string = ""234 prev_is_special = False235 for token in tokens:236 # make sure that special tokens are not decoded using sentencepiece model237 if token in self.all_special_tokens:238 if not prev_is_special:239 out_string += " "240 out_string += self.sp_model.decode(current_sub_tokens) + token241 prev_is_special = True242 current_sub_tokens = []243 else:244 current_sub_tokens.append(token)245 prev_is_special = False246 out_string += self.sp_model.decode(current_sub_tokens)247 return out_string.strip()248 249 def build_inputs_with_special_tokens(250 self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None251 ) -> list[int]:252 """253 Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and254 adding special tokens. An ALBERT sequence has the following format:255 256 - single sequence: `[CLS] X [SEP]`257 - pair of sequences: `[CLS] A [SEP] B [SEP]`258 259 Args:260 token_ids_0 (`List[int]`):261 List of IDs to which the special tokens will be added.262 token_ids_1 (`List[int]`, *optional*):263 Optional second list of IDs for sequence pairs.264 265 Returns:266 `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.267 """268 sep = [self.sep_token_id]269 cls = [self.cls_token_id]270 if token_ids_1 is None:271 return cls + token_ids_0 + sep272 return cls + token_ids_0 + sep + token_ids_1 + sep273 274 def get_special_tokens_mask(275 self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None, already_has_special_tokens: bool = False276 ) -> list[int]:277 """278 Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding279 special tokens using the tokenizer `prepare_for_model` method.280 281 Args:282 token_ids_0 (`List[int]`):283 List of IDs.284 token_ids_1 (`List[int]`, *optional*):285 Optional second list of IDs for sequence pairs.286 already_has_special_tokens (`bool`, *optional*, defaults to `False`):287 Whether or not the token list is already formatted with special tokens for the model.288 289 Returns:290 `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.291 """292 293 if already_has_special_tokens:294 return super().get_special_tokens_mask(295 token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True296 )297 298 if token_ids_1 is not None:299 return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]300 return [1] + ([0] * len(token_ids_0)) + [1]301 302 def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:303 if not os.path.isdir(save_directory):304 logger.error(f"Vocabulary path ({save_directory}) should be a directory")305 return306 out_vocab_file = os.path.join(307 save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]308 )309 310 if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):311 copyfile(self.vocab_file, out_vocab_file)312 elif not os.path.isfile(self.vocab_file):313 with open(out_vocab_file, "wb") as fi:314 content_spiece_model = self.sp_model.serialized_model_proto()315 fi.write(content_spiece_model)316 317 return (out_vocab_file,)318 319 320__all__ = ["AlbertTokenizer"]321