Multilingual-Multimodal-NLP/LoopCoder-V2
49182
1"""Tokenization classes for IQuestCoder."""2 3import os4from shutil import copyfile5from typing import Any, Dict, List, Optional, Tuple, Union6 7import sentencepiece as spm8 9from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer10from transformers.utils import logging11 12 13logger = logging.get_logger(__name__)14 15VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}16 17PRETRAINED_VOCAB_FILES_MAP = {18 "vocab_file": {},19 "tokenizer_file": {},20}21PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}22 23 24 25class IQuestCoderTokenizer(PreTrainedTokenizer):26 27 vocab_files_names = VOCAB_FILES_NAMES28 pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP29 max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES30 model_input_names = ["input_ids", "attention_mask"]31 32 def __init__(33 self,34 vocab_file,35 unk_token="<unk>",36 bos_token="<s>",37 eos_token="</s>",38 pad_token=None,39 sp_model_kwargs: Optional[Dict[str, Any]] = None,40 add_bos_token=True,41 add_eos_token=False,42 clean_up_tokenization_spaces=False,43 add_prefix_space=False,44 legacy=None,45 use_default_system_prompt=False,46 chat_template=None,47 **kwargs,48 ):49 self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs50 bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token51 eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token52 unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token53 pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token54 55 # Legacy behavior handling56 if legacy is None:57 logger.warning_once(58 f"You are using the default legacy behaviour of the {self.__class__.__name__}. This is"59 " expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you."60 " If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it"61 " means, and thoroughly read the reason why this was added as explained in"62 " https://github.com/huggingface/transformers/pull/24565"63 )64 legacy = True65 66 self.legacy = legacy67 self.vocab_file = vocab_file68 self.add_bos_token = add_bos_token69 self.add_eos_token = add_eos_token70 self.add_prefix_space = add_prefix_space71 self.use_default_system_prompt = use_default_system_prompt72 self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)73 self.sp_model.Load(vocab_file)74 75 76 77 super().__init__(78 bos_token=bos_token,79 eos_token=eos_token,80 unk_token=unk_token,81 pad_token=pad_token,82 add_bos_token=add_bos_token,83 add_eos_token=add_eos_token,84 sp_model_kwargs=self.sp_model_kwargs,85 clean_up_tokenization_spaces=clean_up_tokenization_spaces,86 add_prefix_space=add_prefix_space,87 legacy=legacy,88 use_default_system_prompt=use_default_system_prompt,89 chat_template=chat_template,90 **kwargs,91 )92 93 def __getstate__(self):94 state = self.__dict__.copy()95 state["sp_model"] = None96 return state97 98 def __setstate__(self, d):99 self.__dict__ = d100 self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)101 self.sp_model.Load(self.vocab_file)102 103 @property104 def vocab_size(self) -> int:105 """Returns the vocabulary size."""106 return self.sp_model.get_piece_size()107 108 def get_vocab(self) -> Dict[str, int]:109 """Returns the vocabulary as a dictionary of token to index."""110 vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}111 vocab.update(self.added_tokens_encoder)112 return vocab113 114 def _tokenize(self, text: str) -> List[str]:115 """116 Tokenize a string.117 118 Args:119 text (`str`): The text to tokenize.120 121 Returns:122 `List[str]`: The list of tokens.123 """124 if self.add_prefix_space:125 text = " " + text126 127 if self.legacy:128 return self.sp_model.encode(text, out_type=str)129 130 # Non-legacy behavior: handle special tokens properly131 return self.sp_model.encode(text, out_type=str)132 133 def _convert_token_to_id(self, token: str) -> int:134 """Converts a token (str) to an id using the vocab."""135 return self.sp_model.piece_to_id(token)136 137 def _convert_id_to_token(self, index: int) -> str:138 """Converts an index (integer) to a token (str) using the vocab."""139 token = self.sp_model.IdToPiece(index)140 return token141 142 def convert_tokens_to_string(self, tokens: List[str]) -> str:143 """144 Converts a sequence of tokens (strings) to a single string.145 146 This method handles special tokens separately to ensure they are not147 decoded using the SentencePiece model.148 149 Args:150 tokens (`List[str]`): The list of tokens to convert.151 152 Returns:153 `str`: The decoded string.154 """155 current_sub_tokens = []156 out_string = ""157 prev_is_special = False158 for i, token in enumerate(tokens):159 # make sure that special tokens are not decoded using sentencepiece model160 if token in self.all_special_tokens:161 if not prev_is_special and i != 0:162 out_string += " "163 out_string += self.sp_model.decode(current_sub_tokens) + token164 prev_is_special = True165 current_sub_tokens = []166 else:167 current_sub_tokens.append(token)168 prev_is_special = False169 out_string += self.sp_model.decode(current_sub_tokens)170 return out_string171 172 def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:173 """174 Save the vocabulary and special tokens file to a directory.175 176 Args:177 save_directory (`str`):178 The directory in which to save the vocabulary.179 filename_prefix (`str`, *optional*):180 An optional prefix to add to the named of the saved files.181 182 Returns:183 `Tuple(str)`: Paths to the files saved.184 """185 if not os.path.isdir(save_directory):186 logger.error(f"Vocabulary path ({save_directory}) should be a directory")187 return188 out_vocab_file = os.path.join(189 save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]190 )191 192 if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):193 copyfile(self.vocab_file, out_vocab_file)194 elif not os.path.isfile(self.vocab_file):195 with open(out_vocab_file, "wb") as fi:196 content_spiece_model = self.sp_model.serialized_model_proto()197 fi.write(content_spiece_model)198 199 return (out_vocab_file,)200 201 def build_inputs_with_special_tokens(202 self, 203 token_ids_0: List[int], 204 token_ids_1: Optional[List[int]] = None205 ) -> List[int]:206 """207 Build model inputs from a sequence or a pair of sequences for sequence classification tasks by concatenating208 and adding special tokens.209 210 An IQuestCoder sequence has the following format:211 212 - single sequence: `<s> X </s>` (if add_eos_token is True) or `<s> X` (default)213 - pair of sequences: `<s> A </s> <s> B </s>` (if add_eos_token is True) or `<s> A <s> B` (default)214 215 Args:216 token_ids_0 (`List[int]`):217 List of IDs to which the special tokens will be added.218 token_ids_1 (`List[int]`, *optional*):219 Optional second list of IDs for sequence pairs.220 221 Returns:222 `List[int]`: List of input IDs with the appropriate special tokens.223 """224 bos_token_id = [self.bos_token_id] if self.add_bos_token else []225 eos_token_id = [self.eos_token_id] if self.add_eos_token else []226 227 output = bos_token_id + token_ids_0 + eos_token_id228 229 if token_ids_1 is not None:230 output = output + bos_token_id + token_ids_1 + eos_token_id231 232 return output233 234 def get_special_tokens_mask(235 self, 236 token_ids_0: List[int], 237 token_ids_1: Optional[List[int]] = None, 238 already_has_special_tokens: bool = False239 ) -> List[int]:240 """241 Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding242 special tokens using the tokenizer `prepare_for_model` method.243 244 Args:245 token_ids_0 (`List[int]`):246 List of IDs.247 token_ids_1 (`List[int]`, *optional*):248 Optional second list of IDs for sequence pairs.249 already_has_special_tokens (`bool`, *optional*, defaults to `False`):250 Whether or not the token list is already formatted with special tokens for the model.251 252 Returns:253 `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.254 """255 if already_has_special_tokens:256 return super().get_special_tokens_mask(257 token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True258 )259 260 bos_token_id = [1] if self.add_bos_token else []261 eos_token_id = [1] if self.add_eos_token else []262 263 if token_ids_1 is None:264 return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id265 return (266 bos_token_id267 + ([0] * len(token_ids_0))268 + eos_token_id269 + bos_token_id270 + ([0] * len(token_ids_1))271 + eos_token_id272 )273 274 def create_token_type_ids_from_sequences(275 self, 276 token_ids_0: List[int], 277 token_ids_1: Optional[List[int]] = None278 ) -> List[int]:279 """280 Create a mask from the two sequences passed to be used in a sequence-pair classification task.281 282 An IQuestCoder sequence pair mask has the following format:283 284 ```285 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1286 | first sequence | second sequence |287 ```288 289 If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).290 291 Args:292 token_ids_0 (`List[int]`):293 List of IDs.294 token_ids_1 (`List[int]`, *optional*):295 Optional second list of IDs for sequence pairs.296 297 Returns:298 `List[int]`: List of token type IDs according to the given sequence(s).299 """300 bos_token_id = [self.bos_token_id] if self.add_bos_token else []301 eos_token_id = [self.eos_token_id] if self.add_eos_token else []302 303 output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)304 305 if token_ids_1 is not None:306 output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)307 308 return output309 310 @property311 def default_chat_template(self) -> str:312 """313 Returns the default chat template for IQuestCoder.314 315 This template formats conversations with system, user, and assistant roles.316 """317 return DEFAULT_CHAT_TEMPLATE318 319 def apply_chat_template(320 self,321 conversation: Union[List[Dict[str, str]], "Conversation"],322 chat_template: Optional[str] = None,323 add_generation_prompt: bool = False,324 tokenize: bool = True,325 padding: bool = False,326 truncation: bool = False,327 max_length: Optional[int] = None,328 return_tensors: Optional[str] = None,329 return_dict: bool = False,330 **tokenizer_kwargs,331 ):332 """333 Apply a chat template to format a conversation.334 335 Args:336 conversation (`List[Dict[str, str]]` or `Conversation`):337 A list of dicts with "role" and "content" keys, representing the conversation history.338 chat_template (`str`, *optional*):339 A Jinja template to use for formatting. If not provided, the tokenizer's default will be used.340 add_generation_prompt (`bool`, *optional*, defaults to `False`):341 Whether to add a generation prompt at the end for the assistant to continue.342 tokenize (`bool`, *optional*, defaults to `True`):343 Whether to tokenize the output. If `False`, returns a string.344 padding (`bool`, *optional*, defaults to `False`):345 Whether to pad sequences.346 truncation (`bool`, *optional*, defaults to `False`):347 Whether to truncate sequences.348 max_length (`int`, *optional*):349 Maximum length of the output.350 return_tensors (`str`, *optional*):351 The type of tensors to return ("pt", "tf", "np", or None).352 return_dict (`bool`, *optional*, defaults to `False`):353 Whether to return a dictionary with additional information.354 **tokenizer_kwargs:355 Additional keyword arguments passed to the tokenizer.356 357 Returns:358 `Union[str, List[int], BatchEncoding]`: The formatted (and optionally tokenized) conversation.359 360 Example:361 ```python362 >>> tokenizer = IQuestCoderTokenizer.from_pretrained("path/to/model")363 >>> conversation = [364 ... {"role": "system", "content": "You are a helpful assistant."},365 ... {"role": "user", "content": "Hello!"},366 ... {"role": "assistant", "content": "Hi there! How can I help you today?"},367 ... {"role": "user", "content": "What's the weather like?"},368 ... ]369 >>> tokenizer.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)370 '<|system|>\\nYou are a helpful assistant.\\n</|system|><|user|>\\nHello!\\n</|user|>...'371 ```372 """373 # Use parent class implementation with our template374 return super().apply_chat_template(375 conversation,376 chat_template=chat_template,377 add_generation_prompt=add_generation_prompt,378 tokenize=tokenize,379 padding=padding,380 truncation=truncation,381 max_length=max_length,382 return_tensors=return_tensors,383 return_dict=return_dict,384 **tokenizer_kwargs,385 )386 387 388# Try to import and create Fast tokenizer version389try:390 from transformers import PreTrainedTokenizerFast391 from tokenizers import Tokenizer, decoders, models, normalizers, pre_tokenizers, processors392 393 class IQuestCoderTokenizerFast(PreTrainedTokenizerFast):394 """395 Construct a "fast" IQuestCoder tokenizer (backed by HuggingFace's *tokenizers* library).396 397 This is a fast implementation of [`IQuestCoderTokenizer`] using the 🤗 Tokenizers library.398 399 Args:400 vocab_file (`str`, *optional*):401 Path to the vocabulary file (SentencePiece model).402 tokenizer_file (`str`, *optional*):403 Path to a tokenizer JSON file.404 unk_token (`str`, *optional*, defaults to `"<unk>"`):405 The unknown token.406 bos_token (`str`, *optional*, defaults to `"<s>"`):407 The beginning of sequence token.408 eos_token (`str`, *optional*, defaults to `"</s>"`):409 The end of sequence token.410 pad_token (`str`, *optional*):411 The token used for padding.412 add_bos_token (`bool`, *optional*, defaults to `True`):413 Whether to add a BOS token at the start of sequences.414 add_eos_token (`bool`, *optional*, defaults to `False`):415 Whether to add an EOS token at the end of sequences.416 add_prefix_space (`bool`, *optional*, defaults to `False`):417 Whether to add an initial space to the input.418 use_default_system_prompt (`bool`, *optional*, defaults to `False`):419 Whether to use the default system prompt.420 chat_template (`str`, *optional*):421 A Jinja template for formatting conversations.422 423 Example:424 ```python425 >>> from tokenization_iquestcoder import IQuestCoderTokenizerFast426 427 >>> tokenizer = IQuestCoderTokenizerFast.from_pretrained("path/to/model")428 >>> tokenizer.encode("Hello, world!")429 [1, 15043, 29892, 3186, 29991]430 ```431 """432 433 vocab_files_names = VOCAB_FILES_NAMES434 pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP435 max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES436 model_input_names = ["input_ids", "attention_mask"]437 slow_tokenizer_class = IQuestCoderTokenizer438 439 def __init__(440 self,441 vocab_file=None,442 tokenizer_file=None,443 unk_token="<unk>",444 bos_token="<s>",445 eos_token="</s>",446 pad_token=None,447 add_bos_token=True,448 add_eos_token=False,449 add_prefix_space=False,450 use_default_system_prompt=False,451 chat_template=None,452 **kwargs,453 ):454 self.add_bos_token = add_bos_token455 self.add_eos_token = add_eos_token456 self.add_prefix_space = add_prefix_space457 self.use_default_system_prompt = use_default_system_prompt458 self.vocab_file = vocab_file459 460 if chat_template is None:461 chat_template = DEFAULT_CHAT_TEMPLATE462 463 super().__init__(464 vocab_file=vocab_file,465 tokenizer_file=tokenizer_file,466 unk_token=unk_token,467 bos_token=bos_token,468 eos_token=eos_token,469 pad_token=pad_token,470 add_bos_token=add_bos_token,471 add_eos_token=add_eos_token,472 add_prefix_space=add_prefix_space,473 use_default_system_prompt=use_default_system_prompt,474 chat_template=chat_template,475 **kwargs,476 )477 478 @property479 def can_save_slow_tokenizer(self) -> bool:480 vocab_file = getattr(self, "vocab_file", None)481 return bool(vocab_file) and os.path.isfile(vocab_file)482 483 def save_vocabulary(484 self, save_directory: str, filename_prefix: Optional[str] = None485 ) -> Tuple[str]:486 if not self.can_save_slow_tokenizer:487 return ()488 if not os.path.isdir(save_directory):489 logger.error(f"Vocabulary path ({save_directory}) should be a directory")490 return ()491 out_vocab_file = os.path.join(492 save_directory,493 (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"],494 )495 if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):496 copyfile(self.vocab_file, out_vocab_file)497 return (out_vocab_file,)498 499 @property500 def default_chat_template(self) -> str:501 """Returns the default chat template."""502 return DEFAULT_CHAT_TEMPLATE503 504 def build_inputs_with_special_tokens(505 self, 506 token_ids_0: List[int], 507 token_ids_1: Optional[List[int]] = None508 ) -> List[int]:509 """Build model inputs with special tokens."""510 bos_token_id = [self.bos_token_id] if self.add_bos_token else []511 eos_token_id = [self.eos_token_id] if self.add_eos_token else []512 513 output = bos_token_id + token_ids_0 + eos_token_id514 515 if token_ids_1 is not None:516 output = output + bos_token_id + token_ids_1 + eos_token_id517 518 return output519 520 def get_special_tokens_mask(521 self, 522 token_ids_0: List[int], 523 token_ids_1: Optional[List[int]] = None, 524 already_has_special_tokens: bool = False525 ) -> List[int]:526 """Retrieve special tokens mask."""527 if already_has_special_tokens:528 return super().get_special_tokens_mask(529 token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True530 )531 532 bos_token_id = [1] if self.add_bos_token else []533 eos_token_id = [1] if self.add_eos_token else []534 535 if token_ids_1 is None:536 return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id537 return (538 bos_token_id539 + ([0] * len(token_ids_0))540 + eos_token_id541 + bos_token_id542 + ([0] * len(token_ids_1))543 + eos_token_id544 )545 546 def create_token_type_ids_from_sequences(547 self, 548 token_ids_0: List[int], 549 token_ids_1: Optional[List[int]] = None550 ) -> List[int]:551 """Create token type IDs from sequences."""552 bos_token_id = [self.bos_token_id] if self.add_bos_token else []553 eos_token_id = [self.eos_token_id] if self.add_eos_token else []554 555 output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)556 557 if token_ids_1 is not None:558 output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)559 560 return output561 562except ImportError:563 # tokenizers library not available, Fast tokenizer not supported564 IQuestCoderTokenizerFast = None565 logger.info(566 "The `tokenizers` library is not installed. "567 "IQuestCoderTokenizerFast will not be available. "568 "Install it with `pip install tokenizers`."569 )570 571