DoruC/Grounded-Segment-Anything
0
1# coding=utf-82# Copyright 2023 MetaAI and the HuggingFace Inc. team. All rights reserved.3#4#5# Licensed under the Apache License, Version 2.0 (the "License");6# you may not use this file except in compliance with the License.7# You may obtain a copy of the License at8#9# http://www.apache.org/licenses/LICENSE-2.010#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.14# See the License for the specific language governing permissions and15# limitations under the License.16 17"""Tokenization classes for Code LLaMA."""18import os19from shutil import copyfile20from typing import Any, Dict, List, Optional, Tuple21 22import sentencepiece as spm23 24from ...convert_slow_tokenizer import import_protobuf25from ...tokenization_utils import AddedToken, PreTrainedTokenizer26from ...utils import logging, requires_backends27 28 29logger = logging.get_logger(__name__)30 31VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}32 33PRETRAINED_VOCAB_FILES_MAP = {34 "vocab_file": {35 "hf-internal-testing/llama-code-tokenizer": "https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer.model",36 },37 "tokenizer_file": {38 "hf-internal-testing/llama-code-tokenizer": "https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer_config.json",39 },40}41PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {42 "hf-internal-testing/llama-code-tokenizer": 2048,43}44SPIECE_UNDERLINE = "▁"45 46B_INST, E_INST = "[INST]", "[/INST]"47B_SYS, E_SYS = "<<SYS>>\n", "\n<</SYS>>\n\n"48 49# fmt: off50DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your \51answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\52 that your responses are socially unbiased and positive in nature.53 54If a question does not make any sense, or is not factually coherent, explain why instead of answering something not \55correct. If you don't know the answer to a question, please don't share false information."""56# fmt: on57 58 59class CodeLlamaTokenizer(PreTrainedTokenizer):60 """61 Construct a CodeLlama tokenizer. Based on byte-level Byte-Pair-Encoding. The default padding token is unset as62 there is no padding token in the original model.63 64 The default configuration match that of65 [codellama/CodeLlama-7b-Instruct-hf](https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf/blob/main/tokenizer_config.json)66 which supports prompt infilling.67 68 Args:69 vocab_file (`str`):70 Path to the vocabulary file.71 eos_token (`str`, *optional*, defaults to `"</s>"`):72 The end of sequence token.73 74 <Tip>75 76 When building a sequence using special tokens, this is not the token that is used for the end of sequence.77 The token used is the `sep_token`.78 79 </Tip>80 81 unk_token (`str`, *optional*, defaults to `"<unk>"`):82 The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this83 token instead.84 prefix_token (`str`, *optional*, defaults to `"▁<PRE>"`):85 Prefix token used for infilling.86 suffix_token (`str`, *optional*, defaults to `"▁<SUF>"`):87 Suffix token used for infilling.88 middle_token (`str`, *optional*, defaults to `"▁<MID>"`):89 Middle token used for infilling.90 eot_token (`str`, *optional*, defaults to `"▁<EOT>"`):91 End of text token used for infilling.92 fill_token (`str`, *optional*, defaults to `"<FILL_ME>"`):93 The token used to split the input between the prefix and suffix.94 suffix_first (`bool`, *optional*, default to `False`):95 Whether the input prompt and suffix should be formatted with the suffix first.96 additional_special_tokens (`List[str]`, *optional*):97 Additional special tokens used by the tokenizer.98 sp_model_kwargs (`dict`, *optional*):99 Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for100 SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,101 to set:102 103 - `enable_sampling`: Enable subword regularization.104 - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.105 106 - `nbest_size = {0,1}`: No sampling is performed.107 - `nbest_size > 1`: samples from the nbest_size results.108 - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)109 using forward-filtering-and-backward-sampling algorithm.110 111 - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for112 BPE-dropout.113 use_default_system_prompt (`bool`, *optional*, defaults to `False`):114 Whether or not the default system prompt for Llama should be used.115 """116 117 vocab_files_names = VOCAB_FILES_NAMES118 pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP119 max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES120 model_input_names = ["input_ids", "attention_mask"]121 122 def __init__(123 self,124 vocab_file,125 unk_token="<unk>",126 bos_token="<s>",127 eos_token="</s>",128 prefix_token="▁<PRE>",129 middle_token="▁<MID>",130 suffix_token="▁<SUF>",131 eot_token="▁<EOT>",132 fill_token="<FILL_ME>",133 suffix_first=False,134 sp_model_kwargs: Optional[Dict[str, Any]] = None,135 add_bos_token=True,136 add_eos_token=False,137 clean_up_tokenization_spaces=False,138 additional_special_tokens=None,139 use_default_system_prompt=False,140 **kwargs,141 ):142 requires_backends(self, "protobuf")143 self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs144 bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token145 eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token146 unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token147 148 self.use_default_system_prompt = use_default_system_prompt149 # mark tokens special to skip them150 additional_special_tokens = additional_special_tokens or []151 for token in [prefix_token, middle_token, suffix_token, eot_token]:152 additional_special_tokens += [token] if token is not None else []153 154 self.vocab_file = vocab_file155 self.add_bos_token = add_bos_token156 self.add_eos_token = add_eos_token157 self._prefix_token = prefix_token158 self._middle_token = middle_token159 self._suffix_token = suffix_token160 self._eot_token = eot_token161 self.fill_token = fill_token162 self.suffix_first = suffix_first163 self.sp_model = self.get_spm_processor()164 165 super().__init__(166 bos_token=bos_token,167 eos_token=eos_token,168 unk_token=unk_token,169 add_bos_token=add_bos_token,170 add_eos_token=add_eos_token,171 prefix_token=prefix_token,172 middle_token=middle_token,173 suffix_token=suffix_token,174 eot_token=eot_token,175 fill_token=fill_token,176 sp_model_kwargs=self.sp_model_kwargs,177 suffix_first=suffix_first,178 clean_up_tokenization_spaces=clean_up_tokenization_spaces,179 additional_special_tokens=additional_special_tokens,180 use_default_system_prompt=use_default_system_prompt,181 **kwargs,182 )183 184 @property185 def unk_token_length(self):186 return len(self.sp_model.encode(str(self.unk_token)))187 188 def get_spm_processor(self):189 tokenizer = spm.SentencePieceProcessor(**self.sp_model_kwargs)190 with open(self.vocab_file, "rb") as f:191 sp_model = f.read()192 model_pb2 = import_protobuf()193 model = model_pb2.ModelProto.FromString(sp_model)194 normalizer_spec = model_pb2.NormalizerSpec()195 normalizer_spec.add_dummy_prefix = False196 model.normalizer_spec.MergeFrom(normalizer_spec)197 sp_model = model.SerializeToString()198 tokenizer.LoadFromSerializedProto(sp_model)199 return tokenizer200 201 @property202 def prefix_token(self):203 return self._prefix_token204 205 @property206 def prefix_id(self):207 if self._prefix_token is None:208 return None209 return self.convert_tokens_to_ids(self.prefix_token)210 211 @property212 def middle_token(self):213 return self._middle_token214 215 @property216 def middle_id(self):217 if self._middle_token is None:218 return None219 return self.convert_tokens_to_ids(self.middle_token)220 221 @property222 def suffix_token(self):223 return self._suffix_token224 225 @property226 def suffix_id(self):227 if self._suffix_token is None:228 return None229 return self.convert_tokens_to_ids(self.suffix_token)230 231 @property232 def eot_token(self):233 return self._eot_token234 235 @property236 def eot_id(self):237 if self._eot_token is None:238 return None239 return self.convert_tokens_to_ids(self.eot_token)240 241 @property242 def vocab_size(self):243 """Returns vocab size"""244 return self.sp_model.get_piece_size()245 246 # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer.get_vocab247 def get_vocab(self):248 """Returns vocab as a dict"""249 vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}250 vocab.update(self.added_tokens_encoder)251 return vocab252 253 def tokenize(self, prefix, suffix=None, suffix_first=False, **kwargs) -> List[int]:254 # add a prefix space to `prefix`255 if self.fill_token is not None and self.fill_token in prefix and suffix is None:256 prefix, suffix = prefix.split(self.fill_token)257 258 if len(prefix) > 0:259 prefix = SPIECE_UNDERLINE + prefix.replace(SPIECE_UNDERLINE, " ")260 261 if suffix is None or len(suffix) < 1:262 tokens = super().tokenize(prefix, **kwargs)263 if len(tokens) > 1 and tokens[0] == SPIECE_UNDERLINE and tokens[1] in self.all_special_tokens:264 tokens = tokens[1:]265 return tokens266 267 prefix_tokens = self._tokenize(prefix) # prefix has an extra `SPIECE_UNDERLINE`268 269 if None in (self.prefix_id, self.middle_id, self.suffix_id):270 raise ValueError(271 "The input either includes a `prefix` and a `suffix` used for the infilling task,"272 f" or can be split on the {self.fill_token} token, creating a suffix and prefix,"273 " but the model does not support `infilling`."274 )275 suffix_tokens = self._tokenize(suffix) # make sure CodeLlama sp model does not mess up276 277 suffix_first = suffix_first if suffix_first is not None else self.suffix_first278 if suffix_first:279 # format as " <PRE> <SUF>{suf} <MID> {pre}"280 return [self.prefix_token, self.suffix_token] + suffix_tokens + [self.middle_token] + prefix_tokens281 else:282 # format as " <PRE> {pre} <SUF>{suf} <MID>"283 return [self.prefix_token] + prefix_tokens + [self.suffix_token] + suffix_tokens + [self.middle_token]284 285 def _tokenize(self, text, **kwargs):286 """287 Returns a tokenized string.288 289 We de-activated the `add_dummy_prefix` option, thus the sentencepiece internals will always strip any290 SPIECE_UNDERLINE. For example: `self.sp_model.encode(f"{SPIECE_UNDERLINE}Hey", out_type = str)` will give291 `['H', 'e', 'y']` instead of `['▁He', 'y']`. Thus we always encode `f"{unk_token}text"` and strip the292 `unk_token`. Here is an example with `unk_token = "<unk>"` and `unk_token_length = 4`.293 `self.tokenizer.sp_model.encode("<unk> Hey", out_type = str)[4:]`.294 """295 tokens = self.sp_model.encode(text, out_type=str)296 if not text.startswith((SPIECE_UNDERLINE, " ")):297 return tokens298 # 1. Encode string + prefix ex: "<unk> Hey"299 tokens = self.sp_model.encode(self.unk_token + text, out_type=str)300 # 2. Remove self.unk_token from ['<','unk','>', '▁Hey']301 return tokens[self.unk_token_length :] if len(tokens) >= self.unk_token_length else tokens302 303 # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer._convert_token_to_id304 def _convert_token_to_id(self, token):305 """Converts a token (str) in an id using the vocab."""306 return self.sp_model.piece_to_id(token)307 308 # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer._convert_id_to_token309 def _convert_id_to_token(self, index):310 """Converts an index (integer) in a token (str) using the vocab."""311 token = self.sp_model.IdToPiece(index)312 return token313 314 def convert_tokens_to_string(self, tokens):315 """Converts a sequence of tokens (string) in a single string."""316 # since we manually add the prefix space, we have to remove it when decoding317 if tokens[0].startswith(SPIECE_UNDERLINE):318 tokens[0] = tokens[0][1:]319 320 current_sub_tokens = []321 out_string = ""322 for _, token in enumerate(tokens):323 # make sure that special tokens are not decoded using sentencepiece model324 if token in self.all_special_tokens:325 out_string += self.sp_model.decode(current_sub_tokens) + token326 current_sub_tokens = []327 else:328 current_sub_tokens.append(token)329 out_string += self.sp_model.decode(current_sub_tokens)330 return out_string331 332 # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer.save_vocabulary333 def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:334 """335 Save the vocabulary and special tokens file to a directory.336 337 Args:338 save_directory (`str`):339 The directory in which to save the vocabulary.340 341 Returns:342 `Tuple(str)`: Paths to the files saved.343 """344 if not os.path.isdir(save_directory):345 logger.error(f"Vocabulary path ({save_directory}) should be a directory")346 return347 out_vocab_file = os.path.join(348 save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]349 )350 351 if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):352 copyfile(self.vocab_file, out_vocab_file)353 elif not os.path.isfile(self.vocab_file):354 with open(out_vocab_file, "wb") as fi:355 content_spiece_model = self.sp_model.serialized_model_proto()356 fi.write(content_spiece_model)357 358 return (out_vocab_file,)359 360 # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer.build_inputs_with_special_tokens361 def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):362 bos_token_id = [self.bos_token_id] if self.add_bos_token else []363 eos_token_id = [self.eos_token_id] if self.add_eos_token else []364 365 output = bos_token_id + token_ids_0 + eos_token_id366 367 if token_ids_1 is not None:368 output = output + bos_token_id + token_ids_1 + eos_token_id369 370 return output371 372 # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer.get_special_tokens_mask373 def get_special_tokens_mask(374 self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False375 ) -> List[int]:376 """377 Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding378 special tokens using the tokenizer `prepare_for_model` method.379 380 Args:381 token_ids_0 (`List[int]`):382 List of IDs.383 token_ids_1 (`List[int]`, *optional*):384 Optional second list of IDs for sequence pairs.385 already_has_special_tokens (`bool`, *optional*, defaults to `False`):386 Whether or not the token list is already formatted with special tokens for the model.387 388 Returns:389 `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.390 """391 if already_has_special_tokens:392 return super().get_special_tokens_mask(393 token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True394 )395 396 bos_token_id = [1] if self.add_bos_token else []397 eos_token_id = [1] if self.add_eos_token else []398 399 if token_ids_1 is None:400 return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id401 return (402 bos_token_id403 + ([0] * len(token_ids_0))404 + eos_token_id405 + bos_token_id406 + ([0] * len(token_ids_1))407 + eos_token_id408 )409 410 # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer.create_token_type_ids_from_sequences411 def create_token_type_ids_from_sequences(412 self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None413 ) -> List[int]:414 """415 Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT416 sequence pair mask has the following format:417 418 ```419 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1420 | first sequence | second sequence |421 ```422 423 if token_ids_1 is None, only returns the first portion of the mask (0s).424 425 Args:426 token_ids_0 (`List[int]`):427 List of ids.428 token_ids_1 (`List[int]`, *optional*):429 Optional second list of IDs for sequence pairs.430 431 Returns:432 `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).433 """434 bos_token_id = [self.bos_token_id] if self.add_bos_token else []435 eos_token_id = [self.eos_token_id] if self.add_eos_token else []436 437 output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)438 439 if token_ids_1 is not None:440 output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)441 442 return output443 444 @property445 # Copied from transformers.models.llama.tokenization_llama.LlamaTokenizer.default_chat_template446 def default_chat_template(self):447 """448 LLaMA uses [INST] and [/INST] to indicate user messages, and <<SYS>> and <</SYS>> to indicate system messages.449 Assistant messages do not have special tokens, because LLaMA chat models are generally trained with strict450 user/assistant/user/assistant message ordering, and so assistant messages can be identified from the ordering451 rather than needing special tokens. The system message is partly 'embedded' in the first user message, which452 results in an unusual token ordering when it is present. This template should definitely be changed if you wish453 to fine-tune a model with more flexible role ordering!454 455 The output should look something like:456 457 <bos>[INST] B_SYS SystemPrompt E_SYS Prompt [/INST] Answer <eos> <bos>[INST] Prompt [/INST] Answer <eos>458 <bos>[INST] Prompt [/INST]459 """460 461 template = (462 "{% if messages[0]['role'] == 'system' %}"463 "{% set loop_messages = messages[1:] %}" # Extract system message if it's present464 "{% set system_message = messages[0]['content'] %}"465 "{% elif USE_DEFAULT_PROMPT == true and not '<<SYS>>' in messages[0]['content'] %}"466 "{% set loop_messages = messages %}" # Or use the default system message if the flag is set467 "{% set system_message = 'DEFAULT_SYSTEM_MESSAGE' %}"468 "{% else %}"469 "{% set loop_messages = messages %}"470 "{% set system_message = false %}"471 "{% endif %}"472 "{% for message in loop_messages %}" # Loop over all non-system messages473 "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}"474 "{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}"475 "{% endif %}"476 "{% if loop.index0 == 0 and system_message != false %}" # Embed system message in first message477 "{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}"478 "{% else %}"479 "{% set content = message['content'] %}"480 "{% endif %}"481 "{% if message['role'] == 'user' %}" # After all of that, handle messages/roles in a fairly normal way482 "{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}"483 "{% elif message['role'] == 'system' %}"484 "{{ '<<SYS>>\\n' + content.strip() + '\\n<</SYS>>\\n\\n' }}"485 "{% elif message['role'] == 'assistant' %}"486 "{{ ' ' + content.strip() + ' ' + eos_token }}"487 "{% endif %}"488 "{% endfor %}"489 )490 template = template.replace("USE_DEFAULT_PROMPT", "true" if self.use_default_system_prompt else "false")491 default_message = DEFAULT_SYSTEM_PROMPT.replace("\n", "\\n").replace("'", "\\'")492 template = template.replace("DEFAULT_SYSTEM_MESSAGE", default_message)493 494 return template495 496 def __getstate__(self):497 state = self.__dict__.copy()498 state["sp_model"] = None499 state["sp_model_proto"] = self.sp_model.serialized_model_proto()500 return state501 502 def __setstate__(self, d):503 self.__dict__ = d504 self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)505 self.sp_model.LoadFromSerializedProto(self.sp_model_proto)506