DoruC/Grounded-Segment-Anything

1# coding=utf-82# Copyright 2021 Tel AViv University, AllenAI and The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Fast Tokenization classes for Splinter."""16 17import json18from typing import List, Optional, Tuple19 20from tokenizers import normalizers21 22from ...tokenization_utils_fast import PreTrainedTokenizerFast23from ...utils import logging24from .tokenization_splinter import SplinterTokenizer25 26 27logger = logging.get_logger(__name__)28 29VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}30 31PRETRAINED_VOCAB_FILES_MAP = {32    "vocab_file": {33        "tau/splinter-base": "https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt",34        "tau/splinter-base-qass": "https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt",35        "tau/splinter-large": "https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt",36        "tau/splinter-large-qass": "https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt",37    }38}39 40PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {41    "tau/splinter-base": 512,42    "tau/splinter-base-qass": 512,43    "tau/splinter-large": 512,44    "tau/splinter-large-qass": 512,45}46 47PRETRAINED_INIT_CONFIGURATION = {48    "tau/splinter-base": {"do_lower_case": False},49    "tau/splinter-base-qass": {"do_lower_case": False},50    "tau/splinter-large": {"do_lower_case": False},51    "tau/splinter-large-qass": {"do_lower_case": False},52}53 54 55class SplinterTokenizerFast(PreTrainedTokenizerFast):56    r"""57    Construct a "fast" Splinter tokenizer (backed by HuggingFace's *tokenizers* library). Based on WordPiece.58 59    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should60    refer to this superclass for more information regarding those methods.61 62    Args:63        vocab_file (`str`):64            File containing the vocabulary.65        do_lower_case (`bool`, *optional*, defaults to `True`):66            Whether or not to lowercase the input when tokenizing.67        unk_token (`str`, *optional*, defaults to `"[UNK]"`):68            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this69            token instead.70        sep_token (`str`, *optional*, defaults to `"[SEP]"`):71            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for72            sequence classification or for a text and a question for question answering. It is also used as the last73            token of a sequence built with special tokens.74        pad_token (`str`, *optional*, defaults to `"[PAD]"`):75            The token used for padding, for example when batching sequences of different lengths.76        cls_token (`str`, *optional*, defaults to `"[CLS]"`):77            The classifier token which is used when doing sequence classification (classification of the whole sequence78            instead of per-token classification). It is the first token of the sequence when built with special tokens.79        mask_token (`str`, *optional*, defaults to `"[MASK]"`):80            The token used for masking values. This is the token used when training this model with masked language81            modeling. This is the token which the model will try to predict.82        question_token (`str`, *optional*, defaults to `"[QUESTION]"`):83            The token used for constructing question representations.84        clean_text (`bool`, *optional*, defaults to `True`):85            Whether or not to clean the text before tokenization by removing any control characters and replacing all86            whitespaces by the classic one.87        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):88            Whether or not to tokenize Chinese characters. This should likely be deactivated for Japanese (see [this89            issue](https://github.com/huggingface/transformers/issues/328)).90        strip_accents (`bool`, *optional*):91            Whether or not to strip all accents. If this option is not specified, then it will be determined by the92            value for `lowercase` (as in the original BERT).93        wordpieces_prefix (`str`, *optional*, defaults to `"##"`):94            The prefix for subwords.95    """96 97    vocab_files_names = VOCAB_FILES_NAMES98    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP99    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION100    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES101    slow_tokenizer_class = SplinterTokenizer102 103    def __init__(104        self,105        vocab_file=None,106        tokenizer_file=None,107        do_lower_case=True,108        unk_token="[UNK]",109        sep_token="[SEP]",110        pad_token="[PAD]",111        cls_token="[CLS]",112        mask_token="[MASK]",113        question_token="[QUESTION]",114        tokenize_chinese_chars=True,115        strip_accents=None,116        **kwargs,117    ):118        super().__init__(119            vocab_file,120            tokenizer_file=tokenizer_file,121            do_lower_case=do_lower_case,122            unk_token=unk_token,123            sep_token=sep_token,124            pad_token=pad_token,125            cls_token=cls_token,126            mask_token=mask_token,127            tokenize_chinese_chars=tokenize_chinese_chars,128            strip_accents=strip_accents,129            additional_special_tokens=(question_token,),130            **kwargs,131        )132 133        pre_tok_state = json.loads(self.backend_tokenizer.normalizer.__getstate__())134        if (135            pre_tok_state.get("lowercase", do_lower_case) != do_lower_case136            or pre_tok_state.get("strip_accents", strip_accents) != strip_accents137        ):138            pre_tok_class = getattr(normalizers, pre_tok_state.pop("type"))139            pre_tok_state["lowercase"] = do_lower_case140            pre_tok_state["strip_accents"] = strip_accents141            self.backend_tokenizer.normalizer = pre_tok_class(**pre_tok_state)142 143        self.do_lower_case = do_lower_case144 145    @property146    def question_token_id(self):147        """148        `Optional[int]`: Id of the question token in the vocabulary, used to condition the answer on a question149        representation.150        """151        return self.convert_tokens_to_ids(self.question_token)152 153    def build_inputs_with_special_tokens(154        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None155    ) -> List[int]:156        """157        Build model inputs from a pair of sequence for question answering tasks by concatenating and adding special158        tokens. A Splinter sequence has the following format:159 160        - single sequence: `[CLS] X [SEP]`161        - pair of sequences for question answering: `[CLS] question_tokens [QUESTION] . [SEP] context_tokens [SEP]`162 163        Args:164            token_ids_0 (`List[int]`):165                The question token IDs if pad_on_right, else context tokens IDs166            token_ids_1 (`List[int]`, *optional*):167                The context token IDs if pad_on_right, else question token IDs168 169        Returns:170            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.171        """172        if token_ids_1 is None:173            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]174 175        cls = [self.cls_token_id]176        sep = [self.sep_token_id]177        question_suffix = [self.question_token_id] + [self.convert_tokens_to_ids(".")]178        if self.padding_side == "right":179            # Input is question-then-context180            return cls + token_ids_0 + question_suffix + sep + token_ids_1 + sep181        else:182            # Input is context-then-question183            return cls + token_ids_0 + sep + token_ids_1 + question_suffix + sep184 185    def create_token_type_ids_from_sequences(186        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None187    ) -> List[int]:188        """189        Create the token type IDs corresponding to the sequences passed. [What are token type190        IDs?](../glossary#token-type-ids)191 192        Should be overridden in a subclass if the model has a special way of building those.193 194        Args:195            token_ids_0 (`List[int]`): The first tokenized sequence.196            token_ids_1 (`List[int]`, *optional*): The second tokenized sequence.197 198        Returns:199            `List[int]`: The token type ids.200        """201        sep = [self.sep_token_id]202        cls = [self.cls_token_id]203        question_suffix = [self.question_token_id] + [self.convert_tokens_to_ids(".")]204        if token_ids_1 is None:205            return len(cls + token_ids_0 + sep) * [0]206 207        if self.padding_side == "right":208            # Input is question-then-context209            return len(cls + token_ids_0 + question_suffix + sep) * [0] + len(token_ids_1 + sep) * [1]210        else:211            # Input is context-then-question212            return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + question_suffix + sep) * [1]213 214    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:215        files = self._tokenizer.model.save(save_directory, name=filename_prefix)216        return tuple(files)217