DoruC/Grounded-Segment-Anything

1# coding=utf-82# Copyright 2018 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Tokenization classes for ConvBERT."""16import collections17import os18import unicodedata19from typing import List, Optional, Tuple20 21from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace22from ...utils import logging23 24 25logger = logging.get_logger(__name__)26 27VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}28 29PRETRAINED_VOCAB_FILES_MAP = {30    "vocab_file": {31        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",32        "YituTech/conv-bert-medium-small": (33            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"34        ),35        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",36    }37}38 39PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {40    "YituTech/conv-bert-base": 512,41    "YituTech/conv-bert-medium-small": 512,42    "YituTech/conv-bert-small": 512,43}44 45 46PRETRAINED_INIT_CONFIGURATION = {47    "YituTech/conv-bert-base": {"do_lower_case": True},48    "YituTech/conv-bert-medium-small": {"do_lower_case": True},49    "YituTech/conv-bert-small": {"do_lower_case": True},50}51 52 53# Copied from transformers.models.bert.tokenization_bert.load_vocab54def load_vocab(vocab_file):55    """Loads a vocabulary file into a dictionary."""56    vocab = collections.OrderedDict()57    with open(vocab_file, "r", encoding="utf-8") as reader:58        tokens = reader.readlines()59    for index, token in enumerate(tokens):60        token = token.rstrip("\n")61        vocab[token] = index62    return vocab63 64 65# Copied from transformers.models.bert.tokenization_bert.whitespace_tokenize66def whitespace_tokenize(text):67    """Runs basic whitespace cleaning and splitting on a piece of text."""68    text = text.strip()69    if not text:70        return []71    tokens = text.split()72    return tokens73 74 75# Copied from transformers.models.bert.tokenization_bert.BertTokenizer with bert-base-cased->YituTech/conv-bert-base, ConvBertTokenizer->BertTokenizer, BERT->ConvBERT76class ConvBertTokenizer(PreTrainedTokenizer):77    r"""78    Construct a ConvBERT tokenizer. Based on WordPiece.79 80    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to81    this superclass for more information regarding those methods.82 83    Args:84        vocab_file (`str`):85            File containing the vocabulary.86        do_lower_case (`bool`, *optional*, defaults to `True`):87            Whether or not to lowercase the input when tokenizing.88        do_basic_tokenize (`bool`, *optional*, defaults to `True`):89            Whether or not to do basic tokenization before WordPiece.90        never_split (`Iterable`, *optional*):91            Collection of tokens which will never be split during tokenization. Only has an effect when92            `do_basic_tokenize=True`93        unk_token (`str`, *optional*, defaults to `"[UNK]"`):94            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this95            token instead.96        sep_token (`str`, *optional*, defaults to `"[SEP]"`):97            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for98            sequence classification or for a text and a question for question answering. It is also used as the last99            token of a sequence built with special tokens.100        pad_token (`str`, *optional*, defaults to `"[PAD]"`):101            The token used for padding, for example when batching sequences of different lengths.102        cls_token (`str`, *optional*, defaults to `"[CLS]"`):103            The classifier token which is used when doing sequence classification (classification of the whole sequence104            instead of per-token classification). It is the first token of the sequence when built with special tokens.105        mask_token (`str`, *optional*, defaults to `"[MASK]"`):106            The token used for masking values. This is the token used when training this model with masked language107            modeling. This is the token which the model will try to predict.108        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):109            Whether or not to tokenize Chinese characters.110 111            This should likely be deactivated for Japanese (see this112            [issue](https://github.com/huggingface/transformers/issues/328)).113        strip_accents (`bool`, *optional*):114            Whether or not to strip all accents. If this option is not specified, then it will be determined by the115            value for `lowercase` (as in the original ConvBERT).116    """117 118    vocab_files_names = VOCAB_FILES_NAMES119    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP120    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION121    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES122 123    def __init__(124        self,125        vocab_file,126        do_lower_case=True,127        do_basic_tokenize=True,128        never_split=None,129        unk_token="[UNK]",130        sep_token="[SEP]",131        pad_token="[PAD]",132        cls_token="[CLS]",133        mask_token="[MASK]",134        tokenize_chinese_chars=True,135        strip_accents=None,136        **kwargs,137    ):138        if not os.path.isfile(vocab_file):139            raise ValueError(140                f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"141                " model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"142            )143        self.vocab = load_vocab(vocab_file)144        self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])145        self.do_basic_tokenize = do_basic_tokenize146        if do_basic_tokenize:147            self.basic_tokenizer = BasicTokenizer(148                do_lower_case=do_lower_case,149                never_split=never_split,150                tokenize_chinese_chars=tokenize_chinese_chars,151                strip_accents=strip_accents,152            )153 154        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=str(unk_token))155 156        super().__init__(157            do_lower_case=do_lower_case,158            do_basic_tokenize=do_basic_tokenize,159            never_split=never_split,160            unk_token=unk_token,161            sep_token=sep_token,162            pad_token=pad_token,163            cls_token=cls_token,164            mask_token=mask_token,165            tokenize_chinese_chars=tokenize_chinese_chars,166            strip_accents=strip_accents,167            **kwargs,168        )169 170    @property171    def do_lower_case(self):172        return self.basic_tokenizer.do_lower_case173 174    @property175    def vocab_size(self):176        return len(self.vocab)177 178    def get_vocab(self):179        return dict(self.vocab, **self.added_tokens_encoder)180 181    def _tokenize(self, text, split_special_tokens=False):182        split_tokens = []183        if self.do_basic_tokenize:184            for token in self.basic_tokenizer.tokenize(185                text, never_split=self.all_special_tokens if not split_special_tokens else None186            ):187                # If the token is part of the never_split set188                if token in self.basic_tokenizer.never_split:189                    split_tokens.append(token)190                else:191                    split_tokens += self.wordpiece_tokenizer.tokenize(token)192        else:193            split_tokens = self.wordpiece_tokenizer.tokenize(text)194        return split_tokens195 196    def _convert_token_to_id(self, token):197        """Converts a token (str) in an id using the vocab."""198        return self.vocab.get(token, self.vocab.get(self.unk_token))199 200    def _convert_id_to_token(self, index):201        """Converts an index (integer) in a token (str) using the vocab."""202        return self.ids_to_tokens.get(index, self.unk_token)203 204    def convert_tokens_to_string(self, tokens):205        """Converts a sequence of tokens (string) in a single string."""206        out_string = " ".join(tokens).replace(" ##", "").strip()207        return out_string208 209    def build_inputs_with_special_tokens(210        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None211    ) -> List[int]:212        """213        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and214        adding special tokens. A ConvBERT sequence has the following format:215 216        - single sequence: `[CLS] X [SEP]`217        - pair of sequences: `[CLS] A [SEP] B [SEP]`218 219        Args:220            token_ids_0 (`List[int]`):221                List of IDs to which the special tokens will be added.222            token_ids_1 (`List[int]`, *optional*):223                Optional second list of IDs for sequence pairs.224 225        Returns:226            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.227        """228        if token_ids_1 is None:229            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]230        cls = [self.cls_token_id]231        sep = [self.sep_token_id]232        return cls + token_ids_0 + sep + token_ids_1 + sep233 234    def get_special_tokens_mask(235        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False236    ) -> List[int]:237        """238        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding239        special tokens using the tokenizer `prepare_for_model` method.240 241        Args:242            token_ids_0 (`List[int]`):243                List of IDs.244            token_ids_1 (`List[int]`, *optional*):245                Optional second list of IDs for sequence pairs.246            already_has_special_tokens (`bool`, *optional*, defaults to `False`):247                Whether or not the token list is already formatted with special tokens for the model.248 249        Returns:250            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.251        """252 253        if already_has_special_tokens:254            return super().get_special_tokens_mask(255                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True256            )257 258        if token_ids_1 is not None:259            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]260        return [1] + ([0] * len(token_ids_0)) + [1]261 262    def create_token_type_ids_from_sequences(263        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None264    ) -> List[int]:265        """266        Create a mask from the two sequences passed to be used in a sequence-pair classification task. A ConvBERT267        sequence pair mask has the following format:268 269        ```270        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1271        | first sequence    | second sequence |272        ```273 274        If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).275 276        Args:277            token_ids_0 (`List[int]`):278                List of IDs.279            token_ids_1 (`List[int]`, *optional*):280                Optional second list of IDs for sequence pairs.281 282        Returns:283            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).284        """285        sep = [self.sep_token_id]286        cls = [self.cls_token_id]287        if token_ids_1 is None:288            return len(cls + token_ids_0 + sep) * [0]289        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]290 291    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:292        index = 0293        if os.path.isdir(save_directory):294            vocab_file = os.path.join(295                save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]296            )297        else:298            vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory299        with open(vocab_file, "w", encoding="utf-8") as writer:300            for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):301                if index != token_index:302                    logger.warning(303                        f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."304                        " Please check that the vocabulary is not corrupted!"305                    )306                    index = token_index307                writer.write(token + "\n")308                index += 1309        return (vocab_file,)310 311 312# Copied from transformers.models.bert.tokenization_bert.BasicTokenizer313class BasicTokenizer(object):314    """315    Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).316 317    Args:318        do_lower_case (`bool`, *optional*, defaults to `True`):319            Whether or not to lowercase the input when tokenizing.320        never_split (`Iterable`, *optional*):321            Collection of tokens which will never be split during tokenization. Only has an effect when322            `do_basic_tokenize=True`323        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):324            Whether or not to tokenize Chinese characters.325 326            This should likely be deactivated for Japanese (see this327            [issue](https://github.com/huggingface/transformers/issues/328)).328        strip_accents (`bool`, *optional*):329            Whether or not to strip all accents. If this option is not specified, then it will be determined by the330            value for `lowercase` (as in the original BERT).331        do_split_on_punc (`bool`, *optional*, defaults to `True`):332            In some instances we want to skip the basic punctuation splitting so that later tokenization can capture333            the full context of the words, such as contractions.334    """335 336    def __init__(337        self,338        do_lower_case=True,339        never_split=None,340        tokenize_chinese_chars=True,341        strip_accents=None,342        do_split_on_punc=True,343    ):344        if never_split is None:345            never_split = []346        self.do_lower_case = do_lower_case347        self.never_split = set(never_split)348        self.tokenize_chinese_chars = tokenize_chinese_chars349        self.strip_accents = strip_accents350        self.do_split_on_punc = do_split_on_punc351 352    def tokenize(self, text, never_split=None):353        """354        Basic Tokenization of a piece of text. For sub-word tokenization, see WordPieceTokenizer.355 356        Args:357            never_split (`List[str]`, *optional*)358                Kept for backward compatibility purposes. Now implemented directly at the base class level (see359                [`PreTrainedTokenizer.tokenize`]) List of token not to split.360        """361        # union() returns a new set by concatenating the two sets.362        never_split = self.never_split.union(set(never_split)) if never_split else self.never_split363        text = self._clean_text(text)364 365        # This was added on November 1st, 2018 for the multilingual and Chinese366        # models. This is also applied to the English models now, but it doesn't367        # matter since the English models were not trained on any Chinese data368        # and generally don't have any Chinese data in them (there are Chinese369        # characters in the vocabulary because Wikipedia does have some Chinese370        # words in the English Wikipedia.).371        if self.tokenize_chinese_chars:372            text = self._tokenize_chinese_chars(text)373        # prevents treating the same character with different unicode codepoints as different characters374        unicode_normalized_text = unicodedata.normalize("NFC", text)375        orig_tokens = whitespace_tokenize(unicode_normalized_text)376        split_tokens = []377        for token in orig_tokens:378            if token not in never_split:379                if self.do_lower_case:380                    token = token.lower()381                    if self.strip_accents is not False:382                        token = self._run_strip_accents(token)383                elif self.strip_accents:384                    token = self._run_strip_accents(token)385            split_tokens.extend(self._run_split_on_punc(token, never_split))386 387        output_tokens = whitespace_tokenize(" ".join(split_tokens))388        return output_tokens389 390    def _run_strip_accents(self, text):391        """Strips accents from a piece of text."""392        text = unicodedata.normalize("NFD", text)393        output = []394        for char in text:395            cat = unicodedata.category(char)396            if cat == "Mn":397                continue398            output.append(char)399        return "".join(output)400 401    def _run_split_on_punc(self, text, never_split=None):402        """Splits punctuation on a piece of text."""403        if not self.do_split_on_punc or (never_split is not None and text in never_split):404            return [text]405        chars = list(text)406        i = 0407        start_new_word = True408        output = []409        while i < len(chars):410            char = chars[i]411            if _is_punctuation(char):412                output.append([char])413                start_new_word = True414            else:415                if start_new_word:416                    output.append([])417                start_new_word = False418                output[-1].append(char)419            i += 1420 421        return ["".join(x) for x in output]422 423    def _tokenize_chinese_chars(self, text):424        """Adds whitespace around any CJK character."""425        output = []426        for char in text:427            cp = ord(char)428            if self._is_chinese_char(cp):429                output.append(" ")430                output.append(char)431                output.append(" ")432            else:433                output.append(char)434        return "".join(output)435 436    def _is_chinese_char(self, cp):437        """Checks whether CP is the codepoint of a CJK character."""438        # This defines a "chinese character" as anything in the CJK Unicode block:439        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)440        #441        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,442        # despite its name. The modern Korean Hangul alphabet is a different block,443        # as is Japanese Hiragana and Katakana. Those alphabets are used to write444        # space-separated words, so they are not treated specially and handled445        # like the all of the other languages.446        if (447            (cp >= 0x4E00 and cp <= 0x9FFF)448            or (cp >= 0x3400 and cp <= 0x4DBF)  #449            or (cp >= 0x20000 and cp <= 0x2A6DF)  #450            or (cp >= 0x2A700 and cp <= 0x2B73F)  #451            or (cp >= 0x2B740 and cp <= 0x2B81F)  #452            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #453            or (cp >= 0xF900 and cp <= 0xFAFF)454            or (cp >= 0x2F800 and cp <= 0x2FA1F)  #455        ):  #456            return True457 458        return False459 460    def _clean_text(self, text):461        """Performs invalid character removal and whitespace cleanup on text."""462        output = []463        for char in text:464            cp = ord(char)465            if cp == 0 or cp == 0xFFFD or _is_control(char):466                continue467            if _is_whitespace(char):468                output.append(" ")469            else:470                output.append(char)471        return "".join(output)472 473 474# Copied from transformers.models.bert.tokenization_bert.WordpieceTokenizer475class WordpieceTokenizer(object):476    """Runs WordPiece tokenization."""477 478    def __init__(self, vocab, unk_token, max_input_chars_per_word=100):479        self.vocab = vocab480        self.unk_token = unk_token481        self.max_input_chars_per_word = max_input_chars_per_word482 483    def tokenize(self, text):484        """485        Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform486        tokenization using the given vocabulary.487 488        For example, `input = "unaffable"` wil return as output `["un", "##aff", "##able"]`.489 490        Args:491            text: A single token or whitespace separated tokens. This should have492                already been passed through *BasicTokenizer*.493 494        Returns:495            A list of wordpiece tokens.496        """497 498        output_tokens = []499        for token in whitespace_tokenize(text):500            chars = list(token)501            if len(chars) > self.max_input_chars_per_word:502                output_tokens.append(self.unk_token)503                continue504 505            is_bad = False506            start = 0507            sub_tokens = []508            while start < len(chars):509                end = len(chars)510                cur_substr = None511                while start < end:512                    substr = "".join(chars[start:end])513                    if start > 0:514                        substr = "##" + substr515                    if substr in self.vocab:516                        cur_substr = substr517                        break518                    end -= 1519                if cur_substr is None:520                    is_bad = True521                    break522                sub_tokens.append(cur_substr)523                start = end524 525            if is_bad:526                output_tokens.append(self.unk_token)527            else:528                output_tokens.extend(sub_tokens)529        return output_tokens530