DoruC/Grounded-Segment-Anything

1# coding=utf-82# Copyright 2021 Tel AViv University, AllenAI and The HuggingFace Inc. team. All rights reserved.3# All rights reserved.4#5# Licensed under the Apache License, Version 2.0 (the "License");6# you may not use this file except in compliance with the License.7# You may obtain a copy of the License at8#9#     http://www.apache.org/licenses/LICENSE-2.010#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.14# See the License for the specific language governing permissions and15# limitations under the License.16"""Tokenization classes for Splinter."""17 18import collections19import os20import unicodedata21from typing import List, Optional, Tuple22 23from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace24from ...utils import logging25 26 27logger = logging.get_logger(__name__)28 29VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}30 31PRETRAINED_VOCAB_FILES_MAP = {32    "vocab_file": {33        "tau/splinter-base": "https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt",34        "tau/splinter-base-qass": "https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt",35        "tau/splinter-large": "https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt",36        "tau/splinter-large-qass": "https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt",37    }38}39 40PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {41    "tau/splinter-base": 512,42    "tau/splinter-base-qass": 512,43    "tau/splinter-large": 512,44    "tau/splinter-large-qass": 512,45}46 47PRETRAINED_INIT_CONFIGURATION = {48    "tau/splinter-base": {"do_lower_case": False},49    "tau/splinter-base-qass": {"do_lower_case": False},50    "tau/splinter-large": {"do_lower_case": False},51    "tau/splinter-large-qass": {"do_lower_case": False},52}53 54 55def load_vocab(vocab_file):56    """Loads a vocabulary file into a dictionary."""57    vocab = collections.OrderedDict()58    with open(vocab_file, "r", encoding="utf-8") as reader:59        tokens = reader.readlines()60    for index, token in enumerate(tokens):61        token = token.rstrip("\n")62        vocab[token] = index63    return vocab64 65 66def whitespace_tokenize(text):67    """Runs basic whitespace cleaning and splitting on a piece of text."""68    text = text.strip()69    if not text:70        return []71    tokens = text.split()72    return tokens73 74 75class SplinterTokenizer(PreTrainedTokenizer):76    r"""77    Construct a Splinter tokenizer. Based on WordPiece.78 79    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to80    this superclass for more information regarding those methods.81 82    Args:83        vocab_file (`str`):84            File containing the vocabulary.85        do_lower_case (`bool`, *optional*, defaults to `True`):86            Whether or not to lowercase the input when tokenizing.87        do_basic_tokenize (`bool`, *optional*, defaults to `True`):88            Whether or not to do basic tokenization before WordPiece.89        never_split (`Iterable`, *optional*):90            Collection of tokens which will never be split during tokenization. Only has an effect when91            `do_basic_tokenize=True`92        unk_token (`str`, *optional*, defaults to `"[UNK]"`):93            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this94            token instead.95        sep_token (`str`, *optional*, defaults to `"[SEP]"`):96            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for97            sequence classification or for a text and a question for question answering. It is also used as the last98            token of a sequence built with special tokens.99        pad_token (`str`, *optional*, defaults to `"[PAD]"`):100            The token used for padding, for example when batching sequences of different lengths.101        cls_token (`str`, *optional*, defaults to `"[CLS]"`):102            The classifier token which is used when doing sequence classification (classification of the whole sequence103            instead of per-token classification). It is the first token of the sequence when built with special tokens.104        mask_token (`str`, *optional*, defaults to `"[MASK]"`):105            The token used for masking values. This is the token used when training this model with masked language106            modeling. This is the token which the model will try to predict.107        question_token (`str`, *optional*, defaults to `"[QUESTION]"`):108            The token used for constructing question representations.109        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):110            Whether or not to tokenize Chinese characters.111 112            This should likely be deactivated for Japanese (see this113            [issue](https://github.com/huggingface/transformers/issues/328)).114        strip_accents (`bool`, *optional*):115            Whether or not to strip all accents. If this option is not specified, then it will be determined by the116            value for `lowercase` (as in the original BERT).117    """118 119    vocab_files_names = VOCAB_FILES_NAMES120    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP121    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION122    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES123 124    def __init__(125        self,126        vocab_file,127        do_lower_case=True,128        do_basic_tokenize=True,129        never_split=None,130        unk_token="[UNK]",131        sep_token="[SEP]",132        pad_token="[PAD]",133        cls_token="[CLS]",134        mask_token="[MASK]",135        question_token="[QUESTION]",136        tokenize_chinese_chars=True,137        strip_accents=None,138        **kwargs,139    ):140        if not os.path.isfile(vocab_file):141            raise ValueError(142                f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"143                " model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"144            )145        self.vocab = load_vocab(vocab_file)146        self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])147        self.do_basic_tokenize = do_basic_tokenize148        if do_basic_tokenize:149            self.basic_tokenizer = BasicTokenizer(150                do_lower_case=do_lower_case,151                never_split=never_split,152                tokenize_chinese_chars=tokenize_chinese_chars,153                strip_accents=strip_accents,154            )155        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=str(unk_token))156        self.question_token = question_token157        super().__init__(158            do_lower_case=do_lower_case,159            do_basic_tokenize=do_basic_tokenize,160            never_split=never_split,161            unk_token=unk_token,162            sep_token=sep_token,163            pad_token=pad_token,164            cls_token=cls_token,165            mask_token=mask_token,166            tokenize_chinese_chars=tokenize_chinese_chars,167            strip_accents=strip_accents,168            **kwargs,169        )170 171    @property172    def question_token_id(self):173        """174        `Optional[int]`: Id of the question token in the vocabulary, used to condition the answer on a question175        representation.176        """177        return self.convert_tokens_to_ids(self.question_token)178 179    @property180    def do_lower_case(self):181        return self.basic_tokenizer.do_lower_case182 183    @property184    def vocab_size(self):185        return len(self.vocab)186 187    def get_vocab(self):188        return dict(self.vocab, **self.added_tokens_encoder)189 190    def _tokenize(self, text):191        split_tokens = []192        if self.do_basic_tokenize:193            for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens):194                # If the token is part of the never_split set195                if token in self.basic_tokenizer.never_split:196                    split_tokens.append(token)197                else:198                    split_tokens += self.wordpiece_tokenizer.tokenize(token)199        else:200            split_tokens = self.wordpiece_tokenizer.tokenize(text)201        return split_tokens202 203    def _convert_token_to_id(self, token):204        """Converts a token (str) in an id using the vocab."""205        return self.vocab.get(token, self.vocab.get(self.unk_token))206 207    def _convert_id_to_token(self, index):208        """Converts an index (integer) in a token (str) using the vocab."""209        return self.ids_to_tokens.get(index, self.unk_token)210 211    def convert_tokens_to_string(self, tokens):212        """Converts a sequence of tokens (string) in a single string."""213        out_string = " ".join(tokens).replace(" ##", "").strip()214        return out_string215 216    def build_inputs_with_special_tokens(217        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None218    ) -> List[int]:219        """220        Build model inputs from a pair of sequence for question answering tasks by concatenating and adding special221        tokens. A Splinter sequence has the following format:222 223        - single sequence: `[CLS] X [SEP]`224        - pair of sequences for question answering: `[CLS] question_tokens [QUESTION] . [SEP] context_tokens [SEP]`225 226        Args:227            token_ids_0 (`List[int]`):228                The question token IDs if pad_on_right, else context tokens IDs229            token_ids_1 (`List[int]`, *optional*):230                The context token IDs if pad_on_right, else question token IDs231 232        Returns:233            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.234        """235        if token_ids_1 is None:236            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]237 238        cls = [self.cls_token_id]239        sep = [self.sep_token_id]240        question_suffix = [self.question_token_id] + [self.convert_tokens_to_ids(".")]241        if self.padding_side == "right":242            # Input is question-then-context243            return cls + token_ids_0 + question_suffix + sep + token_ids_1 + sep244        else:245            # Input is context-then-question246            return cls + token_ids_0 + sep + token_ids_1 + question_suffix + sep247 248    def get_special_tokens_mask(249        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False250    ) -> List[int]:251        """252        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding253        special tokens using the tokenizer `prepare_for_model` method.254 255        Args:256            token_ids_0 (`List[int]`):257                List of IDs.258            token_ids_1 (`List[int]`, *optional*):259                Optional second list of IDs for sequence pairs.260            already_has_special_tokens (`bool`, *optional*, defaults to `False`):261                Whether or not the token list is already formatted with special tokens for the model.262 263        Returns:264            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.265        """266 267        if already_has_special_tokens:268            return super().get_special_tokens_mask(269                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True270            )271 272        if token_ids_1 is not None:273            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]274        return [1] + ([0] * len(token_ids_0)) + [1]275 276    def create_token_type_ids_from_sequences(277        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None278    ) -> List[int]:279        """280        Create the token type IDs corresponding to the sequences passed. [What are token type281        IDs?](../glossary#token-type-ids)282 283        Should be overridden in a subclass if the model has a special way of building those.284 285        Args:286            token_ids_0 (`List[int]`): The first tokenized sequence.287            token_ids_1 (`List[int]`, *optional*): The second tokenized sequence.288 289        Returns:290            `List[int]`: The token type ids.291        """292        sep = [self.sep_token_id]293        cls = [self.cls_token_id]294        question_suffix = [self.question_token_id] + [self.convert_tokens_to_ids(".")]295        if token_ids_1 is None:296            return len(cls + token_ids_0 + sep) * [0]297 298        if self.padding_side == "right":299            # Input is question-then-context300            return len(cls + token_ids_0 + question_suffix + sep) * [0] + len(token_ids_1 + sep) * [1]301        else:302            # Input is context-then-question303            return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + question_suffix + sep) * [1]304 305    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:306        index = 0307        if os.path.isdir(save_directory):308            vocab_file = os.path.join(309                save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]310            )311        else:312            vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory313        with open(vocab_file, "w", encoding="utf-8") as writer:314            for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):315                if index != token_index:316                    logger.warning(317                        f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."318                        " Please check that the vocabulary is not corrupted!"319                    )320                    index = token_index321                writer.write(token + "\n")322                index += 1323        return (vocab_file,)324 325 326class BasicTokenizer(object):327    """328    Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).329 330    Args:331        do_lower_case (`bool`, *optional*, defaults to `True`):332            Whether or not to lowercase the input when tokenizing.333        never_split (`Iterable`, *optional*):334            Collection of tokens which will never be split during tokenization. Only has an effect when335            `do_basic_tokenize=True`336        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):337            Whether or not to tokenize Chinese characters.338 339            This should likely be deactivated for Japanese (see this340            [issue](https://github.com/huggingface/transformers/issues/328)).341        strip_accents (`bool`, *optional*):342            Whether or not to strip all accents. If this option is not specified, then it will be determined by the343            value for `lowercase` (as in the original BERT).344    """345 346    def __init__(self, do_lower_case=True, never_split=None, tokenize_chinese_chars=True, strip_accents=None):347        if never_split is None:348            never_split = []349        self.do_lower_case = do_lower_case350        self.never_split = set(never_split)351        self.tokenize_chinese_chars = tokenize_chinese_chars352        self.strip_accents = strip_accents353 354    def tokenize(self, text, never_split=None):355        """356        Basic Tokenization of a piece of text. Split on "white spaces" only, for sub-word tokenization, see357        WordPieceTokenizer.358 359        Args:360            **never_split**: (*optional*) list of str361                Kept for backward compatibility purposes. Now implemented directly at the base class level (see362                [`PreTrainedTokenizer.tokenize`]) List of token not to split.363        """364        # union() returns a new set by concatenating the two sets.365        never_split = self.never_split.union(set(never_split)) if never_split else self.never_split366        text = self._clean_text(text)367 368        # This was added on November 1st, 2018 for the multilingual and Chinese369        # models. This is also applied to the English models now, but it doesn't370        # matter since the English models were not trained on any Chinese data371        # and generally don't have any Chinese data in them (there are Chinese372        # characters in the vocabulary because Wikipedia does have some Chinese373        # words in the English Wikipedia.).374        if self.tokenize_chinese_chars:375            text = self._tokenize_chinese_chars(text)376        orig_tokens = whitespace_tokenize(text)377        split_tokens = []378        for token in orig_tokens:379            if token not in never_split:380                if self.do_lower_case:381                    token = token.lower()382                    if self.strip_accents is not False:383                        token = self._run_strip_accents(token)384                elif self.strip_accents:385                    token = self._run_strip_accents(token)386            split_tokens.extend(self._run_split_on_punc(token, never_split))387 388        output_tokens = whitespace_tokenize(" ".join(split_tokens))389        return output_tokens390 391    def _run_strip_accents(self, text):392        """Strips accents from a piece of text."""393        text = unicodedata.normalize("NFD", text)394        output = []395        for char in text:396            cat = unicodedata.category(char)397            if cat == "Mn":398                continue399            output.append(char)400        return "".join(output)401 402    def _run_split_on_punc(self, text, never_split=None):403        """Splits punctuation on a piece of text."""404        if never_split is not None and text in never_split:405            return [text]406        chars = list(text)407        i = 0408        start_new_word = True409        output = []410        while i < len(chars):411            char = chars[i]412            if _is_punctuation(char):413                output.append([char])414                start_new_word = True415            else:416                if start_new_word:417                    output.append([])418                start_new_word = False419                output[-1].append(char)420            i += 1421 422        return ["".join(x) for x in output]423 424    def _tokenize_chinese_chars(self, text):425        """Adds whitespace around any CJK character."""426        output = []427        for char in text:428            cp = ord(char)429            if self._is_chinese_char(cp):430                output.append(" ")431                output.append(char)432                output.append(" ")433            else:434                output.append(char)435        return "".join(output)436 437    def _is_chinese_char(self, cp):438        """Checks whether CP is the codepoint of a CJK character."""439        # This defines a "chinese character" as anything in the CJK Unicode block:440        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)441        #442        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,443        # despite its name. The modern Korean Hangul alphabet is a different block,444        # as is Japanese Hiragana and Katakana. Those alphabets are used to write445        # space-separated words, so they are not treated specially and handled446        # like the all of the other languages.447        if (448            (cp >= 0x4E00 and cp <= 0x9FFF)449            or (cp >= 0x3400 and cp <= 0x4DBF)  #450            or (cp >= 0x20000 and cp <= 0x2A6DF)  #451            or (cp >= 0x2A700 and cp <= 0x2B73F)  #452            or (cp >= 0x2B740 and cp <= 0x2B81F)  #453            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #454            or (cp >= 0xF900 and cp <= 0xFAFF)455            or (cp >= 0x2F800 and cp <= 0x2FA1F)  #456        ):  #457            return True458 459        return False460 461    def _clean_text(self, text):462        """Performs invalid character removal and whitespace cleanup on text."""463        output = []464        for char in text:465            cp = ord(char)466            if cp == 0 or cp == 0xFFFD or _is_control(char):467                continue468            if _is_whitespace(char):469                output.append(" ")470            else:471                output.append(char)472        return "".join(output)473 474 475class WordpieceTokenizer(object):476    """Runs WordPiece tokenization."""477 478    def __init__(self, vocab, unk_token, max_input_chars_per_word=100):479        self.vocab = vocab480        self.unk_token = unk_token481        self.max_input_chars_per_word = max_input_chars_per_word482 483    def tokenize(self, text):484        """485        Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform486        tokenization using the given vocabulary.487 488        For example, `input = "unaffable"` wil return as output `["un", "##aff", "##able"]`.489 490        Args:491          text: A single token or whitespace separated tokens. This should have492            already been passed through *BasicTokenizer*.493 494        Returns:495          A list of wordpiece tokens.496        """497 498        output_tokens = []499        for token in whitespace_tokenize(text):500            chars = list(token)501            if len(chars) > self.max_input_chars_per_word:502                output_tokens.append(self.unk_token)503                continue504 505            is_bad = False506            start = 0507            sub_tokens = []508            while start < len(chars):509                end = len(chars)510                cur_substr = None511                while start < end:512                    substr = "".join(chars[start:end])513                    if start > 0:514                        substr = "##" + substr515                    if substr in self.vocab:516                        cur_substr = substr517                        break518                    end -= 1519                if cur_substr is None:520                    is_bad = True521                    break522                sub_tokens.append(cur_substr)523                start = end524 525            if is_bad:526                output_tokens.append(self.unk_token)527            else:528                output_tokens.extend(sub_tokens)529        return output_tokens530