DoruC/Grounded-Segment-Anything

1# coding=utf-82# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import json17import os18from functools import lru_cache19from typing import List, Optional, Tuple20 21import regex as re22 23from ...tokenization_utils import AddedToken, PreTrainedTokenizer24from ...utils import logging25 26 27logger = logging.get_logger(__name__)28 29 30VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}31 32# See all BART models at https://huggingface.co/models?filter=bart33PRETRAINED_VOCAB_FILES_MAP = {34    "vocab_file": {35        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/vocab.json",36        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/vocab.json",37        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json",38        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json",39        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json",40        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json",41    },42    "merges_file": {43        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/merges.txt",44        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/merges.txt",45        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt",46        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt",47        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt",48        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt",49    },50}51 52PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {53    "facebook/bart-base": 1024,54    "facebook/bart-large": 1024,55    "facebook/bart-large-mnli": 1024,56    "facebook/bart-large-cnn": 1024,57    "facebook/bart-large-xsum": 1024,58    "yjernite/bart_eli5": 1024,59}60 61 62@lru_cache()63def bytes_to_unicode():64    """65    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control66    characters the bpe code barfs on.67 68    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab69    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for70    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup71    tables between utf-8 bytes and unicode strings.72    """73    bs = (74        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))75    )76    cs = bs[:]77    n = 078    for b in range(2**8):79        if b not in bs:80            bs.append(b)81            cs.append(2**8 + n)82            n += 183    cs = [chr(n) for n in cs]84    return dict(zip(bs, cs))85 86 87def get_pairs(word):88    """89    Return set of symbol pairs in a word.90 91    Word is represented as tuple of symbols (symbols being variable-length strings).92    """93    pairs = set()94    prev_char = word[0]95    for char in word[1:]:96        pairs.add((prev_char, char))97        prev_char = char98    return pairs99 100 101class BartTokenizer(PreTrainedTokenizer):102    """103    Constructs a BART tokenizer, which is smilar to the ROBERTa tokenizer, using byte-level Byte-Pair-Encoding.104 105    This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will106    be encoded differently whether it is at the beginning of the sentence (without space) or not:107 108    ```python109    >>> from transformers import BartTokenizer110 111    >>> tokenizer = BartTokenizer.from_pretrained("facebook/bart-base")112    >>> tokenizer("Hello world")["input_ids"]113    [0, 31414, 232, 2]114 115    >>> tokenizer(" Hello world")["input_ids"]116    [0, 20920, 232, 2]117    ```118 119    You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer or when you120    call it on some text, but since the model was not pretrained this way, it might yield a decrease in performance.121 122    <Tip>123 124    When used with `is_split_into_words=True`, this tokenizer will add a space before each word (even the first one).125 126    </Tip>127 128    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to129    this superclass for more information regarding those methods.130 131    Args:132        vocab_file (`str`):133            Path to the vocabulary file.134        merges_file (`str`):135            Path to the merges file.136        errors (`str`, *optional*, defaults to `"replace"`):137            Paradigm to follow when decoding bytes to UTF-8. See138            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.139        bos_token (`str`, *optional*, defaults to `"<s>"`):140            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.141 142            <Tip>143 144            When building a sequence using special tokens, this is not the token that is used for the beginning of145            sequence. The token used is the `cls_token`.146 147            </Tip>148 149        eos_token (`str`, *optional*, defaults to `"</s>"`):150            The end of sequence token.151 152            <Tip>153 154            When building a sequence using special tokens, this is not the token that is used for the end of sequence.155            The token used is the `sep_token`.156 157            </Tip>158 159        sep_token (`str`, *optional*, defaults to `"</s>"`):160            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for161            sequence classification or for a text and a question for question answering. It is also used as the last162            token of a sequence built with special tokens.163        cls_token (`str`, *optional*, defaults to `"<s>"`):164            The classifier token which is used when doing sequence classification (classification of the whole sequence165            instead of per-token classification). It is the first token of the sequence when built with special tokens.166        unk_token (`str`, *optional*, defaults to `"<unk>"`):167            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this168            token instead.169        pad_token (`str`, *optional*, defaults to `"<pad>"`):170            The token used for padding, for example when batching sequences of different lengths.171        mask_token (`str`, *optional*, defaults to `"<mask>"`):172            The token used for masking values. This is the token used when training this model with masked language173            modeling. This is the token which the model will try to predict.174        add_prefix_space (`bool`, *optional*, defaults to `False`):175            Whether or not to add an initial space to the input. This allows to treat the leading word just as any176            other word. (BART tokenizer detect beginning of words by the preceding space).177    """178 179    vocab_files_names = VOCAB_FILES_NAMES180    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP181    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES182    model_input_names = ["input_ids", "attention_mask"]183 184    def __init__(185        self,186        vocab_file,187        merges_file,188        errors="replace",189        bos_token="<s>",190        eos_token="</s>",191        sep_token="</s>",192        cls_token="<s>",193        unk_token="<unk>",194        pad_token="<pad>",195        mask_token="<mask>",196        add_prefix_space=False,197        **kwargs,198    ):199        bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token200        eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token201        sep_token = AddedToken(sep_token, lstrip=False, rstrip=False) if isinstance(sep_token, str) else sep_token202        cls_token = AddedToken(cls_token, lstrip=False, rstrip=False) if isinstance(cls_token, str) else cls_token203        unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token204        pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token205 206        # Mask token behave like a normal word, i.e. include the space before it207        # TODO seems like both slow and fast actually don't strip left and right soooooooo yeah. See `test_embeded_special_tokens`208        # Also this not only will strip the spaces but any punctuation209        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token210 211        with open(vocab_file, encoding="utf-8") as vocab_handle:212            self.encoder = json.load(vocab_handle)213        self.decoder = {v: k for k, v in self.encoder.items()}214        self.errors = errors  # how to handle errors in decoding215        self.byte_encoder = bytes_to_unicode()216        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}217        with open(merges_file, encoding="utf-8") as merges_handle:218            bpe_merges = merges_handle.read().split("\n")[1:-1]219        bpe_merges = [tuple(merge.split()) for merge in bpe_merges]220        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))221        self.cache = {}222        self.add_prefix_space = add_prefix_space223 224        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions225        self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")226 227        super().__init__(228            errors=errors,229            bos_token=bos_token,230            eos_token=eos_token,231            unk_token=unk_token,232            sep_token=sep_token,233            cls_token=cls_token,234            pad_token=pad_token,235            mask_token=mask_token,236            add_prefix_space=add_prefix_space,237            **kwargs,238        )239 240    @property241    def vocab_size(self):242        return len(self.encoder)243 244    def get_vocab(self):245        return dict(self.encoder, **self.added_tokens_encoder)246 247    def bpe(self, token):248        if token in self.cache:249            return self.cache[token]250        word = tuple(token)251        pairs = get_pairs(word)252 253        if not pairs:254            return token255 256        while True:257            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))258            if bigram not in self.bpe_ranks:259                break260            first, second = bigram261            new_word = []262            i = 0263            while i < len(word):264                try:265                    j = word.index(first, i)266                except ValueError:267                    new_word.extend(word[i:])268                    break269                else:270                    new_word.extend(word[i:j])271                    i = j272 273                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:274                    new_word.append(first + second)275                    i += 2276                else:277                    new_word.append(word[i])278                    i += 1279            new_word = tuple(new_word)280            word = new_word281            if len(word) == 1:282                break283            else:284                pairs = get_pairs(word)285        word = " ".join(word)286        self.cache[token] = word287        return word288 289    def _tokenize(self, text):290        """Tokenize a string."""291        bpe_tokens = []292        for token in re.findall(self.pat, text):293            token = "".join(294                self.byte_encoder[b] for b in token.encode("utf-8")295            )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)296            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))297        return bpe_tokens298 299    def _convert_token_to_id(self, token):300        """Converts a token (str) in an id using the vocab."""301        return self.encoder.get(token, self.encoder.get(self.unk_token))302 303    def _convert_id_to_token(self, index):304        """Converts an index (integer) in a token (str) using the vocab."""305        return self.decoder.get(index)306 307    def convert_tokens_to_string(self, tokens):308        """Converts a sequence of tokens (string) in a single string."""309        text = "".join(tokens)310        text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)311        return text312 313    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:314        if not os.path.isdir(save_directory):315            logger.error(f"Vocabulary path ({save_directory}) should be a directory")316            return317        vocab_file = os.path.join(318            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]319        )320        merge_file = os.path.join(321            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]322        )323 324        with open(vocab_file, "w", encoding="utf-8") as f:325            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")326 327        index = 0328        with open(merge_file, "w", encoding="utf-8") as writer:329            writer.write("#version: 0.2\n")330            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):331                if index != token_index:332                    logger.warning(333                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."334                        " Please check that the tokenizer is not corrupted!"335                    )336                    index = token_index337                writer.write(" ".join(bpe_tokens) + "\n")338                index += 1339 340        return vocab_file, merge_file341 342    def build_inputs_with_special_tokens(343        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None344    ) -> List[int]:345        """346        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and347        adding special tokens. A BART sequence has the following format:348 349        - single sequence: `<s> X </s>`350        - pair of sequences: `<s> A </s></s> B </s>`351 352        Args:353            token_ids_0 (`List[int]`):354                List of IDs to which the special tokens will be added.355            token_ids_1 (`List[int]`, *optional*):356                Optional second list of IDs for sequence pairs.357 358        Returns:359            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.360        """361        if token_ids_1 is None:362            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]363        cls = [self.cls_token_id]364        sep = [self.sep_token_id]365        return cls + token_ids_0 + sep + sep + token_ids_1 + sep366 367    def get_special_tokens_mask(368        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False369    ) -> List[int]:370        """371        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding372        special tokens using the tokenizer `prepare_for_model` method.373 374        Args:375            token_ids_0 (`List[int]`):376                List of IDs.377            token_ids_1 (`List[int]`, *optional*):378                Optional second list of IDs for sequence pairs.379            already_has_special_tokens (`bool`, *optional*, defaults to `False`):380                Whether or not the token list is already formatted with special tokens for the model.381 382        Returns:383            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.384        """385        if already_has_special_tokens:386            return super().get_special_tokens_mask(387                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True388            )389 390        if token_ids_1 is None:391            return [1] + ([0] * len(token_ids_0)) + [1]392        return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]393 394    def create_token_type_ids_from_sequences(395        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None396    ) -> List[int]:397        """398        Create a mask from the two sequences passed to be used in a sequence-pair classification task. BART does not399        make use of token type ids, therefore a list of zeros is returned.400 401        Args:402            token_ids_0 (`List[int]`):403                List of IDs.404            token_ids_1 (`List[int]`, *optional*):405                Optional second list of IDs for sequence pairs.406 407        Returns:408            `List[int]`: List of zeros.409        """410        sep = [self.sep_token_id]411        cls = [self.cls_token_id]412 413        if token_ids_1 is None:414            return len(cls + token_ids_0 + sep) * [0]415        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]416 417    def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs):418        add_prefix_space = kwargs.pop("add_prefix_space", self.add_prefix_space)419        if (is_split_into_words or add_prefix_space) and (len(text) > 0 and not text[0].isspace()):420            text = " " + text421        return (text, kwargs)422