DoruC/Grounded-Segment-Anything

1# coding=utf-82# Copyright 2022, UCLA NLP, The Facebook AI Research Team Authors and The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import os17from shutil import copyfile18from typing import Any, Dict, List, Optional, Tuple19 20import sentencepiece as spm21 22from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer23from ...utils import logging24 25 26logger = logging.get_logger(__name__)27 28SPIECE_UNDERLINE = "▁"29 30VOCAB_FILES_NAMES = {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}31 32PRETRAINED_VOCAB_FILES_MAP = {33    "vocab_file": {34        "uclanlp/plbart-base": "https://huggingface.co/uclanlp/plbart-base/resolve/main/sentencepiece.bpe.model",35        "uclanlp/plbart-c-cpp-defect-detection": (36            "https://huggingface.co/uclanlp/plbart-c-cpp-defect-detection/resolve/main/sentencepiece.bpe.model"37        ),38        "uclanlp/plbart-cs-java": "https://huggingface.co/uclanlp/plbart-cs-java/resolve/main/sentencepiece.bpe.model",39        "uclanlp/plbart-en_XX-java": (40            "https://huggingface.co/uclanlp/plbart-en_XX-java/resolve/main/sentencepiece.bpe.model"41        ),42        "uclanlp/plbart-go-en_XX": (43            "https://huggingface.co/uclanlp/plbart-go-en_XX/resolve/main/sentencepiece.bpe.model"44        ),45        "uclanlp/plbart-java-clone-detection": (46            "https://huggingface.co/uclanlp/plbart-java-clone-detection/resolve/main/sentencepiece.bpe.model"47        ),48        "uclanlp/plbart-java-cs": "https://huggingface.co/uclanlp/plbart-java-cs/resolve/main/sentencepiece.bpe.model",49        "uclanlp/plbart-java-en_XX": (50            "https://huggingface.co/uclanlp/plbart-java-en_XX/resolve/main/sentencepiece.bpe.model"51        ),52        "uclanlp/plbart-javascript-en_XX": (53            "https://huggingface.co/uclanlp/plbart-javascript-en_XX/resolve/main/sentencepiece.bpe.model"54        ),55        "uclanlp/plbart-php-en_XX": (56            "https://huggingface.co/uclanlp/plbart-php-en_XX/resolve/main/sentencepiece.bpe.model"57        ),58        "uclanlp/plbart-python-en_XX": (59            "https://huggingface.co/uclanlp/plbart-python-en_XX/resolve/main/sentencepiece.bpe.model"60        ),61        "uclanlp/plbart-refine-java-medium": (62            "https://huggingface.co/uclanlp/plbart-refine-java-medium/resolve/main/sentencepiece.bpe.model"63        ),64        "uclanlp/plbart-refine-java-small": (65            "https://huggingface.co/uclanlp/plbart-refine-java-small/resolve/main/sentencepiece.bpe.model"66        ),67        "uclanlp/plbart-ruby-en_XX": (68            "https://huggingface.co/uclanlp/plbart-ruby-en_XX/resolve/main/sentencepiece.bpe.model"69        ),70    }71}72 73PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {74    "uclanlp/plbart-base": 1024,75    "uclanlp/plbart-c-cpp-defect-detection": 1024,76    "uclanlp/plbart-cs-java": 1024,77    "uclanlp/plbart-en_XX-java": 1024,78    "uclanlp/plbart-go-en_XX": 1024,79    "uclanlp/plbart-java-clone-detection": 1024,80    "uclanlp/plbart-java-cs": 1024,81    "uclanlp/plbart-java-en_XX": 1024,82    "uclanlp/plbart-javascript-en_XX": 1024,83    "uclanlp/plbart-php-en_XX": 1024,84    "uclanlp/plbart-python-en_XX": 1024,85    "uclanlp/plbart-refine-java-medium": 1024,86    "uclanlp/plbart-refine-java-small": 1024,87    "uclanlp/plbart-ruby-en_XX": 1024,88}89 90FAIRSEQ_LANGUAGE_CODES = {91    "base": ["__java__", "__python__", "__en_XX__"],92    "multi": ["__java__", "__python__", "__en_XX__", "__javascript__", "__php__", "__ruby__", "__go__"],93}94 95FAIRSEQ_LANGUAGE_CODES_MAP = {96    "java": "__java__",97    "python": "__python__",98    "en_XX": "__en_XX__",99    "javascript": "__javascript__",100    "php": "__php__",101    "ruby": "__ruby__",102    "go": "__go__",103}104 105 106class PLBartTokenizer(PreTrainedTokenizer):107    """108    Construct an PLBART tokenizer.109 110    Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on111    [SentencePiece](https://github.com/google/sentencepiece).112 113    The tokenization method is `<tokens> <eos> <language code>` for source language documents, and `<language code>114    <tokens> <eos>` for target language documents.115 116    Args:117        vocab_file (`str`):118            Path to the vocabulary file.119        src_lang (`str`, *optional*):120            A string representing the source language.121        tgt_lang (`str`, *optional*):122            A string representing the target language.123        bos_token (`str`, *optional*, defaults to `"<s>"`):124            The start of sequence token.125        eos_token (`str`, *optional*, defaults to `"</s>"`):126            The end of sequence token.127        sep_token (`str`, *optional*, defaults to `"</s>"`):128            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for129            sequence classification or for a text and a question for question answering. It is also used as the last130            token of a sequence built with special tokens.131        cls_token (`str`, *optional*, defaults to `"<s>"`):132            The cls token, which is a special token used as the first token for all tasks.133        unk_token (`str`, *optional*, defaults to `"<unk>"`):134            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this135            token instead.136        pad_token (`str`, *optional*, defaults to `"<pad>"`):137            The token used for padding, for example when batching sequences of different lengths.138        mask_token(`str`, *optional*, defaults to `"<mask>"`):139            The token used for masking values. This is the token used when training this model with masking tasks. This140            is only used in the `"base"` tokenizer type. For `"multi"` tokenizer, masking is never done for the141            downstream tasks.142        language_codes (`str`, *optional*, defaults to `"base"`):143            What language codes to use. Should be one of `"base"` or `"multi"`.144        sp_model_kwargs (`dict`, *optional*):145            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for146            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,147            to set:148            - `enable_sampling`: Enable subword regularization.149            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.150              - `nbest_size = {0,1}`: No sampling is performed.151              - `nbest_size > 1`: samples from the nbest_size results.152              - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)153                using forward-filtering-and-backward-sampling algorithm.154            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for155              BPE-dropout.156 157    Examples:158 159    ```python160    >>> from transformers import PLBartTokenizer161 162    >>> tokenizer = PLBartTokenizer.from_pretrained("uclanlp/plbart-python-en_XX", src_lang="python", tgt_lang="en_XX")163    >>> example_python_phrase = "def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])"164    >>> expected_translation_english = "Returns the maximum value of a b c."165    >>> inputs = tokenizer(example_python_phrase, text_target=expected_translation_english, return_tensors="pt")166    ```"""167 168    vocab_files_names = VOCAB_FILES_NAMES169    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES170    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP171    model_input_names = ["input_ids", "attention_mask"]172 173    prefix_tokens: List[int] = []174    suffix_tokens: List[int] = []175 176    def __init__(177        self,178        vocab_file,179        bos_token="<s>",180        eos_token="</s>",181        sep_token="</s>",182        cls_token="<s>",183        unk_token="<unk>",184        pad_token="<pad>",185        mask_token="<mask>",186        language_codes="base",187        tokenizer_file=None,188        src_lang=None,189        tgt_lang=None,190        sp_model_kwargs: Optional[Dict[str, Any]] = None,191        additional_special_tokens=None,192        **kwargs,193    ):194        # Mask token behave like a normal word, i.e. include the space before it195        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token196 197        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs198        src_lang = self._convert_lang_code_special_format(src_lang)199        tgt_lang = self._convert_lang_code_special_format(tgt_lang)200 201        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)202        self.sp_model.Load(str(vocab_file))203        self.vocab_file = vocab_file204        self.language_codes = language_codes205 206        fairseq_language_codes = FAIRSEQ_LANGUAGE_CODES[self.language_codes]207 208        # Original fairseq vocab and spm vocab must be "aligned":209        # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9210        # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----211        # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'212        # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'213 214        # Mimic fairseq token-to-id alignment for the first 4 token215        self.fairseq_tokens_to_ids = {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}216 217        # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab218        self.fairseq_offset = 1219 220        self.sp_model_size = len(self.sp_model)221        self.lang_code_to_id = {222            code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(fairseq_language_codes)223        }224        self.id_to_lang_code = {v: k for k, v in self.lang_code_to_id.items()}225 226        if self.language_codes == "base":227            self.fairseq_tokens_to_ids["<mask>"] = len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset228 229        self.fairseq_tokens_to_ids.update(self.lang_code_to_id)230        self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}231        _additional_special_tokens = list(self.lang_code_to_id.keys())232 233        if additional_special_tokens is not None:234            # Only add those special tokens if they are not already there.235            _additional_special_tokens.extend(236                [t for t in additional_special_tokens if t not in _additional_special_tokens]237            )238 239        if self.language_codes == "base":240            self._src_lang = src_lang241            self.cur_lang_code_id = (242                self.lang_code_to_id[self._src_lang] if self._src_lang is not None else self._src_lang243            )244        else:245            self._src_lang = src_lang if src_lang is not None else "__en_XX__"246            self.cur_lang_code_id = self.lang_code_to_id[self._src_lang]247 248        super().__init__(249            bos_token=bos_token,250            eos_token=eos_token,251            unk_token=unk_token,252            sep_token=sep_token,253            cls_token=cls_token,254            pad_token=pad_token,255            mask_token=mask_token,256            language_codes=language_codes,257            tokenizer_file=tokenizer_file,258            src_lang=src_lang,259            tgt_lang=tgt_lang,260            additional_special_tokens=_additional_special_tokens,261            sp_model_kwargs=self.sp_model_kwargs,262            **kwargs,263        )264 265        self.tgt_lang = tgt_lang266        self.set_src_lang_special_tokens(self._src_lang)267 268    def __getstate__(self):269        state = self.__dict__.copy()270        state["sp_model"] = None271        state["sp_model_proto"] = self.sp_model.serialized_model_proto()272        return state273 274    def __setstate__(self, d):275        self.__dict__ = d276 277        # for backward compatibility278        if not hasattr(self, "sp_model_kwargs"):279            self.sp_model_kwargs = {}280 281        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)282        self.sp_model.LoadFromSerializedProto(self.sp_model_proto)283 284    @property285    def vocab_size(self):286        if self.language_codes == "base":287            return (288                len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset + 1289            )  # Plus 1 for the mask token290        else:291            return len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset292 293    @property294    def src_lang(self) -> str:295        return self._src_lang296 297    @src_lang.setter298    def src_lang(self, new_src_lang: str) -> None:299        new_src_lang = self._convert_lang_code_special_format(new_src_lang)300        self._src_lang = new_src_lang301        self.set_src_lang_special_tokens(self._src_lang)302 303    def get_special_tokens_mask(304        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False305    ) -> List[int]:306        """307        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding308        special tokens using the tokenizer `prepare_for_model` method.309 310        Args:311            token_ids_0 (`List[int]`):312                List of IDs.313            token_ids_1 (`List[int]`, *optional*):314                Optional second list of IDs for sequence pairs.315            already_has_special_tokens (`bool`, *optional*, defaults to `False`):316                Whether or not the token list is already formatted with special tokens for the model.317 318        Returns:319            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.320        """321 322        if already_has_special_tokens:323            return super().get_special_tokens_mask(324                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True325            )326 327        prefix_ones = [1] * len(self.prefix_tokens)328        suffix_ones = [1] * len(self.suffix_tokens)329        if token_ids_1 is None:330            return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones331        return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones332 333    def build_inputs_with_special_tokens(334        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None335    ) -> List[int]:336        """337        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and338        adding special tokens. An PLBART sequence has the following format, where `X` represents the sequence:339 340        - `input_ids` (for encoder) `X [eos, src_lang_code]`341        - `decoder_input_ids`: (for decoder) `X [eos, tgt_lang_code]`342 343        BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a344        separator.345 346        Args:347            token_ids_0 (`List[int]`):348                List of IDs to which the special tokens will be added.349            token_ids_1 (`List[int]`, *optional*):350                Optional second list of IDs for sequence pairs.351 352        Returns:353            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.354        """355        if token_ids_1 is None:356            return self.prefix_tokens + token_ids_0 + self.suffix_tokens357        # We don't expect to process pairs, but leave the pair logic for API consistency358        return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens359 360    def create_token_type_ids_from_sequences(361        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None362    ) -> List[int]:363        """364        Create a mask from the two sequences passed to be used in a sequence-pair classification task. PLBart does not365        make use of token type ids, therefore a list of zeros is returned.366 367        Args:368            token_ids_0 (`List[int]`):369                List of IDs.370            token_ids_1 (`List[int]`, *optional*):371                Optional second list of IDs for sequence pairs.372 373        Returns:374            `List[int]`: List of zeros.375        """376 377        sep = [self.sep_token_id]378        cls = [self.cls_token_id]379 380        if token_ids_1 is None:381            return len(cls + token_ids_0 + sep) * [0]382        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]383 384    def _build_translation_inputs(385        self, raw_inputs, return_tensors: str, src_lang: Optional[str], tgt_lang: Optional[str], **extra_kwargs386    ):387        """Used by translation pipeline, to prepare inputs for the generate function"""388        if src_lang is None or tgt_lang is None:389            raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model")390        self.src_lang = self._convert_lang_code_special_format(src_lang)391        self.tgt_lang = self._convert_lang_code_special_format(tgt_lang)392        inputs = self(raw_inputs, add_special_tokens=True, return_tensors=return_tensors, **extra_kwargs)393        tgt_lang_id = self.convert_tokens_to_ids(self.tgt_lang)394        inputs["forced_bos_token_id"] = tgt_lang_id395        return inputs396 397    def get_vocab(self):398        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}399        vocab.update(self.added_tokens_encoder)400        return vocab401 402    def _tokenize(self, text: str) -> List[str]:403        return self.sp_model.encode(text, out_type=str)404 405    def _convert_token_to_id(self, token):406        """Converts a token (str) in an id using the vocab."""407        if token in self.fairseq_tokens_to_ids:408            return self.fairseq_tokens_to_ids[token]409        spm_id = self.sp_model.PieceToId(token)410 411        # Need to return unknown token if the SP model returned 0412        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id413 414    def _convert_id_to_token(self, index):415        """Converts an index (integer) in a token (str) using the vocab."""416        if index in self.fairseq_ids_to_tokens:417            return self.fairseq_ids_to_tokens[index]418        return self.sp_model.IdToPiece(index - self.fairseq_offset)419 420    def convert_tokens_to_string(self, tokens):421        """Converts a sequence of tokens (strings for sub-words) in a single string."""422        out_string = "".join(tokens).replace(SPIECE_UNDERLINE, " ").strip()423        return out_string424 425    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:426        if not os.path.isdir(save_directory):427            logger.error(f"Vocabulary path ({save_directory}) should be a directory")428            return429        out_vocab_file = os.path.join(430            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]431        )432 433        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):434            copyfile(self.vocab_file, out_vocab_file)435        elif not os.path.isfile(self.vocab_file):436            with open(out_vocab_file, "wb") as fi:437                content_spiece_model = self.sp_model.serialized_model_proto()438                fi.write(content_spiece_model)439 440        return (out_vocab_file,)441 442    def prepare_seq2seq_batch(443        self,444        src_texts: List[str],445        src_lang: str = "en_XX",446        tgt_texts: Optional[List[str]] = None,447        tgt_lang: str = "python",448        **kwargs,449    ) -> BatchEncoding:450        self.src_lang = self._convert_lang_code_special_format(src_lang)451        self.tgt_lang = self._convert_lang_code_special_format(tgt_lang)452        return super().prepare_seq2seq_batch(src_texts, tgt_texts, **kwargs)453 454    def _switch_to_input_mode(self):455        return self.set_src_lang_special_tokens(self.src_lang)456 457    def _switch_to_target_mode(self):458        return self.set_tgt_lang_special_tokens(self.tgt_lang)459 460    def set_src_lang_special_tokens(self, src_lang) -> None:461        """Reset the special tokens to the source lang setting. No prefix and suffix=[eos, src_lang_code]."""462        src_lang = self._convert_lang_code_special_format(src_lang)463        self.cur_lang_code = self.lang_code_to_id[src_lang] if src_lang is not None else None464        self.prefix_tokens = []465        if self.cur_lang_code is not None:466            self.suffix_tokens = [self.eos_token_id, self.cur_lang_code]467        else:468            self.suffix_tokens = [self.eos_token_id]469 470    def set_tgt_lang_special_tokens(self, lang: str) -> None:471        """Reset the special tokens to the target language setting. No prefix and suffix=[eos, tgt_lang_code]."""472        lang = self._convert_lang_code_special_format(lang)473 474        self.cur_lang_code = self.lang_code_to_id[lang] if lang is not None else None475        self.prefix_tokens = []476        if self.cur_lang_code is not None:477            self.suffix_tokens = [self.eos_token_id, self.cur_lang_code]478        else:479            self.suffix_tokens = [self.eos_token_id]480 481    def _convert_lang_code_special_format(self, lang: str) -> str:482        """Convert Language Codes to format tokenizer uses if required"""483        lang = FAIRSEQ_LANGUAGE_CODES_MAP[lang] if lang in FAIRSEQ_LANGUAGE_CODES_MAP.keys() else lang484        return lang485