XaviXva/Video-LLaVA

1from transformers import CLIPTokenizer2from transformers.utils import logging3 4logger = logging.get_logger(__name__)5 6VOCAB_FILES_NAMES = {7    "vocab_file": "vocab.json",8    "merges_file": "merges.txt",9}10 11PRETRAINED_VOCAB_FILES_MAP = {12    "vocab_file": {13        "lb203/LanguageBind-Image": "https://huggingface.co/lb203/LanguageBind-Image/resolve/main/vocab.json",14    },15    "merges_file": {16        "lb203/LanguageBind-Image": "https://huggingface.co/lb203/LanguageBind-Image/resolve/main/merges.txt",17    },18}19 20PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {21    "lb203/LanguageBind-Image": 77,22}23 24 25PRETRAINED_INIT_CONFIGURATION = {26    "lb203/LanguageBind-Image": {},27}28 29class LanguageBindImageTokenizer(CLIPTokenizer):30    """31    Construct a CLIP tokenizer. Based on byte-level Byte-Pair-Encoding.32 33    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to34    this superclass for more information regarding those methods.35 36    Args:37        vocab_file (`str`):38            Path to the vocabulary file.39        merges_file (`str`):40            Path to the merges file.41        errors (`str`, *optional*, defaults to `"replace"`):42            Paradigm to follow when decoding bytes to UTF-8. See43            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.44        unk_token (`str`, *optional*, defaults to `<|endoftext|>`):45            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this46            token instead.47        bos_token (`str`, *optional*, defaults to `<|startoftext|>`):48            The beginning of sequence token.49        eos_token (`str`, *optional*, defaults to `<|endoftext|>`):50            The end of sequence token.51    """52 53    vocab_files_names = VOCAB_FILES_NAMES54    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP55    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES56    model_input_names = ["input_ids", "attention_mask"]57 58    def __init__(59            self,60            vocab_file,61            merges_file,62            errors="replace",63            unk_token="<|endoftext|>",64            bos_token="<|startoftext|>",65            eos_token="<|endoftext|>",66            pad_token="<|endoftext|>",  # hack to enable padding67            **kwargs,68    ):69        super(LanguageBindImageTokenizer, self).__init__(70            vocab_file,71            merges_file,72            errors,73            unk_token,74            bos_token,75            eos_token,76            pad_token,  # hack to enable padding77            **kwargs,)