Snowflake/snowflake-arctic-instruct-vllm

1"""Tokenization classes for Arctic."""2 3from typing import Any, Dict, Optional4 5from transformers.models.llama import LlamaTokenizer6 7 8class ArcticTokenizer(LlamaTokenizer):9 10    def __init__(11        self,12        vocab_file,13        unk_token="<unk>",14        bos_token="<s>",15        eos_token="</s>",16        pad_token=None,17        sp_model_kwargs: Optional[Dict[str, Any]] = None,18        add_bos_token=True,19        add_eos_token=False,20        clean_up_tokenization_spaces=False,21        use_default_system_prompt=False,22        spaces_between_special_tokens=False,23        legacy=False,24        add_prefix_space=True,25        **kwargs,26    ):27        # Same as LlamaTokenizer except default legacy=False.28        super().__init__(29            vocab_file,30            bos_token=bos_token,31            eos_token=eos_token,32            unk_token=unk_token,33            pad_token=pad_token,34            sp_model_kwargs=sp_model_kwargs,35            add_bos_token=add_bos_token,36            add_eos_token=add_eos_token,37            clean_up_tokenization_spaces=clean_up_tokenization_spaces,38            use_default_system_prompt=use_default_system_prompt,39            spaces_between_special_tokens=spaces_between_special_tokens,40            legacy=legacy,41            add_prefix_space=add_prefix_space,42            **kwargs,43        )44 45    @property46    def default_chat_template(self):47        """48        This template formats inputs in the standard Arctic format.49        """50        return (51            "{% for message in messages %}"52            "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"53            "{% endfor %}"54            "{% if add_generation_prompt %}"55            "{{ '<|im_start|>assistant\n' }}"56            "{% endif %}"57        )58