optimum-intel-internal-testing/tiny-random-minicpmv-2_6

1from transformers.models.qwen2 import Qwen2TokenizerFast2 3 4class MiniCPMVTokenizerFast(Qwen2TokenizerFast):5    def __init__(self, **kwargs):6        super().__init__(**kwargs)7        self.im_start = "<image>"8        self.im_end = "</image>"9        self.ref_start = "<ref>"10        self.ref_end = "</ref>"11        self.box_start = "<box>"12        self.box_end = "</box>"13        self.quad_start = "<quad>"14        self.quad_end = "</quad>"15        self.slice_start = "<slice>"16        self.slice_end = "</slice>"17        self.im_id_start = "<image_id>"18        self.im_id_end = "</image_id>"19 20    @property21    def eos_id(self):22        return self.eos_token_id23 24    @property25    def bos_id(self):26        return self.bos_token_id27 28    @property29    def unk_id(self):30        return self.unk_token_id31 32    @property33    def im_start_id(self):34        return self.convert_tokens_to_ids(self.im_start)35 36    @property37    def im_end_id(self):38        return self.convert_tokens_to_ids(self.im_end)39 40    @property41    def slice_start_id(self):42        return self.convert_tokens_to_ids(self.slice_start)43 44    @property45    def slice_end_id(self):46        return self.convert_tokens_to_ids(self.slice_end)47 48    @property49    def im_id_start_id(self):50        return self.convert_tokens_to_ids(self.im_id_start)51 52    @property53    def im_id_end_id(self):54        return self.convert_tokens_to_ids(self.im_id_end)55 56    @property57    def newline_id(self):58        return self.convert_tokens_to_ids("\n")59 60    @staticmethod61    def escape(text: str) -> str:62        return text63 64    @staticmethod65    def unescape(text: str) -> str:66        return text67