MathLLMs/MathCoder-VL-8B

1# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved.2#3# This code is based on transformers/src/transformers/models/llama/tokenization_llama_fast.py4#5# Licensed under the Apache License, Version 2.0 (the "License");6# you may not use this file except in compliance with the License.7# You may obtain a copy of the License at8#9#     http://www.apache.org/licenses/LICENSE-2.010#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.14# See the License for the specific language governing permissions and15# limitations under the License.16 17"""Tokenization Fast class for InternLM."""18import os19from shutil import copyfile20from typing import Any, Dict, Optional, Tuple21 22from tokenizers import Tokenizer, decoders, normalizers, processors23from tokenizers.models import BPE24from transformers.convert_slow_tokenizer import (SLOW_TO_FAST_CONVERTERS,25                                                 SentencePieceExtractor,26                                                 SpmConverter)27from transformers.tokenization_utils_fast import PreTrainedTokenizerFast28from transformers.utils import logging29 30from .tokenization_internlm2 import InternLM2Tokenizer31 32logger = logging.get_logger(__name__)33 34VOCAB_FILES_NAMES = {'vocab_file': './tokenizer.model'}35 36 37# Modified from transformers.convert_slow_tokenizer.LlamaConverter38class InternLM2Converter(SpmConverter):39    handle_byte_fallback = True40 41    def vocab(self, proto):42        vocab = [43            ('<unk>', 0.0),44            ('<s>', 0.0),45            ('</s>', 0.0),46        ]47        vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]]48        return vocab49 50    def unk_id(self, proto):51        unk_id = 052        return unk_id53 54    def decoder(self, replacement, add_prefix_space):55        return decoders.Sequence(56            [57                decoders.Replace('▁', ' '),58                decoders.ByteFallback(),59                decoders.Fuse(),60                decoders.Strip(content=' ', left=1),61            ]62        )63 64    def tokenizer(self, proto):65        model_type = proto.trainer_spec.model_type66        vocab_scores = self.vocab(proto)67        # special tokens68        added_tokens = self.original_tokenizer.added_tokens_decoder69        for i in range(len(vocab_scores)):70            piece, score = vocab_scores[i]71            if i in added_tokens:72                vocab_scores[i] = (added_tokens[i].content, score)73        if model_type == 1:74            raise RuntimeError('InternLM2 is supposed to be a BPE model!')75 76        elif model_type == 2:77            _, merges = SentencePieceExtractor(self.original_tokenizer.vocab_file).extract(vocab_scores)78            bpe_vocab = {word: i for i, (word, _score) in enumerate(vocab_scores)}79            tokenizer = Tokenizer(80                BPE(bpe_vocab, merges, unk_token=proto.trainer_spec.unk_piece, fuse_unk=True, byte_fallback=True)81            )82            tokenizer.add_special_tokens(83                [ added_token for index, added_token in added_tokens.items()]84            )85        else:86            raise Exception(87                "You're trying to run a `Unigram` model but you're file was trained with a different algorithm"88            )89 90        return tokenizer91 92    def normalizer(self, proto):93        normalizers_list = []94        if proto.normalizer_spec.add_dummy_prefix:95            normalizers_list.append(normalizers.Prepend(prepend='▁'))96        normalizers_list.append(normalizers.Replace(pattern=' ', content='▁'))97        return normalizers.Sequence(normalizers_list)98 99    def pre_tokenizer(self, replacement, add_prefix_space):100        return None101 102 103SLOW_TO_FAST_CONVERTERS['InternLM2Tokenizer'] = InternLM2Converter104 105 106# Modified from transformers.model.llama.tokenization_llama_fast.LlamaTokenizerFast -> InternLM2TokenizerFast107class InternLM2TokenizerFast(PreTrainedTokenizerFast):108    vocab_files_names = VOCAB_FILES_NAMES109    slow_tokenizer_class = InternLM2Tokenizer110    padding_side = 'left'111    model_input_names = ['input_ids', 'attention_mask']112    _auto_class = 'AutoTokenizer'113 114    def __init__(115        self,116        vocab_file,117        unk_token='<unk>',118        bos_token='<s>',119        eos_token='</s>',120        pad_token='</s>',121        sp_model_kwargs: Optional[Dict[str, Any]] = None,122        add_bos_token=True,123        add_eos_token=False,124        decode_with_prefix_space=False,125        clean_up_tokenization_spaces=False,126        **kwargs,127    ):128        super().__init__(129            vocab_file=vocab_file,130            unk_token=unk_token,131            bos_token=bos_token,132            eos_token=eos_token,133            pad_token=pad_token,134            sp_model_kwargs=sp_model_kwargs,135            add_bos_token=add_bos_token,136            add_eos_token=add_eos_token,137            decode_with_prefix_space=decode_with_prefix_space,138            clean_up_tokenization_spaces=clean_up_tokenization_spaces,139            **kwargs,140        )141        self._add_bos_token = add_bos_token142        self._add_eos_token = add_eos_token143        self.update_post_processor()144        self.vocab_file = vocab_file145 146    @property147    def can_save_slow_tokenizer(self) -> bool:148        return os.path.isfile(self.vocab_file) if self.vocab_file else False149 150    def update_post_processor(self):151        """152        Updates the underlying post processor with the current `bos_token` and `eos_token`.153        """154        bos = self.bos_token155        bos_token_id = self.bos_token_id156        if bos is None and self.add_bos_token:157            raise ValueError('add_bos_token = True but bos_token = None')158 159        eos = self.eos_token160        eos_token_id = self.eos_token_id161        if eos is None and self.add_eos_token:162            raise ValueError('add_eos_token = True but eos_token = None')163 164        single = f"{(bos+':0 ') if self.add_bos_token else ''}$A:0{(' '+eos+':0') if self.add_eos_token else ''}"165        pair = f"{single}{(' '+bos+':1') if self.add_bos_token else ''} $B:1{(' '+eos+':1') if self.add_eos_token else ''}"166 167        special_tokens = []168        if self.add_bos_token:169            special_tokens.append((bos, bos_token_id))170        if self.add_eos_token:171            special_tokens.append((eos, eos_token_id))172        self._tokenizer.post_processor = processors.TemplateProcessing(173            single=single, pair=pair, special_tokens=special_tokens174        )175 176    @property177    def add_eos_token(self):178        return self._add_eos_token179 180    @property181    def add_bos_token(self):182        return self._add_bos_token183 184    @add_eos_token.setter185    def add_eos_token(self, value):186        self._add_eos_token = value187        self.update_post_processor()188 189    @add_bos_token.setter190    def add_bos_token(self, value):191        self._add_bos_token = value192        self.update_post_processor()193 194    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:195        if not self.can_save_slow_tokenizer:196            raise ValueError(197                'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '198                'tokenizer.'199            )200 201        if not os.path.isdir(save_directory):202            logger.error(f'Vocabulary path ({save_directory}) should be a directory')203            return204        out_vocab_file = os.path.join(205            save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']206        )207 208        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):209            copyfile(self.vocab_file, out_vocab_file)210 211        return (out_vocab_file,)212