huiqian/tiny-sentiment-classifier

1# tokenization_tinytransformer.py （最完整修复版）2 3from transformers import PreTrainedTokenizer4import json5from typing import List, Dict, Optional6 7class TinyTokenizer(PreTrainedTokenizer):8    vocab_files_names = {"vocab_file": "vocab.json"}9    pretrained_vocab_files_map = {}10    max_model_input_sizes = {"tinytransformer": 512}11    model_input_names = ["input_ids", "attention_mask"]12 13    def __init__(self, vocab_file: Optional[str] = None, **kwargs):14        # 特殊 token15        self.special_tokens = {16            "[PAD]": 0,17            "[UNK]": 1,18            "[CLS]": 2,19            "[SEP]": 3,20        }21 22        # 构建 vocab23        self.vocab: Dict[str, int] = self.special_tokens.copy()24        offset = len(self.vocab)25 26        # ASCII + 常用字符27        for i in range(32, 127):28            char = chr(i)29            self.vocab[char] = offset + i - 3230 31        # 支持中文（常用汉字范围，可扩展）32        for i in range(0x4e00, 0x9fff + 1):33            char = chr(i)34            if char not in self.vocab:35                self.vocab[char] = len(self.vocab)36 37        self.id_to_token = {v: k for k, v in self.vocab.items()}38 39        # 设置特殊 token id40        self.pad_token_id = 041        self.unk_token_id = 142        self.cls_token_id = 243        self.sep_token_id = 344 45        super().__init__(46            pad_token="[PAD]",47            unk_token="[UNK]",48            cls_token="[CLS]",49            sep_token="[SEP]",50            **kwargs51        )52 53    def get_vocab(self) -> Dict[str, int]:54        return self.vocab.copy()55 56    @property57    def vocab_size(self) -> int:58        return len(self.vocab)59 60    def _tokenize(self, text: str) -> List[str]:61        return list(text)  # 字符级分词62 63    def convert_tokens_to_ids(self, tokens: List[str]) -> List[int]:64        return [self.vocab.get(t, self.unk_token_id) for t in tokens]65 66    def convert_ids_to_tokens(self, ids: List[int]) -> List[str]:67        return [self.id_to_token.get(i, "[UNK]") for i in ids]68 69    def build_inputs_with_special_tokens(70        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None71    ) -> List[int]:72        if token_ids_1 is None:73            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]74        return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + token_ids_1 + [self.sep_token_id]75 76    def create_token_type_ids_from_sequences(77        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None78    ) -> List[int]:79        if token_ids_1 is None:80            return [0] * len([self.cls_token_id] + token_ids_0 + [self.sep_token_id])81        len0 = len([self.cls_token_id] + token_ids_0 + [self.sep_token_id])82        len1 = len(token_ids_1 + [self.sep_token_id])83        return [0] * len0 + [1] * len184 85    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple:86        vocab_file = f"{filename_prefix}vocab.json" if filename_prefix else "vocab.json"87        vocab_path = f"{save_directory}/{vocab_file}"88        with open(vocab_path, "w", encoding="utf-8") as f:89            json.dump(self.vocab, f, ensure_ascii=False, indent=2)90        return (vocab_path,)91    92 93# # 文件最底部，类定义之后94# from transformers import TOKENIZER_MAPPING_NAMES95 96# # 注册（只执行一次，放在这里最安全）97# TOKENIZER_MAPPING_NAMES["tinytransformer"] = "TinyTokenizer"