Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2022 The REALM authors and The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Tokenization classes for REALM."""16 17import collections18import os19import unicodedata20from typing import Optional21 22from ....tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace23from ....tokenization_utils_base import BatchEncoding24from ....utils import PaddingStrategy, logging25 26 27logger = logging.get_logger(__name__)28 29VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}30 31 32def load_vocab(vocab_file):33    """Loads a vocabulary file into a dictionary."""34    vocab = collections.OrderedDict()35    with open(vocab_file, "r", encoding="utf-8") as reader:36        tokens = reader.readlines()37    for index, token in enumerate(tokens):38        token = token.rstrip("\n")39        vocab[token] = index40    return vocab41 42 43def whitespace_tokenize(text):44    """Runs basic whitespace cleaning and splitting on a piece of text."""45    text = text.strip()46    if not text:47        return []48    tokens = text.split()49    return tokens50 51 52class RealmTokenizer(PreTrainedTokenizer):53    r"""54    Construct a REALM tokenizer.55 56    [`RealmTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end tokenization: punctuation splitting and57    wordpiece.58 59    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to60    this superclass for more information regarding those methods.61 62    Args:63        vocab_file (`str`):64            File containing the vocabulary.65        do_lower_case (`bool`, *optional*, defaults to `True`):66            Whether or not to lowercase the input when tokenizing.67        do_basic_tokenize (`bool`, *optional*, defaults to `True`):68            Whether or not to do basic tokenization before WordPiece.69        never_split (`Iterable`, *optional*):70            Collection of tokens which will never be split during tokenization. Only has an effect when71            `do_basic_tokenize=True`72        unk_token (`str`, *optional*, defaults to `"[UNK]"`):73            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this74            token instead.75        sep_token (`str`, *optional*, defaults to `"[SEP]"`):76            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for77            sequence classification or for a text and a question for question answering. It is also used as the last78            token of a sequence built with special tokens.79        pad_token (`str`, *optional*, defaults to `"[PAD]"`):80            The token used for padding, for example when batching sequences of different lengths.81        cls_token (`str`, *optional*, defaults to `"[CLS]"`):82            The classifier token which is used when doing sequence classification (classification of the whole sequence83            instead of per-token classification). It is the first token of the sequence when built with special tokens.84        mask_token (`str`, *optional*, defaults to `"[MASK]"`):85            The token used for masking values. This is the token used when training this model with masked language86            modeling. This is the token which the model will try to predict.87        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):88            Whether or not to tokenize Chinese characters.89 90            This should likely be deactivated for Japanese (see this91            [issue](https://github.com/huggingface/transformers/issues/328)).92        strip_accents (`bool`, *optional*):93            Whether or not to strip all accents. If this option is not specified, then it will be determined by the94            value for `lowercase` (as in the original BERT).95    """96 97    vocab_files_names = VOCAB_FILES_NAMES98 99    def __init__(100        self,101        vocab_file,102        do_lower_case=True,103        do_basic_tokenize=True,104        never_split=None,105        unk_token="[UNK]",106        sep_token="[SEP]",107        pad_token="[PAD]",108        cls_token="[CLS]",109        mask_token="[MASK]",110        tokenize_chinese_chars=True,111        strip_accents=None,112        **kwargs,113    ):114        if not os.path.isfile(vocab_file):115            raise ValueError(116                f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"117                " model use `tokenizer = RealmTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"118            )119        self.vocab = load_vocab(vocab_file)120        self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])121        self.do_basic_tokenize = do_basic_tokenize122        if do_basic_tokenize:123            self.basic_tokenizer = BasicTokenizer(124                do_lower_case=do_lower_case,125                never_split=never_split,126                tokenize_chinese_chars=tokenize_chinese_chars,127                strip_accents=strip_accents,128            )129        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=str(unk_token))130        super().__init__(131            do_lower_case=do_lower_case,132            do_basic_tokenize=do_basic_tokenize,133            never_split=never_split,134            unk_token=unk_token,135            sep_token=sep_token,136            pad_token=pad_token,137            cls_token=cls_token,138            mask_token=mask_token,139            tokenize_chinese_chars=tokenize_chinese_chars,140            strip_accents=strip_accents,141            **kwargs,142        )143 144    @property145    def do_lower_case(self):146        return self.basic_tokenizer.do_lower_case147 148    @property149    def vocab_size(self):150        return len(self.vocab)151 152    def get_vocab(self):153        return dict(self.vocab, **self.added_tokens_encoder)154 155    def _tokenize(self, text):156        split_tokens = []157        if self.do_basic_tokenize:158            for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens):159                # If the token is part of the never_split set160                if token in self.basic_tokenizer.never_split:161                    split_tokens.append(token)162                else:163                    split_tokens += self.wordpiece_tokenizer.tokenize(token)164        else:165            split_tokens = self.wordpiece_tokenizer.tokenize(text)166        return split_tokens167 168    def _convert_token_to_id(self, token):169        """Converts a token (str) in an id using the vocab."""170        return self.vocab.get(token, self.vocab.get(self.unk_token))171 172    def _convert_id_to_token(self, index):173        """Converts an index (integer) in a token (str) using the vocab."""174        return self.ids_to_tokens.get(index, self.unk_token)175 176    def convert_tokens_to_string(self, tokens):177        """Converts a sequence of tokens (string) in a single string."""178        out_string = " ".join(tokens).replace(" ##", "").strip()179        return out_string180 181    def batch_encode_candidates(self, text, **kwargs):182        r"""183        Encode a batch of text or text pair. This method is similar to regular __call__ method but has the following184        differences:185 186            1. Handle additional num_candidate axis. (batch_size, num_candidates, text)187            2. Always pad the sequences to *max_length*.188            3. Must specify *max_length* in order to stack packs of candidates into a batch.189 190            - single sequence: `[CLS] X [SEP]`191            - pair of sequences: `[CLS] A [SEP] B [SEP]`192 193        Args:194            text (`List[List[str]]`):195                The batch of sequences to be encoded. Each sequence must be in this format: (batch_size,196                num_candidates, text).197            text_pair (`List[List[str]]`, *optional*):198                The batch of sequences to be encoded. Each sequence must be in this format: (batch_size,199                num_candidates, text).200            **kwargs:201                Keyword arguments of the __call__ method.202 203        Returns:204            [`BatchEncoding`]: Encoded text or text pair.205 206        Example:207 208        ```python209        >>> from transformers import RealmTokenizer210 211        >>> # batch_size = 2, num_candidates = 2212        >>> text = [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]213 214        >>> tokenizer = RealmTokenizer.from_pretrained("google/realm-cc-news-pretrained-encoder")215        >>> tokenized_text = tokenizer.batch_encode_candidates(text, max_length=10, return_tensors="pt")216        ```"""217 218        # Always using a fixed sequence length to encode in order to stack candidates into a batch.219        kwargs["padding"] = PaddingStrategy.MAX_LENGTH220 221        batch_text = text222        batch_text_pair = kwargs.pop("text_pair", None)223        return_tensors = kwargs.pop("return_tensors", None)224 225        output_data = {226            "input_ids": [],227            "attention_mask": [],228            "token_type_ids": [],229        }230 231        for idx, candidate_text in enumerate(batch_text):232            if batch_text_pair is not None:233                candidate_text_pair = batch_text_pair[idx]234            else:235                candidate_text_pair = None236 237            encoded_candidates = super().__call__(candidate_text, candidate_text_pair, return_tensors=None, **kwargs)238 239            encoded_input_ids = encoded_candidates.get("input_ids")240            encoded_attention_mask = encoded_candidates.get("attention_mask")241            encoded_token_type_ids = encoded_candidates.get("token_type_ids")242 243            if encoded_input_ids is not None:244                output_data["input_ids"].append(encoded_input_ids)245            if encoded_attention_mask is not None:246                output_data["attention_mask"].append(encoded_attention_mask)247            if encoded_token_type_ids is not None:248                output_data["token_type_ids"].append(encoded_token_type_ids)249 250        output_data = {key: item for key, item in output_data.items() if len(item) != 0}251 252        return BatchEncoding(output_data, tensor_type=return_tensors)253 254    def build_inputs_with_special_tokens(255        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None256    ) -> list[int]:257        """258        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and259        adding special tokens. A REALM sequence has the following format:260 261        - single sequence: `[CLS] X [SEP]`262        - pair of sequences: `[CLS] A [SEP] B [SEP]`263 264        Args:265            token_ids_0 (`List[int]`):266                List of IDs to which the special tokens will be added.267            token_ids_1 (`List[int]`, *optional*):268                Optional second list of IDs for sequence pairs.269 270        Returns:271            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.272        """273        if token_ids_1 is None:274            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]275        cls = [self.cls_token_id]276        sep = [self.sep_token_id]277        return cls + token_ids_0 + sep + token_ids_1 + sep278 279    def get_special_tokens_mask(280        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None, already_has_special_tokens: bool = False281    ) -> list[int]:282        """283        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding284        special tokens using the tokenizer `prepare_for_model` method.285 286        Args:287            token_ids_0 (`List[int]`):288                List of IDs.289            token_ids_1 (`List[int]`, *optional*):290                Optional second list of IDs for sequence pairs.291            already_has_special_tokens (`bool`, *optional*, defaults to `False`):292                Whether or not the token list is already formatted with special tokens for the model.293 294        Returns:295            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.296        """297 298        if already_has_special_tokens:299            return super().get_special_tokens_mask(300                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True301            )302 303        if token_ids_1 is not None:304            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]305        return [1] + ([0] * len(token_ids_0)) + [1]306 307    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:308        index = 0309        if os.path.isdir(save_directory):310            vocab_file = os.path.join(311                save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]312            )313        else:314            vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory315        with open(vocab_file, "w", encoding="utf-8") as writer:316            for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):317                if index != token_index:318                    logger.warning(319                        f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."320                        " Please check that the vocabulary is not corrupted!"321                    )322                    index = token_index323                writer.write(token + "\n")324                index += 1325        return (vocab_file,)326 327 328class BasicTokenizer:329    """330    Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).331 332    Args:333        do_lower_case (`bool`, *optional*, defaults to `True`):334            Whether or not to lowercase the input when tokenizing.335        never_split (`Iterable`, *optional*):336            Collection of tokens which will never be split during tokenization. Only has an effect when337            `do_basic_tokenize=True`338        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):339            Whether or not to tokenize Chinese characters.340 341            This should likely be deactivated for Japanese (see this342            [issue](https://github.com/huggingface/transformers/issues/328)).343        strip_accents (`bool`, *optional*):344            Whether or not to strip all accents. If this option is not specified, then it will be determined by the345            value for `lowercase` (as in the original BERT).346    """347 348    def __init__(self, do_lower_case=True, never_split=None, tokenize_chinese_chars=True, strip_accents=None):349        if never_split is None:350            never_split = []351        self.do_lower_case = do_lower_case352        self.never_split = set(never_split)353        self.tokenize_chinese_chars = tokenize_chinese_chars354        self.strip_accents = strip_accents355 356    def tokenize(self, text, never_split=None):357        """358        Basic Tokenization of a piece of text. Split on "white spaces" only, for sub-word tokenization, see359        WordPieceTokenizer.360 361        Args:362            never_split (`List[str]`, *optional*)363                Kept for backward compatibility purposes. Now implemented directly at the base class level (see364                [`PreTrainedTokenizer.tokenize`]) List of token not to split.365        """366        # union() returns a new set by concatenating the two sets.367        never_split = self.never_split.union(set(never_split)) if never_split else self.never_split368        text = self._clean_text(text)369 370        # This was added on November 1st, 2018 for the multilingual and Chinese371        # models. This is also applied to the English models now, but it doesn't372        # matter since the English models were not trained on any Chinese data373        # and generally don't have any Chinese data in them (there are Chinese374        # characters in the vocabulary because Wikipedia does have some Chinese375        # words in the English Wikipedia.).376        if self.tokenize_chinese_chars:377            text = self._tokenize_chinese_chars(text)378        orig_tokens = whitespace_tokenize(text)379        split_tokens = []380        for token in orig_tokens:381            if token not in never_split:382                if self.do_lower_case:383                    token = token.lower()384                    if self.strip_accents is not False:385                        token = self._run_strip_accents(token)386                elif self.strip_accents:387                    token = self._run_strip_accents(token)388            split_tokens.extend(self._run_split_on_punc(token, never_split))389 390        output_tokens = whitespace_tokenize(" ".join(split_tokens))391        return output_tokens392 393    def _run_strip_accents(self, text):394        """Strips accents from a piece of text."""395        text = unicodedata.normalize("NFD", text)396        output = []397        for char in text:398            cat = unicodedata.category(char)399            if cat == "Mn":400                continue401            output.append(char)402        return "".join(output)403 404    def _run_split_on_punc(self, text, never_split=None):405        """Splits punctuation on a piece of text."""406        if never_split is not None and text in never_split:407            return [text]408        chars = list(text)409        i = 0410        start_new_word = True411        output = []412        while i < len(chars):413            char = chars[i]414            if _is_punctuation(char):415                output.append([char])416                start_new_word = True417            else:418                if start_new_word:419                    output.append([])420                start_new_word = False421                output[-1].append(char)422            i += 1423 424        return ["".join(x) for x in output]425 426    def _tokenize_chinese_chars(self, text):427        """Adds whitespace around any CJK character."""428        output = []429        for char in text:430            cp = ord(char)431            if self._is_chinese_char(cp):432                output.append(" ")433                output.append(char)434                output.append(" ")435            else:436                output.append(char)437        return "".join(output)438 439    def _is_chinese_char(self, cp):440        """Checks whether CP is the codepoint of a CJK character."""441        # This defines a "chinese character" as anything in the CJK Unicode block:442        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)443        #444        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,445        # despite its name. The modern Korean Hangul alphabet is a different block,446        # as is Japanese Hiragana and Katakana. Those alphabets are used to write447        # space-separated words, so they are not treated specially and handled448        # like the all of the other languages.449        if (450            (cp >= 0x4E00 and cp <= 0x9FFF)451            or (cp >= 0x3400 and cp <= 0x4DBF)452            or (cp >= 0x20000 and cp <= 0x2A6DF)453            or (cp >= 0x2A700 and cp <= 0x2B73F)454            or (cp >= 0x2B740 and cp <= 0x2B81F)455            or (cp >= 0x2B820 and cp <= 0x2CEAF)456            or (cp >= 0xF900 and cp <= 0xFAFF)457            or (cp >= 0x2F800 and cp <= 0x2FA1F)458        ):459            return True460 461        return False462 463    def _clean_text(self, text):464        """Performs invalid character removal and whitespace cleanup on text."""465        output = []466        for char in text:467            cp = ord(char)468            if cp == 0 or cp == 0xFFFD or _is_control(char):469                continue470            if _is_whitespace(char):471                output.append(" ")472            else:473                output.append(char)474        return "".join(output)475 476 477class WordpieceTokenizer:478    """Runs WordPiece tokenization."""479 480    def __init__(self, vocab, unk_token, max_input_chars_per_word=100):481        self.vocab = vocab482        self.unk_token = unk_token483        self.max_input_chars_per_word = max_input_chars_per_word484 485    def tokenize(self, text):486        """487        Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform488        tokenization using the given vocabulary.489 490        For example, `input = "unaffable"` will return as output `["un", "##aff", "##able"]`.491 492        Args:493            text: A single token or whitespace separated tokens. This should have494                already been passed through *BasicTokenizer*.495 496        Returns:497            A list of wordpiece tokens.498        """499 500        output_tokens = []501        for token in whitespace_tokenize(text):502            chars = list(token)503            if len(chars) > self.max_input_chars_per_word:504                output_tokens.append(self.unk_token)505                continue506 507            is_bad = False508            start = 0509            sub_tokens = []510            while start < len(chars):511                end = len(chars)512                cur_substr = None513                while start < end:514                    substr = "".join(chars[start:end])515                    if start > 0:516                        substr = "##" + substr517                    if substr in self.vocab:518                        cur_substr = substr519                        break520                    end -= 1521                if cur_substr is None:522                    is_bad = True523                    break524                sub_tokens.append(cur_substr)525                start = end526 527            if is_bad:528                output_tokens.append(self.unk_token)529            else:530                output_tokens.extend(sub_tokens)531        return output_tokens532 533 534__all__ = ["RealmTokenizer"]535