Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2020 The Google AI Language Team Authors, Allegro.pl, Facebook Inc. and the HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15import json16import os17import re18import unicodedata19from typing import Optional20 21from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace22from ...utils import logging23 24 25logger = logging.get_logger(__name__)26 27VOCAB_FILES_NAMES = {28    "vocab_file": "vocab.json",29    "merges_file": "merges.txt",30}31 32 33# Copied from transformers.models.xlm.tokenization_xlm.get_pairs34def get_pairs(word):35    """36    Return set of symbol pairs in a word. word is represented as tuple of symbols (symbols being variable-length37    strings)38    """39    pairs = set()40    prev_char = word[0]41    for char in word[1:]:42        pairs.add((prev_char, char))43        prev_char = char44    return pairs45 46 47# Copied from transformers.models.xlm.tokenization_xlm.replace_unicode_punct48def replace_unicode_punct(text):49    """50    Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl51    """52    text = text.replace("，", ",")53    text = re.sub(r"。\s*", ". ", text)54    text = text.replace("、", ",")55    text = text.replace("”", '"')56    text = text.replace("“", '"')57    text = text.replace("∶", ":")58    text = text.replace("：", ":")59    text = text.replace("？", "?")60    text = text.replace("《", '"')61    text = text.replace("》", '"')62    text = text.replace("）", ")")63    text = text.replace("！", "!")64    text = text.replace("（", "(")65    text = text.replace("；", ";")66    text = text.replace("１", "1")67    text = text.replace("」", '"')68    text = text.replace("「", '"')69    text = text.replace("０", "0")70    text = text.replace("３", "3")71    text = text.replace("２", "2")72    text = text.replace("５", "5")73    text = text.replace("６", "6")74    text = text.replace("９", "9")75    text = text.replace("７", "7")76    text = text.replace("８", "8")77    text = text.replace("４", "4")78    text = re.sub(r"．\s*", ". ", text)79    text = text.replace("～", "~")80    text = text.replace("’", "'")81    text = text.replace("…", "...")82    text = text.replace("━", "-")83    text = text.replace("〈", "<")84    text = text.replace("〉", ">")85    text = text.replace("【", "[")86    text = text.replace("】", "]")87    text = text.replace("％", "%")88    return text89 90 91# Copied from transformers.models.xlm.tokenization_xlm.remove_non_printing_char92def remove_non_printing_char(text):93    """94    Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/remove-non-printing-char.perl95    """96    output = []97    for char in text:98        cat = unicodedata.category(char)99        if cat.startswith("C"):100            continue101        output.append(char)102    return "".join(output)103 104 105# Copied from transformers.models.bert.tokenization_bert.whitespace_tokenize106def whitespace_tokenize(text):107    """Runs basic whitespace cleaning and splitting on a piece of text."""108    text = text.strip()109    if not text:110        return []111    tokens = text.split()112    return tokens113 114 115# Copied from transformers.models.bert.tokenization_bert.BasicTokenizer116class BasicTokenizer:117    """118    Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).119 120    Args:121        do_lower_case (`bool`, *optional*, defaults to `True`):122            Whether or not to lowercase the input when tokenizing.123        never_split (`Iterable`, *optional*):124            Collection of tokens which will never be split during tokenization. Only has an effect when125            `do_basic_tokenize=True`126        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):127            Whether or not to tokenize Chinese characters.128 129            This should likely be deactivated for Japanese (see this130            [issue](https://github.com/huggingface/transformers/issues/328)).131        strip_accents (`bool`, *optional*):132            Whether or not to strip all accents. If this option is not specified, then it will be determined by the133            value for `lowercase` (as in the original BERT).134        do_split_on_punc (`bool`, *optional*, defaults to `True`):135            In some instances we want to skip the basic punctuation splitting so that later tokenization can capture136            the full context of the words, such as contractions.137    """138 139    def __init__(140        self,141        do_lower_case=True,142        never_split=None,143        tokenize_chinese_chars=True,144        strip_accents=None,145        do_split_on_punc=True,146    ):147        if never_split is None:148            never_split = []149        self.do_lower_case = do_lower_case150        self.never_split = set(never_split)151        self.tokenize_chinese_chars = tokenize_chinese_chars152        self.strip_accents = strip_accents153        self.do_split_on_punc = do_split_on_punc154 155    def tokenize(self, text, never_split=None):156        """157        Basic Tokenization of a piece of text. For sub-word tokenization, see WordPieceTokenizer.158 159        Args:160            never_split (`List[str]`, *optional*)161                Kept for backward compatibility purposes. Now implemented directly at the base class level (see162                [`PreTrainedTokenizer.tokenize`]) List of token not to split.163        """164        # union() returns a new set by concatenating the two sets.165        never_split = self.never_split.union(set(never_split)) if never_split else self.never_split166        text = self._clean_text(text)167 168        # This was added on November 1st, 2018 for the multilingual and Chinese169        # models. This is also applied to the English models now, but it doesn't170        # matter since the English models were not trained on any Chinese data171        # and generally don't have any Chinese data in them (there are Chinese172        # characters in the vocabulary because Wikipedia does have some Chinese173        # words in the English Wikipedia.).174        if self.tokenize_chinese_chars:175            text = self._tokenize_chinese_chars(text)176        # prevents treating the same character with different unicode codepoints as different characters177        unicode_normalized_text = unicodedata.normalize("NFC", text)178        orig_tokens = whitespace_tokenize(unicode_normalized_text)179        split_tokens = []180        for token in orig_tokens:181            if token not in never_split:182                if self.do_lower_case:183                    token = token.lower()184                    if self.strip_accents is not False:185                        token = self._run_strip_accents(token)186                elif self.strip_accents:187                    token = self._run_strip_accents(token)188            split_tokens.extend(self._run_split_on_punc(token, never_split))189 190        output_tokens = whitespace_tokenize(" ".join(split_tokens))191        return output_tokens192 193    def _run_strip_accents(self, text):194        """Strips accents from a piece of text."""195        text = unicodedata.normalize("NFD", text)196        output = []197        for char in text:198            cat = unicodedata.category(char)199            if cat == "Mn":200                continue201            output.append(char)202        return "".join(output)203 204    def _run_split_on_punc(self, text, never_split=None):205        """Splits punctuation on a piece of text."""206        if not self.do_split_on_punc or (never_split is not None and text in never_split):207            return [text]208        chars = list(text)209        i = 0210        start_new_word = True211        output = []212        while i < len(chars):213            char = chars[i]214            if _is_punctuation(char):215                output.append([char])216                start_new_word = True217            else:218                if start_new_word:219                    output.append([])220                start_new_word = False221                output[-1].append(char)222            i += 1223 224        return ["".join(x) for x in output]225 226    def _tokenize_chinese_chars(self, text):227        """Adds whitespace around any CJK character."""228        output = []229        for char in text:230            cp = ord(char)231            if self._is_chinese_char(cp):232                output.append(" ")233                output.append(char)234                output.append(" ")235            else:236                output.append(char)237        return "".join(output)238 239    def _is_chinese_char(self, cp):240        """Checks whether CP is the codepoint of a CJK character."""241        # This defines a "chinese character" as anything in the CJK Unicode block:242        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)243        #244        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,245        # despite its name. The modern Korean Hangul alphabet is a different block,246        # as is Japanese Hiragana and Katakana. Those alphabets are used to write247        # space-separated words, so they are not treated specially and handled248        # like the all of the other languages.249        if (250            (cp >= 0x4E00 and cp <= 0x9FFF)251            or (cp >= 0x3400 and cp <= 0x4DBF)252            or (cp >= 0x20000 and cp <= 0x2A6DF)253            or (cp >= 0x2A700 and cp <= 0x2B73F)254            or (cp >= 0x2B740 and cp <= 0x2B81F)255            or (cp >= 0x2B820 and cp <= 0x2CEAF)256            or (cp >= 0xF900 and cp <= 0xFAFF)257            or (cp >= 0x2F800 and cp <= 0x2FA1F)258        ):259            return True260 261        return False262 263    def _clean_text(self, text):264        """Performs invalid character removal and whitespace cleanup on text."""265        output = []266        for char in text:267            cp = ord(char)268            if cp == 0 or cp == 0xFFFD or _is_control(char):269                continue270            if _is_whitespace(char):271                output.append(" ")272            else:273                output.append(char)274        return "".join(output)275 276 277class HerbertTokenizer(PreTrainedTokenizer):278    """279    Construct a BPE tokenizer for HerBERT.280 281    Peculiarities:282 283    - uses BERT's pre-tokenizer: BaseTokenizer splits tokens on spaces, and also on punctuation. Each occurrence of a284      punctuation character will be treated separately.285 286    - Such pretokenized input is BPE subtokenized287 288    This tokenizer inherits from [`XLMTokenizer`] which contains most of the methods. Users should refer to the289    superclass for more information regarding methods.290    """291 292    vocab_files_names = VOCAB_FILES_NAMES293 294    def __init__(295        self,296        vocab_file,297        merges_file,298        tokenizer_file=None,299        cls_token="<s>",300        unk_token="<unk>",301        pad_token="<pad>",302        mask_token="<mask>",303        sep_token="</s>",304        bos_token="<s>",305        do_lowercase_and_remove_accent=False,306        additional_special_tokens=[307            "<special0>",308            "<special1>",309            "<special2>",310            "<special3>",311            "<special4>",312            "<special5>",313            "<special6>",314            "<special7>",315            "<special8>",316            "<special9>",317        ],318        lang2id=None,319        id2lang=None,320        **kwargs,321    ):322        try:323            import sacremoses324        except ImportError:325            raise ImportError(326                "You need to install sacremoses to use HerbertTokenizer. "327                "See https://pypi.org/project/sacremoses/ for installation."328            )329 330        self.sm = sacremoses331 332        # cache of sm.MosesPunctNormalizer instance333        self.cache_moses_punct_normalizer = {}334        # cache of sm.MosesTokenizer instance335        self.cache_moses_tokenizer = {}336        self.lang_with_custom_tokenizer = {"zh", "th", "ja"}337        # True for current supported model (v1.2.0), False for XLM-17 & 100338        self.do_lowercase_and_remove_accent = do_lowercase_and_remove_accent339        self.lang2id = lang2id340        self.id2lang = id2lang341        if lang2id is not None and id2lang is not None:342            assert len(lang2id) == len(id2lang)343 344        self.ja_word_tokenizer = None345        self.zh_word_tokenizer = None346 347        with open(vocab_file, encoding="utf-8") as vocab_handle:348            self.encoder = json.load(vocab_handle)349        self.decoder = {v: k for k, v in self.encoder.items()}350        with open(merges_file, encoding="utf-8") as merges_handle:351            merges = merges_handle.read().split("\n")[:-1]352        merges = [tuple(merge.split()[:2]) for merge in merges]353        self.bpe_ranks = dict(zip(merges, range(len(merges))))354        self.cache = {}355 356        super().__init__(357            unk_token=unk_token,358            bos_token=bos_token,359            sep_token=sep_token,360            pad_token=pad_token,361            cls_token=cls_token,362            mask_token=mask_token,363            additional_special_tokens=additional_special_tokens,364            lang2id=lang2id,365            id2lang=id2lang,366            do_lowercase_and_remove_accent=do_lowercase_and_remove_accent,367            tokenizer_file=None,368            **kwargs,369        )370 371        self.bert_pre_tokenizer = BasicTokenizer(372            do_lower_case=False,373            never_split=self.all_special_tokens,374            tokenize_chinese_chars=False,375            strip_accents=False,376        )377 378    @property379    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.do_lower_case380    def do_lower_case(self):381        return self.do_lowercase_and_remove_accent382 383    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.moses_punct_norm384    def moses_punct_norm(self, text, lang):385        if lang not in self.cache_moses_punct_normalizer:386            punct_normalizer = self.sm.MosesPunctNormalizer(lang=lang)387            self.cache_moses_punct_normalizer[lang] = punct_normalizer388        else:389            punct_normalizer = self.cache_moses_punct_normalizer[lang]390        return punct_normalizer.normalize(text)391 392    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.moses_tokenize393    def moses_tokenize(self, text, lang):394        if lang not in self.cache_moses_tokenizer:395            moses_tokenizer = self.sm.MosesTokenizer(lang=lang)396            self.cache_moses_tokenizer[lang] = moses_tokenizer397        else:398            moses_tokenizer = self.cache_moses_tokenizer[lang]399        return moses_tokenizer.tokenize(text, return_str=False, escape=False)400 401    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.moses_pipeline402    def moses_pipeline(self, text, lang):403        text = replace_unicode_punct(text)404        text = self.moses_punct_norm(text, lang)405        text = remove_non_printing_char(text)406        return text407 408    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.ja_tokenize409    def ja_tokenize(self, text):410        if self.ja_word_tokenizer is None:411            try:412                import Mykytea413 414                self.ja_word_tokenizer = Mykytea.Mykytea(415                    f"-model {os.path.expanduser('~')}/local/share/kytea/model.bin"416                )417            except (AttributeError, ImportError):418                logger.error(419                    "Make sure you install KyTea (https://github.com/neubig/kytea) and it's python wrapper"420                    " (https://github.com/chezou/Mykytea-python) with the following steps"421                )422                logger.error("1. git clone git@github.com:neubig/kytea.git && cd kytea")423                logger.error("2. autoreconf -i")424                logger.error("3. ./configure --prefix=$HOME/local")425                logger.error("4. make && make install")426                logger.error("5. pip install kytea")427                raise428        return list(self.ja_word_tokenizer.getWS(text))429 430    @property431    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.vocab_size432    def vocab_size(self):433        return len(self.encoder)434 435    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.get_vocab436    def get_vocab(self):437        return dict(self.encoder, **self.added_tokens_encoder)438 439    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.bpe440    def bpe(self, token):441        word = tuple(token[:-1]) + (token[-1] + "</w>",)442        if token in self.cache:443            return self.cache[token]444        pairs = get_pairs(word)445 446        if not pairs:447            return token + "</w>"448 449        while True:450            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))451            if bigram not in self.bpe_ranks:452                break453            first, second = bigram454            new_word = []455            i = 0456            while i < len(word):457                try:458                    j = word.index(first, i)459                except ValueError:460                    new_word.extend(word[i:])461                    break462                else:463                    new_word.extend(word[i:j])464                    i = j465 466                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:467                    new_word.append(first + second)468                    i += 2469                else:470                    new_word.append(word[i])471                    i += 1472            new_word = tuple(new_word)473            word = new_word474            if len(word) == 1:475                break476            else:477                pairs = get_pairs(word)478        word = " ".join(word)479        if word == "\n  </w>":480            word = "\n</w>"481        self.cache[token] = word482        return word483 484    def _tokenize(self, text):485        pre_tokens = self.bert_pre_tokenizer.tokenize(text)486 487        split_tokens = []488        for token in pre_tokens:489            if token:490                split_tokens.extend(list(self.bpe(token).split(" ")))491 492        return split_tokens493 494    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer._convert_token_to_id495    def _convert_token_to_id(self, token):496        """Converts a token (str) in an id using the vocab."""497        return self.encoder.get(token, self.encoder.get(self.unk_token))498 499    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer._convert_id_to_token500    def _convert_id_to_token(self, index):501        """Converts an index (integer) in a token (str) using the vocab."""502        return self.decoder.get(index, self.unk_token)503 504    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.convert_tokens_to_string505    def convert_tokens_to_string(self, tokens):506        """Converts a sequence of tokens (string) in a single string."""507        out_string = "".join(tokens).replace("</w>", " ").strip()508        return out_string509 510    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.build_inputs_with_special_tokens511    def build_inputs_with_special_tokens(512        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None513    ) -> list[int]:514        """515        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and516        adding special tokens. An XLM sequence has the following format:517 518        - single sequence: `<s> X </s>`519        - pair of sequences: `<s> A </s> B </s>`520 521        Args:522            token_ids_0 (`List[int]`):523                List of IDs to which the special tokens will be added.524            token_ids_1 (`List[int]`, *optional*):525                Optional second list of IDs for sequence pairs.526 527        Returns:528            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.529 530        """531        bos = [self.bos_token_id]532        sep = [self.sep_token_id]533 534        if token_ids_1 is None:535            return bos + token_ids_0 + sep536        return bos + token_ids_0 + sep + token_ids_1 + sep537 538    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.get_special_tokens_mask539    def get_special_tokens_mask(540        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None, already_has_special_tokens: bool = False541    ) -> list[int]:542        """543        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding544        special tokens using the tokenizer `prepare_for_model` method.545 546        Args:547            token_ids_0 (`List[int]`):548                List of IDs.549            token_ids_1 (`List[int]`, *optional*):550                Optional second list of IDs for sequence pairs.551            already_has_special_tokens (`bool`, *optional*, defaults to `False`):552                Whether or not the token list is already formatted with special tokens for the model.553 554        Returns:555            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.556        """557 558        if already_has_special_tokens:559            return super().get_special_tokens_mask(560                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True561            )562 563        if token_ids_1 is not None:564            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]565        return [1] + ([0] * len(token_ids_0)) + [1]566 567    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.save_vocabulary568    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:569        if not os.path.isdir(save_directory):570            logger.error(f"Vocabulary path ({save_directory}) should be a directory")571            return572        vocab_file = os.path.join(573            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]574        )575        merge_file = os.path.join(576            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]577        )578 579        with open(vocab_file, "w", encoding="utf-8") as f:580            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")581 582        index = 0583        with open(merge_file, "w", encoding="utf-8") as writer:584            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):585                if index != token_index:586                    logger.warning(587                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."588                        " Please check that the tokenizer is not corrupted!"589                    )590                    index = token_index591                writer.write(" ".join(bpe_tokens) + "\n")592                index += 1593 594        return vocab_file, merge_file595 596    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.__getstate__597    def __getstate__(self):598        state = self.__dict__.copy()599        state["sm"] = None600        return state601 602    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.__setstate__603    def __setstate__(self, d):604        self.__dict__ = d605 606        try:607            import sacremoses608        except ImportError:609            raise ImportError(610                "You need to install sacremoses to use XLMTokenizer. "611                "See https://pypi.org/project/sacremoses/ for installation."612            )613 614        self.sm = sacremoses615 616 617__all__ = ["HerbertTokenizer"]618