Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2019-present CNRS, Facebook Inc. and the HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Tokenization classes for Flaubert."""16 17import json18import os19import re20import unicodedata21from typing import Optional22 23from ...tokenization_utils import PreTrainedTokenizer24from ...utils import logging25 26 27logger = logging.get_logger(__name__)28 29VOCAB_FILES_NAMES = {30    "vocab_file": "vocab.json",31    "merges_file": "merges.txt",32}33 34 35def convert_to_unicode(text):36    """37    Converts `text` to Unicode (if it's not already), assuming UTF-8 input.38    """39 40    def ensure_text(s, encoding="utf-8", errors="strict"):41        if isinstance(s, bytes):42            return s.decode(encoding, errors)43        elif isinstance(s, str):44            return s45        else:46            raise TypeError(f"not expecting type '{type(s)}'")47 48    return ensure_text(text, encoding="utf-8", errors="ignore")49 50 51# Copied from transformers.models.xlm.tokenization_xlm.get_pairs52def get_pairs(word):53    """54    Return set of symbol pairs in a word. word is represented as tuple of symbols (symbols being variable-length55    strings)56    """57    pairs = set()58    prev_char = word[0]59    for char in word[1:]:60        pairs.add((prev_char, char))61        prev_char = char62    return pairs63 64 65# Copied from transformers.models.xlm.tokenization_xlm.replace_unicode_punct66def replace_unicode_punct(text):67    """68    Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl69    """70    text = text.replace("，", ",")71    text = re.sub(r"。\s*", ". ", text)72    text = text.replace("、", ",")73    text = text.replace("”", '"')74    text = text.replace("“", '"')75    text = text.replace("∶", ":")76    text = text.replace("：", ":")77    text = text.replace("？", "?")78    text = text.replace("《", '"')79    text = text.replace("》", '"')80    text = text.replace("）", ")")81    text = text.replace("！", "!")82    text = text.replace("（", "(")83    text = text.replace("；", ";")84    text = text.replace("１", "1")85    text = text.replace("」", '"')86    text = text.replace("「", '"')87    text = text.replace("０", "0")88    text = text.replace("３", "3")89    text = text.replace("２", "2")90    text = text.replace("５", "5")91    text = text.replace("６", "6")92    text = text.replace("９", "9")93    text = text.replace("７", "7")94    text = text.replace("８", "8")95    text = text.replace("４", "4")96    text = re.sub(r"．\s*", ". ", text)97    text = text.replace("～", "~")98    text = text.replace("’", "'")99    text = text.replace("…", "...")100    text = text.replace("━", "-")101    text = text.replace("〈", "<")102    text = text.replace("〉", ">")103    text = text.replace("【", "[")104    text = text.replace("】", "]")105    text = text.replace("％", "%")106    return text107 108 109# Copied from transformers.models.xlm.tokenization_xlm.remove_non_printing_char110def remove_non_printing_char(text):111    """112    Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/remove-non-printing-char.perl113    """114    output = []115    for char in text:116        cat = unicodedata.category(char)117        if cat.startswith("C"):118            continue119        output.append(char)120    return "".join(output)121 122 123class FlaubertTokenizer(PreTrainedTokenizer):124    """125    Construct a Flaubert tokenizer. Based on Byte-Pair Encoding. The tokenization process is the following:126 127    - Moses preprocessing and tokenization.128    - Normalizing all inputs text.129    - The arguments `special_tokens` and the function `set_special_tokens`, can be used to add additional symbols (like130      "__classify__") to a vocabulary.131    - The argument `do_lowercase` controls lower casing (automatically set for pretrained vocabularies).132 133    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to134    this superclass for more information regarding those methods.135 136    Args:137        vocab_file (`str`):138            Vocabulary file.139        merges_file (`str`):140            Merges file.141        do_lowercase (`bool`, *optional*, defaults to `False`):142            Controls lower casing.143        unk_token (`str`, *optional*, defaults to `"<unk>"`):144            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this145            token instead.146        bos_token (`str`, *optional*, defaults to `"<s>"`):147            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.148 149            <Tip>150 151            When building a sequence using special tokens, this is not the token that is used for the beginning of152            sequence. The token used is the `cls_token`.153 154            </Tip>155 156        sep_token (`str`, *optional*, defaults to `"</s>"`):157            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for158            sequence classification or for a text and a question for question answering. It is also used as the last159            token of a sequence built with special tokens.160        pad_token (`str`, *optional*, defaults to `"<pad>"`):161            The token used for padding, for example when batching sequences of different lengths.162        cls_token (`str`, *optional*, defaults to `"</s>"`):163            The classifier token which is used when doing sequence classification (classification of the whole sequence164            instead of per-token classification). It is the first token of the sequence when built with special tokens.165        mask_token (`str`, *optional*, defaults to `"<special1>"`):166            The token used for masking values. This is the token used when training this model with masked language167            modeling. This is the token which the model will try to predict.168        additional_special_tokens (`List[str]`, *optional*, defaults to `['<special0>', '<special1>', '<special2>', '<special3>', '<special4>', '<special5>', '<special6>', '<special7>', '<special8>', '<special9>']`):169            List of additional special tokens.170        lang2id (`Dict[str, int]`, *optional*):171            Dictionary mapping languages string identifiers to their IDs.172        id2lang (`Dict[int, str]`, *optional*):173            Dictionary mapping language IDs to their string identifiers.174    """175 176    vocab_files_names = VOCAB_FILES_NAMES177 178    def __init__(179        self,180        vocab_file,181        merges_file,182        do_lowercase=False,183        unk_token="<unk>",184        bos_token="<s>",185        sep_token="</s>",186        pad_token="<pad>",187        cls_token="</s>",188        mask_token="<special1>",189        additional_special_tokens=[190            "<special0>",191            "<special1>",192            "<special2>",193            "<special3>",194            "<special4>",195            "<special5>",196            "<special6>",197            "<special7>",198            "<special8>",199            "<special9>",200        ],201        lang2id=None,202        id2lang=None,203        **kwargs,204    ):205        do_lowercase_and_remove_accent = kwargs.pop("do_lowercase_and_remove_accent", None)206        if do_lowercase_and_remove_accent is not None:207            logger.warning(208                "`do_lowercase_and_remove_accent` is passed as a keyword argument, but this won't do anything."209                " `FlaubertTokenizer` will always set it to `False`."210            )211        # always `False`212        self.do_lowercase_and_remove_accent = False213 214        self.do_lowercase = do_lowercase215 216        try:217            import sacremoses218        except ImportError:219            raise ImportError(220                "You need to install sacremoses to use FlaubertTokenizer. "221                "See https://pypi.org/project/sacremoses/ for installation."222            )223 224        self.sm = sacremoses225 226        # cache of sm.MosesPunctNormalizer instance227        self.cache_moses_punct_normalizer = {}228        # cache of sm.MosesTokenizer instance229        self.cache_moses_tokenizer = {}230        self.lang_with_custom_tokenizer = {"zh", "th", "ja"}231        self.lang2id = lang2id232        self.id2lang = id2lang233        if lang2id is not None and id2lang is not None:234            assert len(lang2id) == len(id2lang)235 236        self.ja_word_tokenizer = None237        self.zh_word_tokenizer = None238 239        with open(vocab_file, encoding="utf-8") as vocab_handle:240            self.encoder = json.load(vocab_handle)241        self.decoder = {v: k for k, v in self.encoder.items()}242        with open(merges_file, encoding="utf-8") as merges_handle:243            merges = merges_handle.read().split("\n")[:-1]244        merges = [tuple(merge.split()[:2]) for merge in merges]245        self.bpe_ranks = dict(zip(merges, range(len(merges))))246        self.cache = {}247 248        super().__init__(249            do_lowercase=do_lowercase,250            unk_token=unk_token,251            bos_token=bos_token,252            sep_token=sep_token,253            pad_token=pad_token,254            cls_token=cls_token,255            mask_token=mask_token,256            additional_special_tokens=additional_special_tokens,257            lang2id=lang2id,258            id2lang=id2lang,259            **kwargs,260        )261 262    @property263    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.do_lower_case264    def do_lower_case(self):265        return self.do_lowercase_and_remove_accent266 267    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.moses_punct_norm268    def moses_punct_norm(self, text, lang):269        if lang not in self.cache_moses_punct_normalizer:270            punct_normalizer = self.sm.MosesPunctNormalizer(lang=lang)271            self.cache_moses_punct_normalizer[lang] = punct_normalizer272        else:273            punct_normalizer = self.cache_moses_punct_normalizer[lang]274        return punct_normalizer.normalize(text)275 276    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.moses_tokenize277    def moses_tokenize(self, text, lang):278        if lang not in self.cache_moses_tokenizer:279            moses_tokenizer = self.sm.MosesTokenizer(lang=lang)280            self.cache_moses_tokenizer[lang] = moses_tokenizer281        else:282            moses_tokenizer = self.cache_moses_tokenizer[lang]283        return moses_tokenizer.tokenize(text, return_str=False, escape=False)284 285    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.moses_pipeline286    def moses_pipeline(self, text, lang):287        text = replace_unicode_punct(text)288        text = self.moses_punct_norm(text, lang)289        text = remove_non_printing_char(text)290        return text291 292    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.ja_tokenize293    def ja_tokenize(self, text):294        if self.ja_word_tokenizer is None:295            try:296                import Mykytea297 298                self.ja_word_tokenizer = Mykytea.Mykytea(299                    f"-model {os.path.expanduser('~')}/local/share/kytea/model.bin"300                )301            except (AttributeError, ImportError):302                logger.error(303                    "Make sure you install KyTea (https://github.com/neubig/kytea) and it's python wrapper"304                    " (https://github.com/chezou/Mykytea-python) with the following steps"305                )306                logger.error("1. git clone git@github.com:neubig/kytea.git && cd kytea")307                logger.error("2. autoreconf -i")308                logger.error("3. ./configure --prefix=$HOME/local")309                logger.error("4. make && make install")310                logger.error("5. pip install kytea")311                raise312        return list(self.ja_word_tokenizer.getWS(text))313 314    @property315    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.vocab_size316    def vocab_size(self):317        return len(self.encoder)318 319    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.get_vocab320    def get_vocab(self):321        return dict(self.encoder, **self.added_tokens_encoder)322 323    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.bpe324    def bpe(self, token):325        word = tuple(token[:-1]) + (token[-1] + "</w>",)326        if token in self.cache:327            return self.cache[token]328        pairs = get_pairs(word)329 330        if not pairs:331            return token + "</w>"332 333        while True:334            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))335            if bigram not in self.bpe_ranks:336                break337            first, second = bigram338            new_word = []339            i = 0340            while i < len(word):341                try:342                    j = word.index(first, i)343                except ValueError:344                    new_word.extend(word[i:])345                    break346                else:347                    new_word.extend(word[i:j])348                    i = j349 350                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:351                    new_word.append(first + second)352                    i += 2353                else:354                    new_word.append(word[i])355                    i += 1356            new_word = tuple(new_word)357            word = new_word358            if len(word) == 1:359                break360            else:361                pairs = get_pairs(word)362        word = " ".join(word)363        if word == "\n  </w>":364            word = "\n</w>"365        self.cache[token] = word366        return word367 368    def preprocess_text(self, text):369        text = text.replace("``", '"').replace("''", '"')370        text = convert_to_unicode(text)371        text = unicodedata.normalize("NFC", text)372 373        if self.do_lowercase:374            text = text.lower()375 376        return text377 378    def _tokenize(self, text, bypass_tokenizer=False):379        """380        Tokenize a string given language code using Moses.381 382        Details of tokenization:383 384            - [sacremoses](https://github.com/alvations/sacremoses): port of Moses385            - Install with `pip install sacremoses`386 387        Args:388            - bypass_tokenizer: Allow users to preprocess and tokenize the sentences externally (default = False)389              (bool). If True, we only apply BPE.390 391        Returns:392            List of tokens.393        """394        lang = "fr"395        if lang and self.lang2id and lang not in self.lang2id:396            logger.error(397                "Supplied language code not found in lang2id mapping. Please check that your language is supported by"398                " the loaded pretrained model."399            )400 401        if bypass_tokenizer:402            text = text.split()403        else:404            text = self.preprocess_text(text)405            text = self.moses_pipeline(text, lang=lang)406            text = self.moses_tokenize(text, lang=lang)407 408        split_tokens = []409        for token in text:410            if token:411                split_tokens.extend(list(self.bpe(token).split(" ")))412 413        return split_tokens414 415    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer._convert_token_to_id416    def _convert_token_to_id(self, token):417        """Converts a token (str) in an id using the vocab."""418        return self.encoder.get(token, self.encoder.get(self.unk_token))419 420    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer._convert_id_to_token421    def _convert_id_to_token(self, index):422        """Converts an index (integer) in a token (str) using the vocab."""423        return self.decoder.get(index, self.unk_token)424 425    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.convert_tokens_to_string426    def convert_tokens_to_string(self, tokens):427        """Converts a sequence of tokens (string) in a single string."""428        out_string = "".join(tokens).replace("</w>", " ").strip()429        return out_string430 431    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.build_inputs_with_special_tokens432    def build_inputs_with_special_tokens(433        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None434    ) -> list[int]:435        """436        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and437        adding special tokens. An XLM sequence has the following format:438 439        - single sequence: `<s> X </s>`440        - pair of sequences: `<s> A </s> B </s>`441 442        Args:443            token_ids_0 (`List[int]`):444                List of IDs to which the special tokens will be added.445            token_ids_1 (`List[int]`, *optional*):446                Optional second list of IDs for sequence pairs.447 448        Returns:449            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.450 451        """452        bos = [self.bos_token_id]453        sep = [self.sep_token_id]454 455        if token_ids_1 is None:456            return bos + token_ids_0 + sep457        return bos + token_ids_0 + sep + token_ids_1 + sep458 459    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.get_special_tokens_mask460    def get_special_tokens_mask(461        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None, already_has_special_tokens: bool = False462    ) -> list[int]:463        """464        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding465        special tokens using the tokenizer `prepare_for_model` method.466 467        Args:468            token_ids_0 (`List[int]`):469                List of IDs.470            token_ids_1 (`List[int]`, *optional*):471                Optional second list of IDs for sequence pairs.472            already_has_special_tokens (`bool`, *optional*, defaults to `False`):473                Whether or not the token list is already formatted with special tokens for the model.474 475        Returns:476            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.477        """478 479        if already_has_special_tokens:480            return super().get_special_tokens_mask(481                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True482            )483 484        if token_ids_1 is not None:485            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]486        return [1] + ([0] * len(token_ids_0)) + [1]487 488    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.save_vocabulary489    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:490        if not os.path.isdir(save_directory):491            logger.error(f"Vocabulary path ({save_directory}) should be a directory")492            return493        vocab_file = os.path.join(494            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]495        )496        merge_file = os.path.join(497            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]498        )499 500        with open(vocab_file, "w", encoding="utf-8") as f:501            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")502 503        index = 0504        with open(merge_file, "w", encoding="utf-8") as writer:505            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):506                if index != token_index:507                    logger.warning(508                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."509                        " Please check that the tokenizer is not corrupted!"510                    )511                    index = token_index512                writer.write(" ".join(bpe_tokens) + "\n")513                index += 1514 515        return vocab_file, merge_file516 517    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.__getstate__518    def __getstate__(self):519        state = self.__dict__.copy()520        state["sm"] = None521        return state522 523    # Copied from transformers.models.xlm.tokenization_xlm.XLMTokenizer.__setstate__524    def __setstate__(self, d):525        self.__dict__ = d526 527        try:528            import sacremoses529        except ImportError:530            raise ImportError(531                "You need to install sacremoses to use XLMTokenizer. "532                "See https://pypi.org/project/sacremoses/ for installation."533            )534 535        self.sm = sacremoses536 537 538__all__ = ["FlaubertTokenizer"]539