Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2021 The Facebook Inc. and The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Tokenization class for Wav2Vec2."""16 17import json18import os19import warnings20from dataclasses import dataclass21from itertools import groupby22from typing import TYPE_CHECKING, Optional, Union23 24import numpy as np25 26from ...tokenization_utils import PreTrainedTokenizer27from ...tokenization_utils_base import AddedToken, BatchEncoding28from ...utils import (29    ModelOutput,30    PaddingStrategy,31    TensorType,32    add_end_docstrings,33    is_flax_available,34    is_tf_available,35    is_torch_available,36    logging,37    to_py_obj,38)39 40 41logger = logging.get_logger(__name__)42 43 44if TYPE_CHECKING:45    if is_torch_available():46        import torch47    if is_tf_available():48        import tensorflow as tf49    if is_flax_available():50        import jax.numpy as jnp  # noqa: F40151 52 53VOCAB_FILES_NAMES = {54    "vocab_file": "vocab.json",55    "tokenizer_config_file": "tokenizer_config.json",56}57 58 59# Wav2Vec2 has no max input length60 61WAV2VEC2_KWARGS_DOCSTRING = r"""62            padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):63                Activates and controls padding. Accepts the following values:64 65                - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single66                  sequence if provided).67                - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum68                  acceptable input length for the model if that argument is not provided.69                - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different70                  lengths).71            max_length (`int`, *optional*):72                Controls the maximum length to use by one of the truncation/padding parameters.73 74                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length75                is required by one of the truncation/padding parameters. If the model has no specific maximum input76                length (like XLNet) truncation/padding to a maximum length will be deactivated.77            pad_to_multiple_of (`int`, *optional*):78                If set will pad the sequence to a multiple of the provided value. This is especially useful to enable79                the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta).80            return_tensors (`str` or [`~utils.TensorType`], *optional*):81                If set, will return tensors instead of list of python integers. Acceptable values are:82 83                - `'tf'`: Return TensorFlow `tf.constant` objects.84                - `'pt'`: Return PyTorch `torch.Tensor` objects.85                - `'np'`: Return Numpy `np.ndarray` objects.86            verbose (`bool`, *optional*, defaults to `True`):87                Whether or not to print more information and warnings.88"""89 90ListOfDict = list[dict[str, Union[int, str]]]91 92 93@dataclass94class Wav2Vec2CTCTokenizerOutput(ModelOutput):95    """96    Output type of [` Wav2Vec2CTCTokenizer`], with transcription.97 98    Args:99        text (list of `str` or `str`):100            Decoded logits in text from. Usually the speech transcription.101        char_offsets (list of `list[dict[str, Union[int, str]]]` or `list[dict[str, Union[int, str]]]`):102            Offsets of the decoded characters. In combination with sampling rate and model downsampling rate char103            offsets can be used to compute time stamps for each character. Total logit score of the beam associated with104            produced text.105        word_offsets (list of `list[dict[str, Union[int, str]]]` or `list[dict[str, Union[int, str]]]`):106            Offsets of the decoded words. In combination with sampling rate and model downsampling rate word offsets107            can be used to compute time stamps for each word.108    """109 110    text: Union[list[str], str]111    char_offsets: Union[list[ListOfDict], ListOfDict] = None112    word_offsets: Union[list[ListOfDict], ListOfDict] = None113 114 115class Wav2Vec2CTCTokenizer(PreTrainedTokenizer):116    """117    Constructs a Wav2Vec2CTC tokenizer.118 119    This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to120    the superclass for more information regarding such methods.121 122    Args:123        vocab_file (`str`):124            File containing the vocabulary.125        bos_token (`str`, *optional*, defaults to `"<s>"`):126            The beginning of sentence token.127        eos_token (`str`, *optional*, defaults to `"</s>"`):128            The end of sentence token.129        unk_token (`str`, *optional*, defaults to `"<unk>"`):130            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this131            token instead.132        pad_token (`str`, *optional*, defaults to `"<pad>"`):133            The token used for padding, for example when batching sequences of different lengths.134        word_delimiter_token (`str`, *optional*, defaults to `"|"`):135            The token used for defining the end of a word.136        do_lower_case (`bool`, *optional*, defaults to `False`):137            Whether or not to accept lowercase input and lowercase the output when decoding.138        target_lang (`str`, *optional*):139            A target language the tokenizer should set by default. `target_lang` has to be defined for multi-lingual,140            nested vocabulary such as [facebook/mms-1b-all](https://huggingface.co/facebook/mms-1b-all).141 142        **kwargs143            Additional keyword arguments passed along to [`PreTrainedTokenizer`]144    """145 146    vocab_files_names = VOCAB_FILES_NAMES147    model_input_names = ["input_ids", "attention_mask"]148 149    def __init__(150        self,151        vocab_file,152        bos_token="<s>",153        eos_token="</s>",154        unk_token="<unk>",155        pad_token="<pad>",156        word_delimiter_token="|",157        replace_word_delimiter_char=" ",158        do_lower_case=False,159        target_lang=None,160        **kwargs,161    ):162        self._word_delimiter_token = word_delimiter_token163 164        self.do_lower_case = do_lower_case165        self.replace_word_delimiter_char = replace_word_delimiter_char166        self.target_lang = target_lang167 168        with open(vocab_file, encoding="utf-8") as vocab_handle:169            self.vocab = json.load(vocab_handle)170 171        # if target lang is defined vocab must be a nested dict172        # with each target lang being one vocabulary173        if target_lang is not None:174            self.encoder = self.vocab[target_lang]175        else:176            self.encoder = self.vocab177 178        self.decoder = {v: k for k, v in self.encoder.items()}179 180        super().__init__(181            unk_token=unk_token,182            bos_token=bos_token,183            eos_token=eos_token,184            pad_token=pad_token,185            do_lower_case=do_lower_case,186            word_delimiter_token=word_delimiter_token,187            replace_word_delimiter_char=replace_word_delimiter_char,188            target_lang=target_lang,189            **kwargs,190        )191 192        # make sure that tokens made of several193        # characters are not split at tokenization194        for token in self.encoder:195            if len(token) > 1:196                self.add_tokens(AddedToken(token, rstrip=True, lstrip=True, normalized=False))197 198    def set_target_lang(self, target_lang: str):199        """200        Set the target language of a nested multi-lingual dictionary201        """202        if self.vocab == self.encoder:203            raise ValueError(f"{self.vocab} is not a multi-lingual, nested tokenizer. Cannot set target language.")204 205        if target_lang not in self.vocab:206            raise ValueError(f"{target_lang} does not exist. Choose one of {', '.join(self.vocab.keys())}.")207 208        self.target_lang = target_lang209        self.init_kwargs["target_lang"] = target_lang210        self.encoder = self.vocab[target_lang]211        self.decoder = {v: k for k, v in self.encoder.items()}212 213        # make sure that tokens made of several214        # characters are not split at tokenization215        for token in self.encoder:216            if len(token) > 1:217                self.add_tokens(AddedToken(token, rstrip=True, lstrip=True, normalized=False))218 219    @property220    def word_delimiter_token(self) -> str:221        """222        `str`: Word delimiter token. Log an error if used while not having been set.223        """224        if self._word_delimiter_token is None and self.verbose:225            logger.error("Using word_delimiter_token, but it is not set yet.")226            return None227        return str(self._word_delimiter_token)228 229    @property230    def word_delimiter_token_id(self) -> Optional[int]:231        """232        `Optional[int]`: Id of the word_delimiter_token in the vocabulary. Returns `None` if the token has not been233        set.234        """235        if self._word_delimiter_token is None:236            return None237        return self.convert_tokens_to_ids(self.word_delimiter_token)238 239    @word_delimiter_token.setter240    def word_delimiter_token(self, value):241        self._word_delimiter_token = value242 243    @word_delimiter_token_id.setter244    def word_delimiter_token_id(self, value):245        self._word_delimiter_token = self.convert_tokens_to_ids(value)246 247    @property248    def vocab_size(self) -> int:249        return len(self.decoder)250 251    def get_vocab(self) -> dict:252        vocab = dict(self.encoder)253        vocab.update(self.added_tokens_encoder)254        return vocab255 256    def _add_tokens(self, new_tokens: Union[list[str], list[AddedToken]], special_tokens: bool = False) -> int:257        # Overwritten to never strip!258        to_add = []259        for token in new_tokens:260            if isinstance(token, str):261                to_add.append(AddedToken(token, rstrip=False, lstrip=False, normalized=False))262            else:263                to_add.append(token)264 265        return super()._add_tokens(to_add, special_tokens)266 267    def _tokenize(self, text, **kwargs):268        """269        Converts a string into a sequence of tokens (string), using the tokenizer.270        """271        if self.do_lower_case:272            text = text.upper()273 274        return list(text.replace(" ", self.word_delimiter_token))275 276    def _convert_token_to_id(self, token: str) -> int:277        """Converts a token (str) in an index (integer) using the vocab."""278        return self.encoder.get(token, self.encoder.get(self.unk_token))279 280    def _convert_id_to_token(self, index: int) -> str:281        """Converts an index (integer) in a token (str) using the vocab."""282        result = self.decoder.get(index, self.unk_token)283        return result284 285    def convert_tokens_to_string(286        self,287        tokens: list[str],288        group_tokens: bool = True,289        spaces_between_special_tokens: bool = False,290        output_char_offsets: bool = False,291        output_word_offsets: bool = False,292    ) -> dict[str, Union[str, float]]:293        """294        Converts a connectionist-temporal-classification (CTC) output tokens into a single string.295        """296        if len(tokens) == 0:297            return {"text": "", "char_offsets": [], "word_offsets": []}298        # group same tokens into non-repeating tokens in CTC style decoding299        if group_tokens:300            chars, char_repetitions = zip(*((token, len(list(group_iter))) for token, group_iter in groupby(tokens)))301        else:302            chars = tokens303            char_repetitions = len(tokens) * [1]304 305        # filter self.pad_token which is used as CTC-blank token306        processed_chars = list(filter(lambda char: char != self.pad_token, chars))307 308        # replace delimiter token309        processed_chars = [310            self.replace_word_delimiter_char if char == self.word_delimiter_token else char for char in processed_chars311        ]312 313        # retrieve offsets314        char_offsets = word_offsets = None315        if output_char_offsets or output_word_offsets:316            char_offsets = self._compute_offsets(char_repetitions, chars, self.pad_token)317 318            if len(char_offsets) != len(processed_chars):319                raise ValueError(320                    f"`char_offsets`: {char_offsets} and `processed_tokens`: {processed_chars}"321                    " have to be of the same length, but are: "322                    f"`len(offsets)`: {len(char_offsets)} and `len(processed_tokens)`:"323                    f" {len(processed_chars)}"324                )325 326            # set tokens to correct processed token327            for i, char in enumerate(processed_chars):328                char_offsets[i]["char"] = char329 330            # retrieve word offsets from character offsets331            word_offsets = None332            if output_word_offsets:333                word_offsets = self._get_word_offsets(char_offsets, self.replace_word_delimiter_char)334 335            # don't output chars if not set to True336            if not output_char_offsets:337                char_offsets = None338 339        # join to string340        join_char = " " if spaces_between_special_tokens else ""341        string = join_char.join(processed_chars).strip()342 343        if self.do_lower_case:344            string = string.lower()345 346        return {"text": string, "char_offsets": char_offsets, "word_offsets": word_offsets}347 348    @staticmethod349    def _compute_offsets(350        char_repetitions: list[int], chars: list[str], ctc_token: int351    ) -> list[dict[str, Union[str, int]]]:352        end_indices = np.asarray(char_repetitions).cumsum()353        start_indices = np.concatenate(([0], end_indices[:-1]))354 355        offsets = [356            {"char": t, "start_offset": s, "end_offset": e} for t, s, e in zip(chars, start_indices, end_indices)357        ]358 359        # filter out CTC token360        offsets = list(filter(lambda offsets: offsets["char"] != ctc_token, offsets))361        return offsets362 363    @staticmethod364    def _get_word_offsets(365        offsets: dict[str, Union[str, float]], word_delimiter_char: str = " "366    ) -> dict[str, Union[str, float]]:367        word_offsets = []368 369        last_state = "SPACE"370        word = ""371        start_offset = 0372        end_offset = 0373        for i, offset in enumerate(offsets):374            char = offset["char"]375            state = "SPACE" if char == word_delimiter_char else "WORD"376 377            if state == last_state:378                # If we are in the same state as before, we simply repeat what we've done before379                end_offset = offset["end_offset"]380                word += char381            else:382                # Switching state383                if state == "SPACE":384                    # Finishing a word385                    word_offsets.append({"word": word, "start_offset": start_offset, "end_offset": end_offset})386                else:387                    # Starting a new word388                    start_offset = offset["start_offset"]389                    end_offset = offset["end_offset"]390                    word = char391 392            last_state = state393        if last_state == "WORD":394            word_offsets.append({"word": word, "start_offset": start_offset, "end_offset": end_offset})395 396        return word_offsets397 398    def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs):399        if is_split_into_words:400            text = " " + text401        return (text, kwargs)402 403    def _decode(404        self,405        token_ids: list[int],406        skip_special_tokens: bool = False,407        clean_up_tokenization_spaces: Optional[bool] = None,408        group_tokens: bool = True,409        spaces_between_special_tokens: bool = False,410        output_word_offsets: Optional[bool] = False,411        output_char_offsets: Optional[bool] = False,412    ) -> str:413        """414        special _decode function is needed for Wav2Vec2Tokenizer because added tokens should be treated exactly the415        same as tokens of the base vocabulary and therefore the function `convert_tokens_to_string` has to be called on416        the whole token list and not individually on added tokens417        """418        filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)419 420        result = []421        for token in filtered_tokens:422            if skip_special_tokens and (423                token in self.all_special_ids or (token != self.pad_token and token in self.all_special_tokens)424            ):425                continue426            result.append(token)427 428        string_output = self.convert_tokens_to_string(429            result,430            group_tokens=group_tokens,431            spaces_between_special_tokens=spaces_between_special_tokens,432            output_word_offsets=output_word_offsets,433            output_char_offsets=output_char_offsets,434        )435 436        text = string_output["text"]437 438        clean_up_tokenization_spaces = (439            clean_up_tokenization_spaces440            if clean_up_tokenization_spaces is not None441            else self.clean_up_tokenization_spaces442        )443        if clean_up_tokenization_spaces:444            text = self.clean_up_tokenization(text)445 446        if output_word_offsets or output_char_offsets:447            return Wav2Vec2CTCTokenizerOutput(448                text=text,449                char_offsets=string_output["char_offsets"],450                word_offsets=string_output["word_offsets"],451            )452        else:453            return text454 455    # overwritten from `tokenization_utils_base.py` because tokenizer can output456    # `ModelOutput` which should not be a list for batched output and457    # because we need docs for `output_char_offsets` here458    def batch_decode(459        self,460        sequences: Union[list[int], list[list[int]], "np.ndarray", "torch.Tensor", "tf.Tensor"],461        skip_special_tokens: bool = False,462        clean_up_tokenization_spaces: Optional[bool] = None,463        output_char_offsets: bool = False,464        output_word_offsets: bool = False,465        **kwargs,466    ) -> list[str]:467        """468        Convert a list of lists of token ids into a list of strings by calling decode.469 470        Args:471            sequences (`Union[list[int], list[list[int]], np.ndarray, torch.Tensor, tf.Tensor]`):472                List of tokenized input ids. Can be obtained using the `__call__` method.473            skip_special_tokens (`bool`, *optional*, defaults to `False`):474                Whether or not to remove special tokens in the decoding.475            clean_up_tokenization_spaces (`bool`, *optional*):476                Whether or not to clean up the tokenization spaces.477            output_char_offsets (`bool`, *optional*, defaults to `False`):478                Whether or not to output character offsets. Character offsets can be used in combination with the479                sampling rate and model downsampling rate to compute the time-stamps of transcribed characters.480 481                <Tip>482 483                Please take a look at the Example of [`~Wav2Vec2CTCTokenizer.decode`] to better understand how to make484                use of `output_char_offsets`. [`~Wav2Vec2CTCTokenizer.batch_decode`] works the same way with batched485                output.486 487                </Tip>488 489            output_word_offsets (`bool`, *optional*, defaults to `False`):490                Whether or not to output word offsets. Word offsets can be used in combination with the sampling rate491                and model downsampling rate to compute the time-stamps of transcribed words.492 493                <Tip>494 495                Please take a look at the Example of [`~Wav2Vec2CTCTokenizer.decode`] to better understand how to make496                use of `output_word_offsets`. [`~Wav2Vec2CTCTokenizer.batch_decode`] works the same way with batched497                output.498 499                </Tip>500 501            kwargs (additional keyword arguments, *optional*):502                Will be passed to the underlying model specific decode method.503 504        Returns:505            `list[str]` or [`~models.wav2vec2.tokenization_wav2vec2.Wav2Vec2CTCTokenizerOutput`]: The list of decoded506            sentences. Will be a [`~models.wav2vec2.tokenization_wav2vec2.Wav2Vec2CTCTokenizerOutput`] when507            `output_char_offsets == True` or `output_word_offsets == True`.508        """509        batch_decoded = [510            self.decode(511                seq,512                skip_special_tokens=skip_special_tokens,513                clean_up_tokenization_spaces=clean_up_tokenization_spaces,514                output_char_offsets=output_char_offsets,515                output_word_offsets=output_word_offsets,516                **kwargs,517            )518            for seq in sequences519        ]520        if output_char_offsets or output_word_offsets:521            # transform list of dicts to dict of lists522            return Wav2Vec2CTCTokenizerOutput({k: [d[k] for d in batch_decoded] for k in batch_decoded[0]})523 524        return batch_decoded525 526    # overwritten from `tokenization_utils_base.py` because we need docs for `output_char_offsets`527    # and `output_word_offsets` here528    def decode(529        self,530        token_ids: Union[int, list[int], "np.ndarray", "torch.Tensor", "tf.Tensor"],531        skip_special_tokens: bool = False,532        clean_up_tokenization_spaces: Optional[bool] = None,533        output_char_offsets: bool = False,534        output_word_offsets: bool = False,535        **kwargs,536    ) -> str:537        """538        Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special539        tokens and clean up tokenization spaces.540 541        Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.542 543        Args:544            token_ids (`Union[int, list[int], np.ndarray, torch.Tensor, tf.Tensor]`):545                List of tokenized input ids. Can be obtained using the `__call__` method.546            skip_special_tokens (`bool`, *optional*, defaults to `False`):547                Whether or not to remove special tokens in the decoding.548            clean_up_tokenization_spaces (`bool`, *optional*):549                Whether or not to clean up the tokenization spaces.550            output_char_offsets (`bool`, *optional*, defaults to `False`):551                Whether or not to output character offsets. Character offsets can be used in combination with the552                sampling rate and model downsampling rate to compute the time-stamps of transcribed characters.553 554                <Tip>555 556                Please take a look at the example below to better understand how to make use of `output_char_offsets`.557 558                </Tip>559 560            output_word_offsets (`bool`, *optional*, defaults to `False`):561                Whether or not to output word offsets. Word offsets can be used in combination with the sampling rate562                and model downsampling rate to compute the time-stamps of transcribed words.563 564                <Tip>565 566                Please take a look at the example below to better understand how to make use of `output_word_offsets`.567 568                </Tip>569 570            kwargs (additional keyword arguments, *optional*):571                Will be passed to the underlying model specific decode method.572 573        Returns:574            `str` or [`~models.wav2vec2.tokenization_wav2vec2.Wav2Vec2CTCTokenizerOutput`]: The list of decoded575            sentences. Will be a [`~models.wav2vec2.tokenization_wav2vec2.Wav2Vec2CTCTokenizerOutput`] when576            `output_char_offsets == True` or `output_word_offsets == True`.577 578        Example:579 580        ```python581        >>> # Let's see how to retrieve time steps for a model582        >>> from transformers import AutoTokenizer, AutoFeatureExtractor, AutoModelForCTC583        >>> from datasets import load_dataset584        >>> import datasets585        >>> import torch586 587        >>> # import model, feature extractor, tokenizer588        >>> model = AutoModelForCTC.from_pretrained("facebook/wav2vec2-base-960h")589        >>> tokenizer = AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h")590        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h")591 592        >>> # load first sample of English common_voice593        >>> dataset = load_dataset("mozilla-foundation/common_voice_11_0", "en", split="train", streaming=True)594        >>> dataset = dataset.cast_column("audio", datasets.Audio(sampling_rate=16_000))595        >>> dataset_iter = iter(dataset)596        >>> sample = next(dataset_iter)597 598        >>> # forward sample through model to get greedily predicted transcription ids599        >>> input_values = feature_extractor(sample["audio"]["array"], return_tensors="pt").input_values600        >>> logits = model(input_values).logits[0]601        >>> pred_ids = torch.argmax(logits, axis=-1)602 603        >>> # retrieve word stamps (analogous commands for `output_char_offsets`)604        >>> outputs = tokenizer.decode(pred_ids, output_word_offsets=True)605        >>> # compute `time_offset` in seconds as product of downsampling ratio and sampling_rate606        >>> time_offset = model.config.inputs_to_logits_ratio / feature_extractor.sampling_rate607 608        >>> word_offsets = [609        ...     {610        ...         "word": d["word"],611        ...         "start_time": round(d["start_offset"] * time_offset, 2),612        ...         "end_time": round(d["end_offset"] * time_offset, 2),613        ...     }614        ...     for d in outputs.word_offsets615        ... ]616        >>> # compare word offsets with audio `en_train_0/common_voice_en_19121553.mp3` online on the dataset viewer:617        >>> # https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/viewer/en618        >>> word_offsets[:3]619        [{'word': 'THE', 'start_time': 0.7, 'end_time': 0.78}, {'word': 'TRICK', 'start_time': 0.88, 'end_time': 1.08}, {'word': 'APPEARS', 'start_time': 1.2, 'end_time': 1.64}]620        ```"""621        # Convert inputs to python lists622        token_ids = to_py_obj(token_ids)623 624        return self._decode(625            token_ids=token_ids,626            skip_special_tokens=skip_special_tokens,627            clean_up_tokenization_spaces=clean_up_tokenization_spaces,628            output_char_offsets=output_char_offsets,629            output_word_offsets=output_word_offsets,630            **kwargs,631        )632 633    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:634        if not os.path.isdir(save_directory):635            logger.error(f"Vocabulary path ({save_directory}) should be a directory")636            return637        vocab_file = os.path.join(638            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]639        )640 641        with open(vocab_file, "w", encoding="utf-8") as f:642            f.write(json.dumps(self.vocab, indent=2, sort_keys=True, ensure_ascii=False) + "\n")643 644        return (vocab_file,)645 646 647class Wav2Vec2Tokenizer(PreTrainedTokenizer):648    """649    Constructs a Wav2Vec2 tokenizer.650 651    This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to652    the superclass for more information regarding such methods.653 654    Args:655        vocab_file (`str`):656            File containing the vocabulary.657        bos_token (`str`, *optional*, defaults to `"<s>"`):658            The beginning of sentence token.659        eos_token (`str`, *optional*, defaults to `"</s>"`):660            The end of sentence token.661        unk_token (`str`, *optional*, defaults to `"<unk>"`):662            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this663            token instead.664        pad_token (`str`, *optional*, defaults to `"<pad>"`):665            The token used for padding, for example when batching sequences of different lengths.666        word_delimiter_token (`str`, *optional*, defaults to `"|"`):667            The token used for defining the end of a word.668        do_lower_case (`bool`, *optional*, defaults to `False`):669            Whether or not to lowercase the output when decoding.670        do_normalize (`bool`, *optional*, defaults to `False`):671            Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly672            improve the performance for some models, *e.g.*,673            [wav2vec2-lv60](https://huggingface.co/models?search=lv60).674        return_attention_mask (`bool`, *optional*, defaults to `False`):675            Whether or not [`~Wav2Vec2Tokenizer.__call__`] should return `attention_mask`.676 677            <Tip>678 679            Wav2Vec2 models that have set `config.feat_extract_norm == "group"`, such as680            [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), have **not** been trained using681            `attention_mask`. For such models, `input_values` should simply be padded with 0 and no `attention_mask`682            should be passed.683 684            For Wav2Vec2 models that have set `config.feat_extract_norm == "layer"`, such as685            [wav2vec2-lv60](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self), `attention_mask` should be686            passed for batched inference.687 688            </Tip>689 690        **kwargs691            Additional keyword arguments passed along to [`PreTrainedTokenizer`]692    """693 694    vocab_files_names = VOCAB_FILES_NAMES695    pretrained_vocab_files_map = {696        "vocab_file": {697            "facebook/wav2vec2-base-960h": "https://huggingface.co/facebook/wav2vec2-base-960h/resolve/main/vocab.json"698        },699        "tokenizer_config_file": {700            "facebook/wav2vec2-base-960h": (701                "https://huggingface.co/facebook/wav2vec2-base-960h/resolve/main/tokenizer.json"702            ),703        },704    }705    model_input_names = ["input_values", "attention_mask"]706 707    def __init__(708        self,709        vocab_file,710        bos_token="<s>",711        eos_token="</s>",712        unk_token="<unk>",713        pad_token="<pad>",714        word_delimiter_token="|",715        do_lower_case=False,716        do_normalize=False,717        return_attention_mask=False,718        **kwargs,719    ):720        warnings.warn(721            "The class `Wav2Vec2Tokenizer` is deprecated and will be removed in version 5 of Transformers. Please use"722            " `Wav2Vec2Processor` or `Wav2Vec2CTCTokenizer` instead.",723            FutureWarning,724        )725 726        self._word_delimiter_token = word_delimiter_token727 728        self.do_lower_case = do_lower_case729        self.return_attention_mask = return_attention_mask730        self.do_normalize = do_normalize731 732        with open(vocab_file, encoding="utf-8") as vocab_handle:733            self.encoder = json.load(vocab_handle)734 735        self.decoder = {v: k for k, v in self.encoder.items()}736 737        super().__init__(738            unk_token=unk_token,739            bos_token=bos_token,740            eos_token=eos_token,741            pad_token=pad_token,742            do_lower_case=do_lower_case,743            do_normalize=do_normalize,744            return_attention_mask=return_attention_mask,745            word_delimiter_token=word_delimiter_token,746            **kwargs,747        )748 749    @property750    def word_delimiter_token(self) -> str:751        """752        `str`: Padding token. Log an error if used while not having been set.753        """754        if self._word_delimiter_token is None and self.verbose:755            logger.error("Using word_delimiter_token, but it is not set yet.")756            return None757        return str(self._word_delimiter_token)758 759    @property760    def word_delimiter_token_id(self) -> Optional[int]:761        """762        `Optional[int]`: Id of the word_delimiter_token in the vocabulary. Returns `None` if the token has not been763        set.764        """765        if self._word_delimiter_token is None:766            return None767        return self.convert_tokens_to_ids(self.word_delimiter_token)768 769    @word_delimiter_token.setter770    def word_delimiter_token(self, value):771        self._word_delimiter_token = value772 773    @word_delimiter_token_id.setter774    def word_delimiter_token_id(self, value):775        self._word_delimiter_token = self.convert_tokens_to_ids(value)776 777    @add_end_docstrings(WAV2VEC2_KWARGS_DOCSTRING)778    def __call__(779        self,780        raw_speech: Union[np.ndarray, list[float], list[np.ndarray], list[list[float]]],781        padding: Union[bool, str, PaddingStrategy] = False,782        max_length: Optional[int] = None,783        pad_to_multiple_of: Optional[int] = None,784        padding_side: Optional[str] = None,785        return_tensors: Optional[Union[str, TensorType]] = None,786        verbose: bool = True,787        **kwargs,788    ) -> BatchEncoding:789        """790        Main method to tokenize and prepare for the model one or several sequence(s) or one or several pair(s) of791        sequences.792 793        Args:794            raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):795                The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float796                values, a list of numpy array or a list of list of float values. Must be mono channel audio, not797                stereo, i.e. single float per timestep.798 799            padding_side (`str`, *optional*):800                The side on which the model should have padding applied. Should be selected between ['right', 'left'].801                Default value is picked from the class attribute of the same name.802        """803 804        is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1805        if is_batched_numpy and len(raw_speech.shape) > 2:806            raise ValueError(f"Only mono-channel audio is supported for input to {self}")807        is_batched = is_batched_numpy or (808            isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))809        )810 811        # make sure input is in list format812        if is_batched and not isinstance(raw_speech[0], np.ndarray):813            raw_speech = [np.asarray(speech) for speech in raw_speech]814        elif not is_batched and not isinstance(raw_speech, np.ndarray):815            raw_speech = np.asarray(raw_speech)816 817        # always return batch818        if not is_batched:819            raw_speech = [raw_speech]820 821        # zero-mean and unit-variance normalization822        if self.do_normalize:823            raw_speech = [(x - np.mean(x)) / np.sqrt(np.var(x) + 1e-5) for x in raw_speech]824 825        # convert into correct format for padding826        encoded_inputs = BatchEncoding({"input_values": raw_speech})827 828        padded_inputs = self.pad(829            encoded_inputs,830            padding=padding,831            max_length=max_length,832            pad_to_multiple_of=pad_to_multiple_of,833            padding_side=padding_side,834            return_attention_mask=self.return_attention_mask,835            return_tensors=return_tensors,836            verbose=verbose,837        )838 839        return padded_inputs840 841    @property842    def vocab_size(self) -> int:843        return len(self.decoder)844 845    def get_vocab(self) -> dict:846        return dict(self.encoder, **self.added_tokens_encoder)847 848    def _convert_token_to_id(self, token: str) -> int:849        """Converts a token (str) in an index (integer) using the vocab."""850        return self.encoder.get(token, self.encoder.get(self.unk_token))851 852    def _convert_id_to_token(self, index: int) -> str:853        """Converts an index (integer) in a token (str) using the vocab."""854        result = self.decoder.get(index, self.unk_token)855        return result856 857    def convert_tokens_to_string(self, tokens: list[str]) -> str:858        """859        Converts a connectionist-temporal-classification (CTC) output tokens into a single string.860        """861        # group same tokens into non-repeating tokens in CTC style decoding862        grouped_tokens = [token_group[0] for token_group in groupby(tokens)]863 864        # filter self.pad_token which is used as CTC-blank token865        filtered_tokens = list(filter(lambda token: token != self.pad_token, grouped_tokens))866 867        # replace delimiter token868        string = "".join([" " if token == self.word_delimiter_token else token for token in filtered_tokens]).strip()869 870        if self.do_lower_case:871            string = string.lower()872 873        return string874 875    def _decode(876        self,877        token_ids: list[int],878        skip_special_tokens: bool = False,879        clean_up_tokenization_spaces: Optional[bool] = None,880        **kwargs,881    ) -> str:882        """883        special _decode function is needed for Wav2Vec2Tokenizer because added tokens should be treated exactly the884        same as tokens of the base vocabulary and therefore the function `convert_tokens_to_string` has to be called on885        the whole token list and not individually on added tokens886        """887        filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)888 889        result = []890        for token in filtered_tokens:891            if skip_special_tokens and (892                token in self.all_special_ids or (token != self.pad_token and token in self.all_special_tokens)893            ):894                continue895            result.append(token)896 897        text = self.convert_tokens_to_string(result)898 899        clean_up_tokenization_spaces = (900            clean_up_tokenization_spaces901            if clean_up_tokenization_spaces is not None902            else self.clean_up_tokenization_spaces903        )904        if clean_up_tokenization_spaces:905            clean_text = self.clean_up_tokenization(text)906            return clean_text907        else:908            return text909 910    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:911        if not os.path.isdir(save_directory):912            logger.error(f"Vocabulary path ({save_directory}) should be a directory")913            return914        vocab_file = os.path.join(915            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]916        )917 918        with open(vocab_file, "w", encoding="utf-8") as f:919            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")920 921        return (vocab_file,)922 923 924__all__ = ["Wav2Vec2CTCTokenizer", "Wav2Vec2Tokenizer"]925