Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2023 The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Feature extractor class for SpeechT5."""16 17import warnings18from typing import Any, Optional, Union19 20import numpy as np21 22from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function23from ...feature_extraction_sequence_utils import SequenceFeatureExtractor24from ...feature_extraction_utils import BatchFeature25from ...utils import PaddingStrategy, TensorType, logging26 27 28logger = logging.get_logger(__name__)29 30 31class SpeechT5FeatureExtractor(SequenceFeatureExtractor):32    r"""33    Constructs a SpeechT5 feature extractor.34 35    This class can pre-process a raw speech signal by (optionally) normalizing to zero-mean unit-variance, for use by36    the SpeechT5 speech encoder prenet.37 38    This class can also extract log-mel filter bank features from raw speech, for use by the SpeechT5 speech decoder39    prenet.40 41    This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains42    most of the main methods. Users should refer to this superclass for more information regarding those methods.43 44    Args:45        feature_size (`int`, *optional*, defaults to 1):46            The feature dimension of the extracted features.47        sampling_rate (`int`, *optional*, defaults to 16000):48            The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).49        padding_value (`float`, *optional*, defaults to 0.0):50            The value that is used to fill the padding values.51        do_normalize (`bool`, *optional*, defaults to `False`):52            Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly53            improve the performance for some models.54        num_mel_bins (`int`, *optional*, defaults to 80):55            The number of mel-frequency bins in the extracted spectrogram features.56        hop_length (`int`, *optional*, defaults to 16):57            Number of ms between windows. Otherwise referred to as "shift" in many papers.58        win_length (`int`, *optional*, defaults to 64):59            Number of ms per window.60        win_function (`str`, *optional*, defaults to `"hann_window"`):61            Name for the window function used for windowing, must be accessible via `torch.{win_function}`62        frame_signal_scale (`float`, *optional*, defaults to 1.0):63            Constant multiplied in creating the frames before applying DFT. This argument is deprecated.64        fmin (`float`, *optional*, defaults to 80):65            Minimum mel frequency in Hz.66        fmax (`float`, *optional*, defaults to 7600):67            Maximum mel frequency in Hz.68        mel_floor (`float`, *optional*, defaults to 1e-10):69            Minimum value of mel frequency banks.70        reduction_factor (`int`, *optional*, defaults to 2):71            Spectrogram length reduction factor. This argument is deprecated.72        return_attention_mask (`bool`, *optional*, defaults to `True`):73            Whether or not [`~SpeechT5FeatureExtractor.__call__`] should return `attention_mask`.74    """75 76    model_input_names = ["input_values", "attention_mask"]77 78    def __init__(79        self,80        feature_size: int = 1,81        sampling_rate: int = 16000,82        padding_value: float = 0.0,83        do_normalize: bool = False,84        num_mel_bins: int = 80,85        hop_length: int = 16,86        win_length: int = 64,87        win_function: str = "hann_window",88        frame_signal_scale: float = 1.0,89        fmin: float = 80,90        fmax: float = 7600,91        mel_floor: float = 1e-10,92        reduction_factor: int = 2,93        return_attention_mask: bool = True,94        **kwargs,95    ):96        super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)97        self.do_normalize = do_normalize98        self.return_attention_mask = return_attention_mask99 100        self.num_mel_bins = num_mel_bins101        self.hop_length = hop_length102        self.win_length = win_length103        self.win_function = win_function104        self.frame_signal_scale = frame_signal_scale105        self.fmin = fmin106        self.fmax = fmax107        self.mel_floor = mel_floor108        self.reduction_factor = reduction_factor109 110        self.sample_size = win_length * sampling_rate // 1000111        self.sample_stride = hop_length * sampling_rate // 1000112        self.n_fft = optimal_fft_length(self.sample_size)113        self.n_freqs = (self.n_fft // 2) + 1114 115        self.window = window_function(window_length=self.sample_size, name=self.win_function, periodic=True)116 117        self.mel_filters = mel_filter_bank(118            num_frequency_bins=self.n_freqs,119            num_mel_filters=self.num_mel_bins,120            min_frequency=self.fmin,121            max_frequency=self.fmax,122            sampling_rate=self.sampling_rate,123            norm="slaney",124            mel_scale="slaney",125        )126 127        if frame_signal_scale != 1.0:128            warnings.warn(129                "The argument `frame_signal_scale` is deprecated and will be removed in version 4.30.0 of Transformers",130                FutureWarning,131            )132        if reduction_factor != 2.0:133            warnings.warn(134                "The argument `reduction_factor` is deprecated and will be removed in version 4.30.0 of Transformers",135                FutureWarning,136            )137 138    @staticmethod139    # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm140    def zero_mean_unit_var_norm(141        input_values: list[np.ndarray], attention_mask: list[np.ndarray], padding_value: float = 0.0142    ) -> list[np.ndarray]:143        """144        Every array in the list is normalized to have zero mean and unit variance145        """146        if attention_mask is not None:147            attention_mask = np.array(attention_mask, np.int32)148            normed_input_values = []149 150            for vector, length in zip(input_values, attention_mask.sum(-1)):151                normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)152                if length < normed_slice.shape[0]:153                    normed_slice[length:] = padding_value154 155                normed_input_values.append(normed_slice)156        else:157            normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]158 159        return normed_input_values160 161    def _extract_mel_features(162        self,163        one_waveform: np.ndarray,164    ) -> np.ndarray:165        """166        Extracts log-mel filterbank features for one waveform array (unbatched).167        """168        log_mel_spec = spectrogram(169            one_waveform,170            window=self.window,171            frame_length=self.sample_size,172            hop_length=self.sample_stride,173            fft_length=self.n_fft,174            mel_filters=self.mel_filters,175            mel_floor=self.mel_floor,176            log_mel="log10",177        )178        return log_mel_spec.T179 180    def __call__(181        self,182        audio: Optional[Union[np.ndarray, list[float], list[np.ndarray], list[list[float]]]] = None,183        audio_target: Optional[Union[np.ndarray, list[float], list[np.ndarray], list[list[float]]]] = None,184        padding: Union[bool, str, PaddingStrategy] = False,185        max_length: Optional[int] = None,186        truncation: bool = False,187        pad_to_multiple_of: Optional[int] = None,188        return_attention_mask: Optional[bool] = None,189        return_tensors: Optional[Union[str, TensorType]] = None,190        sampling_rate: Optional[int] = None,191        **kwargs,192    ) -> BatchFeature:193        """194        Main method to featurize and prepare for the model one or several sequence(s).195 196        Pass in a value for `audio` to extract waveform features. Pass in a value for `audio_target` to extract log-mel197        spectrogram features.198 199        Args:200            audio (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`, *optional*):201                The sequence or batch of sequences to be processed. Each sequence can be a numpy array, a list of float202                values, a list of numpy arrays or a list of list of float values. This outputs waveform features. Must203                be mono channel audio, not stereo, i.e. single float per timestep.204            audio_target (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`, *optional*):205                The sequence or batch of sequences to be processed as targets. Each sequence can be a numpy array, a206                list of float values, a list of numpy arrays or a list of list of float values. This outputs log-mel207                spectrogram features.208            padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):209                Select a strategy to pad the returned sequences (according to the model's padding side and padding210                index) among:211 212                - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single213                  sequence if provided).214                - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum215                  acceptable input length for the model if that argument is not provided.216                - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different217                  lengths).218            max_length (`int`, *optional*):219                Maximum length of the returned list and optionally padding length (see above).220            truncation (`bool`):221                Activates truncation to cut input sequences longer than *max_length* to *max_length*.222            pad_to_multiple_of (`int`, *optional*):223                If set will pad the sequence to a multiple of the provided value.224 225                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability226                `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.227            return_attention_mask (`bool`, *optional*):228                Whether to return the attention mask. If left to the default, will return the attention mask according229                to the specific feature_extractor's default.230 231                [What are attention masks?](../glossary#attention-mask)232 233            return_tensors (`str` or [`~utils.TensorType`], *optional*):234                If set, will return tensors instead of list of python integers. Acceptable values are:235 236                - `'tf'`: Return TensorFlow `tf.constant` objects.237                - `'pt'`: Return PyTorch `torch.Tensor` objects.238                - `'np'`: Return Numpy `np.ndarray` objects.239            sampling_rate (`int`, *optional*):240                The sampling rate at which the `audio` or `audio_target` input was sampled. It is strongly recommended241                to pass `sampling_rate` at the forward call to prevent silent errors.242        """243        if audio is None and audio_target is None:244            raise ValueError("You must provide either `audio` or `audio_target` values.")245 246        if sampling_rate is not None:247            if sampling_rate != self.sampling_rate:248                raise ValueError(249                    f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"250                    f" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"251                    f" {self.sampling_rate} and not {sampling_rate}."252                )253        else:254            logger.warning(255                f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "256                "Failing to do so can result in silent errors that might be hard to debug."257            )258 259        if audio is not None:260            inputs = self._process_audio(261                audio,262                False,263                padding,264                max_length,265                truncation,266                pad_to_multiple_of,267                return_attention_mask,268                return_tensors,269                **kwargs,270            )271        else:272            inputs = None273 274        if audio_target is not None:275            inputs_target = self._process_audio(276                audio_target,277                True,278                padding,279                max_length,280                truncation,281                pad_to_multiple_of,282                return_attention_mask,283                return_tensors,284                **kwargs,285            )286 287            if inputs is None:288                return inputs_target289            else:290                inputs["labels"] = inputs_target["input_values"]291                decoder_attention_mask = inputs_target.get("attention_mask")292                if decoder_attention_mask is not None:293                    inputs["decoder_attention_mask"] = decoder_attention_mask294 295        return inputs296 297    def _process_audio(298        self,299        speech: Union[np.ndarray, list[float], list[np.ndarray], list[list[float]]],300        is_target: bool = False,301        padding: Union[bool, str, PaddingStrategy] = False,302        max_length: Optional[int] = None,303        truncation: bool = False,304        pad_to_multiple_of: Optional[int] = None,305        return_attention_mask: Optional[bool] = None,306        return_tensors: Optional[Union[str, TensorType]] = None,307        **kwargs,308    ) -> BatchFeature:309        is_batched_numpy = isinstance(speech, np.ndarray) and len(speech.shape) > 1310        if is_batched_numpy and len(speech.shape) > 2:311            raise ValueError(f"Only mono-channel audio is supported for input to {self}")312        is_batched = is_batched_numpy or (313            isinstance(speech, (list, tuple)) and (isinstance(speech[0], (np.ndarray, tuple, list)))314        )315 316        if is_batched:317            speech = [np.asarray(speech, dtype=np.float32) for speech in speech]318        elif not is_batched and not isinstance(speech, np.ndarray):319            speech = np.asarray(speech, dtype=np.float32)320        elif isinstance(speech, np.ndarray) and speech.dtype is np.dtype(np.float64):321            speech = speech.astype(np.float32)322 323        # always return batch324        if not is_batched:325            speech = [speech]326 327        # needed to make pad() work on spectrogram inputs328        feature_size_hack = self.feature_size329 330        # convert into correct format for padding331        if is_target:332            features = [self._extract_mel_features(waveform) for waveform in speech]333            encoded_inputs = BatchFeature({"input_values": features})334            self.feature_size = self.num_mel_bins335        else:336            encoded_inputs = BatchFeature({"input_values": speech})337 338        padded_inputs = self.pad(339            encoded_inputs,340            padding=padding,341            max_length=max_length,342            truncation=truncation,343            pad_to_multiple_of=pad_to_multiple_of,344            return_attention_mask=return_attention_mask,345            **kwargs,346        )347 348        self.feature_size = feature_size_hack349 350        # convert input values to correct format351        input_values = padded_inputs["input_values"]352        if not isinstance(input_values[0], np.ndarray):353            padded_inputs["input_values"] = [np.asarray(array, dtype=np.float32) for array in input_values]354        elif (355            not isinstance(input_values, np.ndarray)356            and isinstance(input_values[0], np.ndarray)357            and input_values[0].dtype is np.dtype(np.float64)358        ):359            padded_inputs["input_values"] = [array.astype(np.float32) for array in input_values]360        elif isinstance(input_values, np.ndarray) and input_values.dtype is np.dtype(np.float64):361            padded_inputs["input_values"] = input_values.astype(np.float32)362 363        # convert attention_mask to correct format364        attention_mask = padded_inputs.get("attention_mask")365        if attention_mask is not None:366            padded_inputs["attention_mask"] = [np.asarray(array, dtype=np.int32) for array in attention_mask]367 368        # zero-mean and unit-variance normalization369        if not is_target and self.do_normalize:370            attention_mask = (371                attention_mask372                if self._get_padding_strategies(padding, max_length=max_length) is not PaddingStrategy.DO_NOT_PAD373                else None374            )375            padded_inputs["input_values"] = self.zero_mean_unit_var_norm(376                padded_inputs["input_values"], attention_mask=attention_mask, padding_value=self.padding_value377            )378 379        if return_tensors is not None:380            padded_inputs = padded_inputs.convert_to_tensors(return_tensors)381 382        return padded_inputs383 384    def to_dict(self) -> dict[str, Any]:385        output = super().to_dict()386 387        # Don't serialize these as they are derived from the other properties.388        names = ["window", "mel_filters", "sample_size", "sample_stride", "n_fft", "n_freqs"]389        for name in names:390            if name in output:391                del output[name]392 393        return output394 395 396__all__ = ["SpeechT5FeatureExtractor"]397