Aluode/PerceptionLabPortable

1# Copyright 2021 The HuggingFace Inc. team.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""15Sequence feature extraction class for common feature extractors to preprocess sequences.16"""17 18from typing import Optional, Union19 20import numpy as np21 22from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin23from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy24 25 26logger = logging.get_logger(__name__)27 28 29class SequenceFeatureExtractor(FeatureExtractionMixin):30    """31    This is a general feature extraction class for speech recognition.32 33    Args:34        feature_size (`int`):35            The feature dimension of the extracted features.36        sampling_rate (`int`):37            The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).38        padding_value (`float`):39            The value that is used to fill the padding values / vectors.40    """41 42    def __init__(self, feature_size: int, sampling_rate: int, padding_value: float, **kwargs):43        self.feature_size = feature_size44        self.sampling_rate = sampling_rate45        self.padding_value = padding_value46 47        self.padding_side = kwargs.pop("padding_side", "right")48        self.return_attention_mask = kwargs.pop("return_attention_mask", True)49 50        super().__init__(**kwargs)51 52    def pad(53        self,54        processed_features: Union[55            BatchFeature,56            list[BatchFeature],57            dict[str, BatchFeature],58            dict[str, list[BatchFeature]],59            list[dict[str, BatchFeature]],60        ],61        padding: Union[bool, str, PaddingStrategy] = True,62        max_length: Optional[int] = None,63        truncation: bool = False,64        pad_to_multiple_of: Optional[int] = None,65        return_attention_mask: Optional[bool] = None,66        return_tensors: Optional[Union[str, TensorType]] = None,67    ) -> BatchFeature:68        """69        Pad input values / input vectors or a batch of input values / input vectors up to predefined length or to the70        max sequence length in the batch.71 72        Padding side (left/right) padding values are defined at the feature extractor level (with `self.padding_side`,73        `self.padding_value`)74 75        <Tip>76 77        If the `processed_features` passed are dictionary of numpy arrays, PyTorch tensors or TensorFlow tensors, the78        result will use the same type unless you provide a different tensor type with `return_tensors`. In the case of79        PyTorch tensors, you will lose the specific device of your tensors however.80 81        </Tip>82 83        Args:84            processed_features ([`BatchFeature`], list of [`BatchFeature`], `dict[str, list[float]]`, `dict[str, list[list[float]]` or `list[dict[str, list[float]]]`):85                Processed inputs. Can represent one input ([`BatchFeature`] or `dict[str, list[float]]`) or a batch of86                input values / vectors (list of [`BatchFeature`], *dict[str, list[list[float]]]* or *list[dict[str,87                list[float]]]*) so you can use this method during preprocessing as well as in a PyTorch Dataloader88                collate function.89 90                Instead of `list[float]` you can have tensors (numpy arrays, PyTorch tensors or TensorFlow tensors),91                see the note above for the return type.92            padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):93                Select a strategy to pad the returned sequences (according to the model's padding side and padding94                index) among:95 96                - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single97                  sequence if provided).98                - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum99                  acceptable input length for the model if that argument is not provided.100                - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different101                  lengths).102            max_length (`int`, *optional*):103                Maximum length of the returned list and optionally padding length (see above).104            truncation (`bool`):105                Activates truncation to cut input sequences longer than `max_length` to `max_length`.106            pad_to_multiple_of (`int`, *optional*):107                If set will pad the sequence to a multiple of the provided value.108 109                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability110                `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.111            return_attention_mask (`bool`, *optional*):112                Whether to return the attention mask. If left to the default, will return the attention mask according113                to the specific feature_extractor's default.114 115                [What are attention masks?](../glossary#attention-mask)116            return_tensors (`str` or [`~utils.TensorType`], *optional*):117                If set, will return tensors instead of list of python integers. Acceptable values are:118 119                - `'tf'`: Return TensorFlow `tf.constant` objects.120                - `'pt'`: Return PyTorch `torch.Tensor` objects.121                - `'np'`: Return Numpy `np.ndarray` objects.122        """123        # If we have a list of dicts, let's convert it in a dict of lists124        # We do this to allow using this method as a collate_fn function in PyTorch Dataloader125        if isinstance(processed_features, (list, tuple)) and isinstance(processed_features[0], (dict, BatchFeature)):126            processed_features = {127                key: [example[key] for example in processed_features] for key in processed_features[0]128            }129 130        # The model's main input name, usually `input_values`, has be passed for padding131        if self.model_input_names[0] not in processed_features:132            raise ValueError(133                "You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`"134                f" to this method that includes {self.model_input_names[0]}, but you provided"135                f" {list(processed_features.keys())}"136            )137 138        required_input = processed_features[self.model_input_names[0]]139        return_attention_mask = (140            return_attention_mask if return_attention_mask is not None else self.return_attention_mask141        )142 143        if len(required_input) == 0:144            if return_attention_mask:145                processed_features["attention_mask"] = []146            return processed_features147 148        # If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays149        # and rebuild them afterwards if no return_tensors is specified150        # Note that we lose the specific device the tensor may be on for PyTorch151 152        first_element = required_input[0]153        if isinstance(first_element, (list, tuple)):154            # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.155            index = 0156            while len(required_input[index]) == 0:157                index += 1158            if index < len(required_input):159                first_element = required_input[index][0]160 161        if return_tensors is None:162            if is_tf_tensor(first_element):163                return_tensors = "tf"164            elif is_torch_tensor(first_element):165                return_tensors = "pt"166            elif isinstance(first_element, (int, float, list, tuple, np.ndarray)):167                return_tensors = "np"168            else:169                raise ValueError(170                    f"type of {first_element} unknown: {type(first_element)}. "171                    "Should be one of a python, numpy, pytorch or tensorflow object."172                )173 174        for key, value in processed_features.items():175            if isinstance(value[0], (int, float)):176                processed_features[key] = to_numpy(value)177            else:178                processed_features[key] = [to_numpy(v) for v in value]179 180        # Convert padding_strategy in PaddingStrategy181        padding_strategy = self._get_padding_strategies(padding=padding, max_length=max_length)182 183        required_input = processed_features[self.model_input_names[0]]184 185        batch_size = len(required_input)186        if not all(len(v) == batch_size for v in processed_features.values()):187            raise ValueError("Some items in the output dictionary have a different batch size than others.")188 189        truncated_inputs = []190        for i in range(batch_size):191            inputs = {k: v[i] for k, v in processed_features.items()}192            # truncation193            inputs_slice = self._truncate(194                inputs,195                max_length=max_length,196                pad_to_multiple_of=pad_to_multiple_of,197                truncation=truncation,198            )199            truncated_inputs.append(inputs_slice)200 201        if padding_strategy == PaddingStrategy.LONGEST:202            # make sure that `max_length` cannot be longer than the longest truncated length203            max_length = max(len(input_slice[self.model_input_names[0]]) for input_slice in truncated_inputs)204            padding_strategy = PaddingStrategy.MAX_LENGTH205 206        batch_outputs = {}207        for i in range(batch_size):208            # padding209            outputs = self._pad(210                truncated_inputs[i],211                max_length=max_length,212                padding_strategy=padding_strategy,213                pad_to_multiple_of=pad_to_multiple_of,214                return_attention_mask=return_attention_mask,215            )216 217            for key, value in outputs.items():218                if key not in batch_outputs:219                    batch_outputs[key] = []220                if value.dtype is np.dtype(np.float64):221                    value = value.astype(np.float32)222                batch_outputs[key].append(value)223 224        return BatchFeature(batch_outputs, tensor_type=return_tensors)225 226    def _pad(227        self,228        processed_features: Union[dict[str, np.ndarray], BatchFeature],229        max_length: Optional[int] = None,230        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,231        pad_to_multiple_of: Optional[int] = None,232        return_attention_mask: Optional[bool] = None,233    ) -> dict:234        """235        Pad inputs (on left/right and up to predefined length or max length in the batch)236 237        Args:238            processed_features (`Union[dict[str, np.ndarray], BatchFeature]`):239                Dictionary of input values (`np.ndarray[float]`) / input vectors (`list[np.ndarray[float]]`) or batch240                of inputs values (`list[np.ndarray[int]]`) / input vectors (`list[np.ndarray[int]]`)241            max_length (`int`, *optional*):242                Maximum length of the returned list and optionally padding length (see below)243            padding_strategy (`PaddingStrategy`, *optional*, default to `PaddingStrategy.DO_NOT_PAD`):244                PaddingStrategy to use for padding.245 246                - PaddingStrategy.LONGEST Pad to the longest sequence in the batch247                - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)248                - PaddingStrategy.DO_NOT_PAD: Do not pad249                The feature_extractor padding sides are defined in self.padding_side:250 251                    - 'left': pads on the left of the sequences252                    - 'right': pads on the right of the sequences253            pad_to_multiple_of (`int`, *optional*):254                Integer if set will pad the sequence to a multiple of the provided value. This is especially useful to255                enable the use of Tensor Core on NVIDIA hardware with compute capability `>= 7.5` (Volta), or on TPUs256                which benefit from having sequence lengths be a multiple of 128.257            return_attention_mask (`bool`, *optional*):258                Set to False to avoid returning attention mask (default: set to model specifics)259        """260        required_input = processed_features[self.model_input_names[0]]261 262        if padding_strategy == PaddingStrategy.LONGEST:263            max_length = len(required_input)264 265        if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):266            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of267 268        needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) < max_length269 270        if return_attention_mask and "attention_mask" not in processed_features:271            processed_features["attention_mask"] = np.ones(len(required_input), dtype=np.int32)272 273        if needs_to_be_padded:274            difference = max_length - len(required_input)275            if self.padding_side == "right":276                if return_attention_mask:277                    processed_features["attention_mask"] = np.pad(278                        processed_features["attention_mask"], (0, difference)279                    )280                padding_shape = ((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)281                processed_features[self.model_input_names[0]] = np.pad(282                    required_input, padding_shape, "constant", constant_values=self.padding_value283                )284            elif self.padding_side == "left":285                if return_attention_mask:286                    processed_features["attention_mask"] = np.pad(287                        processed_features["attention_mask"], (difference, 0)288                    )289                padding_shape = ((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)290                processed_features[self.model_input_names[0]] = np.pad(291                    required_input, padding_shape, "constant", constant_values=self.padding_value292                )293            else:294                raise ValueError("Invalid padding strategy:" + str(self.padding_side))295 296        return processed_features297 298    def _truncate(299        self,300        processed_features: Union[dict[str, np.ndarray], BatchFeature],301        max_length: Optional[int] = None,302        pad_to_multiple_of: Optional[int] = None,303        truncation: Optional[bool] = None,304    ):305        """306        Truncate inputs to predefined length or max length in the batch307 308        Args:309            processed_features(`Union[dict[str, np.ndarray], BatchFeature]`):310                Dictionary of input values (`np.ndarray[float]`) / input vectors (`list[np.ndarray[float]]`) or batch311                of inputs values (`list[np.ndarray[int]]`) / input vectors (`list[np.ndarray[int]]`)312            max_length (`int`, *optional*):313                maximum length of the returned list and optionally padding length (see below)314            pad_to_multiple_of (`int`, *optional*) :315                Integer if set will pad the sequence to a multiple of the provided value. This is especially useful to316                enable the use of Tensor Core on NVIDIA hardware with compute capability `>= 7.5` (Volta), or on TPUs317                which benefit from having sequence lengths be a multiple of 128.318            truncation (`bool`, *optional*):319                Activates truncation to cut input sequences longer than `max_length` to `max_length`.320        """321        if not truncation:322            return processed_features323        elif truncation and max_length is None:324            raise ValueError("When setting ``truncation=True``, make sure that ``max_length`` is defined.")325 326        required_input = processed_features[self.model_input_names[0]]327 328        # find `max_length` that fits `pad_to_multiple_of`329        if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):330            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of331 332        needs_to_be_truncated = len(required_input) > max_length333 334        if needs_to_be_truncated:335            processed_features[self.model_input_names[0]] = processed_features[self.model_input_names[0]][:max_length]336            if "attention_mask" in processed_features:337                processed_features["attention_mask"] = processed_features["attention_mask"][:max_length]338 339        return processed_features340 341    def _get_padding_strategies(self, padding=False, max_length=None):342        """343        Find the correct padding strategy344        """345 346        # Get padding strategy347        if padding is not False:348            if padding is True:349                padding_strategy = PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch350            elif not isinstance(padding, PaddingStrategy):351                padding_strategy = PaddingStrategy(padding)352            elif isinstance(padding, PaddingStrategy):353                padding_strategy = padding354        else:355            padding_strategy = PaddingStrategy.DO_NOT_PAD356 357        # Set max length if needed358        if max_length is None:359            if padding_strategy == PaddingStrategy.MAX_LENGTH:360                raise ValueError(361                    f"When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined"362                )363 364        # Test if we have a padding value365        if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):366            raise ValueError(367                "Asking to pad but the feature_extractor does not have a padding value. Please select a value to use"368                " as `padding_value`. For example: `feature_extractor.padding_value = 0.0`."369            )370 371        return padding_strategy372