Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2023 The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Speech processor class for SpeechT5."""16 17from ...processing_utils import ProcessorMixin18 19 20class SpeechT5Processor(ProcessorMixin):21    r"""22    Constructs a SpeechT5 processor which wraps a feature extractor and a tokenizer into a single processor.23 24    [`SpeechT5Processor`] offers all the functionalities of [`SpeechT5FeatureExtractor`] and [`SpeechT5Tokenizer`]. See25    the docstring of [`~SpeechT5Processor.__call__`] and [`~SpeechT5Processor.decode`] for more information.26 27    Args:28        feature_extractor (`SpeechT5FeatureExtractor`):29            An instance of [`SpeechT5FeatureExtractor`]. The feature extractor is a required input.30        tokenizer (`SpeechT5Tokenizer`):31            An instance of [`SpeechT5Tokenizer`]. The tokenizer is a required input.32    """33 34    feature_extractor_class = "SpeechT5FeatureExtractor"35    tokenizer_class = "SpeechT5Tokenizer"36 37    def __init__(self, feature_extractor, tokenizer):38        super().__init__(feature_extractor, tokenizer)39 40    def __call__(self, *args, **kwargs):41        """42        Processes audio and text input, as well as audio and text targets.43 44        You can process audio by using the argument `audio`, or process audio targets by using the argument45        `audio_target`. This forwards the arguments to SpeechT5FeatureExtractor's46        [`~SpeechT5FeatureExtractor.__call__`].47 48        You can process text by using the argument `text`, or process text labels by using the argument `text_target`.49        This forwards the arguments to SpeechT5Tokenizer's [`~SpeechT5Tokenizer.__call__`].50 51        Valid input combinations are:52 53        - `text` only54        - `audio` only55        - `text_target` only56        - `audio_target` only57        - `text` and `audio_target`58        - `audio` and `audio_target`59        - `text` and `text_target`60        - `audio` and `text_target`61 62        Please refer to the docstring of the above two methods for more information.63        """64        audio = kwargs.pop("audio", None)65        text = kwargs.pop("text", None)66        text_target = kwargs.pop("text_target", None)67        audio_target = kwargs.pop("audio_target", None)68        sampling_rate = kwargs.pop("sampling_rate", None)69 70        if audio is not None and text is not None:71            raise ValueError(72                "Cannot process both `audio` and `text` inputs. Did you mean `audio_target` or `text_target`?"73            )74        if audio_target is not None and text_target is not None:75            raise ValueError(76                "Cannot process both `audio_target` and `text_target` inputs. Did you mean `audio` or `text`?"77            )78        if audio is None and audio_target is None and text is None and text_target is None:79            raise ValueError(80                "You need to specify either an `audio`, `audio_target`, `text`, or `text_target` input to process."81            )82 83        if audio is not None:84            inputs = self.feature_extractor(audio, *args, sampling_rate=sampling_rate, **kwargs)85        elif text is not None:86            inputs = self.tokenizer(text, **kwargs)87        else:88            inputs = None89 90        if audio_target is not None:91            targets = self.feature_extractor(audio_target=audio_target, *args, sampling_rate=sampling_rate, **kwargs)92            labels = targets["input_values"]93        elif text_target is not None:94            targets = self.tokenizer(text_target, **kwargs)95            labels = targets["input_ids"]96        else:97            targets = None98 99        if inputs is None:100            return targets101 102        if targets is not None:103            inputs["labels"] = labels104 105            decoder_attention_mask = targets.get("attention_mask")106            if decoder_attention_mask is not None:107                inputs["decoder_attention_mask"] = decoder_attention_mask108 109        return inputs110 111    def pad(self, *args, **kwargs):112        """113        Collates the audio and text inputs, as well as their targets, into a padded batch.114 115        Audio inputs are padded by SpeechT5FeatureExtractor's [`~SpeechT5FeatureExtractor.pad`]. Text inputs are padded116        by SpeechT5Tokenizer's [`~SpeechT5Tokenizer.pad`].117 118        Valid input combinations are:119 120        - `input_ids` only121        - `input_values` only122        - `labels` only, either log-mel spectrograms or text tokens123        - `input_ids` and log-mel spectrogram `labels`124        - `input_values` and text `labels`125 126        Please refer to the docstring of the above two methods for more information.127        """128        input_values = kwargs.pop("input_values", None)129        input_ids = kwargs.pop("input_ids", None)130        labels = kwargs.pop("labels", None)131 132        if input_values is not None and input_ids is not None:133            raise ValueError("Cannot process both `input_values` and `input_ids` inputs.")134        if input_values is None and input_ids is None and labels is None:135            raise ValueError(136                "You need to specify either an `input_values`, `input_ids`, or `labels` input to be padded."137            )138 139        if input_values is not None:140            inputs = self.feature_extractor.pad(input_values, *args, **kwargs)141        elif input_ids is not None:142            inputs = self.tokenizer.pad(input_ids, **kwargs)143        else:144            inputs = None145 146        if labels is not None:147            if "input_ids" in labels or (isinstance(labels, list) and "input_ids" in labels[0]):148                targets = self.tokenizer.pad(labels, **kwargs)149                labels = targets["input_ids"]150            else:151                feature_size_hack = self.feature_extractor.feature_size152                self.feature_extractor.feature_size = self.feature_extractor.num_mel_bins153                targets = self.feature_extractor.pad(labels, *args, **kwargs)154                self.feature_extractor.feature_size = feature_size_hack155                labels = targets["input_values"]156        else:157            targets = None158 159        if inputs is None:160            return targets161 162        if targets is not None:163            inputs["labels"] = labels164 165            decoder_attention_mask = targets.get("attention_mask")166            if decoder_attention_mask is not None:167                inputs["decoder_attention_mask"] = decoder_attention_mask168 169        return inputs170 171 172__all__ = ["SpeechT5Processor"]173