Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2024 Meta AI and The HuggingFace Inc. team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""16Text/audio processor class for MusicGen Melody17"""18 19from typing import Any20 21import numpy as np22 23from ...processing_utils import ProcessorMixin24from ...utils import to_numpy25from ...utils.import_utils import requires26 27 28@requires(backends=("torchaudio",))29class MusicgenMelodyProcessor(ProcessorMixin):30    r"""31    Constructs a MusicGen Melody processor which wraps a Wav2Vec2 feature extractor - for raw audio waveform processing - and a T5 tokenizer into a single processor32    class.33 34    [`MusicgenProcessor`] offers all the functionalities of [`MusicgenMelodyFeatureExtractor`] and [`T5Tokenizer`]. See35    [`~MusicgenProcessor.__call__`] and [`~MusicgenProcessor.decode`] for more information.36 37    Args:38        feature_extractor (`MusicgenMelodyFeatureExtractor`):39            An instance of [`MusicgenMelodyFeatureExtractor`]. The feature extractor is a required input.40        tokenizer (`T5Tokenizer`):41            An instance of [`T5Tokenizer`]. The tokenizer is a required input.42    """43 44    feature_extractor_class = "MusicgenMelodyFeatureExtractor"45    tokenizer_class = ("T5Tokenizer", "T5TokenizerFast")46 47    def __init__(self, feature_extractor, tokenizer):48        super().__init__(feature_extractor, tokenizer)49 50    # Copied from transformers.models.musicgen.processing_musicgen.MusicgenProcessor.get_decoder_prompt_ids51    def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True):52        return self.tokenizer.get_decoder_prompt_ids(task=task, language=language, no_timestamps=no_timestamps)53 54    def __call__(self, *args, **kwargs):55        """56        Forwards the `audio` argument to EncodecFeatureExtractor's [`~EncodecFeatureExtractor.__call__`] and the `text`57        argument to [`~T5Tokenizer.__call__`]. Please refer to the docstring of the above two methods for more58        information.59        """60 61        if len(args) > 0:62            kwargs["audio"] = args[0]63        return super().__call__(*args, **kwargs)64 65    # Copied from transformers.models.musicgen.processing_musicgen.MusicgenProcessor.batch_decode with padding_mask->attention_mask66    def batch_decode(self, *args, **kwargs):67        """68        This method is used to decode either batches of audio outputs from the MusicGen model, or batches of token ids69        from the tokenizer. In the case of decoding token ids, this method forwards all its arguments to T5Tokenizer's70        [`~PreTrainedTokenizer.batch_decode`]. Please refer to the docstring of this method for more information.71        """72        audio_values = kwargs.pop("audio", None)73        attention_mask = kwargs.pop("attention_mask", None)74 75        if len(args) > 0:76            audio_values = args[0]77            args = args[1:]78 79        if audio_values is not None:80            return self._decode_audio(audio_values, attention_mask=attention_mask)81        else:82            return self.tokenizer.batch_decode(*args, **kwargs)83 84    # Copied from transformers.models.musicgen.processing_musicgen.MusicgenProcessor._decode_audio with padding_mask->attention_mask85    def _decode_audio(self, audio_values, attention_mask: Any = None) -> list[np.ndarray]:86        """87        This method strips any padding from the audio values to return a list of numpy audio arrays.88        """89        audio_values = to_numpy(audio_values)90        bsz, channels, seq_len = audio_values.shape91 92        if attention_mask is None:93            return list(audio_values)94 95        attention_mask = to_numpy(attention_mask)96 97        # match the sequence length of the padding mask to the generated audio arrays by padding with the **non-padding**98        # token (so that the generated audio values are **not** treated as padded tokens)99        difference = seq_len - attention_mask.shape[-1]100        padding_value = 1 - self.feature_extractor.padding_value101        attention_mask = np.pad(attention_mask, ((0, 0), (0, difference)), "constant", constant_values=padding_value)102 103        audio_values = audio_values.tolist()104        for i in range(bsz):105            sliced_audio = np.asarray(audio_values[i])[106                attention_mask[i][None, :] != self.feature_extractor.padding_value107            ]108            audio_values[i] = sliced_audio.reshape(channels, -1)109 110        return audio_values111 112    def get_unconditional_inputs(self, num_samples=1, return_tensors="pt"):113        """114        Helper function to get null inputs for unconditional generation, enabling the model to be used without the115        feature extractor or tokenizer.116 117        Args:118            num_samples (int, *optional*):119                Number of audio samples to unconditionally generate.120 121        Example:122        ```python123        >>> from transformers import MusicgenMelodyForConditionalGeneration, MusicgenMelodyProcessor124 125        >>> model = MusicgenMelodyForConditionalGeneration.from_pretrained("facebook/musicgen-melody")126 127        >>> # get the unconditional (or 'null') inputs for the model128        >>> processor = MusicgenMelodyProcessor.from_pretrained("facebook/musicgen-melody")129        >>> unconditional_inputs = processor.get_unconditional_inputs(num_samples=1)130 131        >>> audio_samples = model.generate(**unconditional_inputs, max_new_tokens=256)132        ```"""133        inputs = self.tokenizer([""] * num_samples, return_tensors=return_tensors, return_attention_mask=True)134        inputs["attention_mask"][:] = 0135 136        return inputs137 138 139__all__ = ["MusicgenMelodyProcessor"]140