softwareweaver/MusicGen

1# Copyright (c) Meta Platforms, Inc. and affiliates.2# All rights reserved.3#4# This source code is licensed under the license found in the5# LICENSE file in the root directory of this source tree.6 7import math8import typing as tp9 10import torch11 12from .base import BaseQuantizer, QuantizedResult13from .core_vq import ResidualVectorQuantization14 15 16class ResidualVectorQuantizer(BaseQuantizer):17    """Residual Vector Quantizer.18 19    Args:20        dimension (int): Dimension of the codebooks.21        n_q (int): Number of residual vector quantizers used.22        q_dropout (bool): Random quantizer drop out at train time.23        bins (int): Codebook size.24        decay (float): Decay for exponential moving average over the codebooks.25        kmeans_init (bool): Whether to use kmeans to initialize the codebooks.26        kmeans_iters (int): Number of iterations used for kmeans initialization.27        threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes28            that have an exponential moving average cluster size less than the specified threshold with29            randomly selected vector from the current batch.30        orthogonal_reg_weight (float): Orthogonal regularization weights.31        orthogonal_reg_active_codes_only (bool): Apply orthogonal regularization only on active codes.32        orthogonal_reg_max_codes (optional int): Maximum number of codes to consider.33            for orthogonal regularization.34    """35    def __init__(36        self,37        dimension: int = 256,38        n_q: int = 8,39        q_dropout: bool = False,40        bins: int = 1024,41        decay: float = 0.99,42        kmeans_init: bool = True,43        kmeans_iters: int = 10,44        threshold_ema_dead_code: int = 2,45        orthogonal_reg_weight: float = 0.0,46        orthogonal_reg_active_codes_only: bool = False,47        orthogonal_reg_max_codes: tp.Optional[int] = None,48    ):49        super().__init__()50        self.max_n_q = n_q51        self.n_q = n_q52        self.q_dropout = q_dropout53        self.dimension = dimension54        self.bins = bins55        self.decay = decay56        self.kmeans_init = kmeans_init57        self.kmeans_iters = kmeans_iters58        self.threshold_ema_dead_code = threshold_ema_dead_code59        self.orthogonal_reg_weight = orthogonal_reg_weight60        self.orthogonal_reg_active_codes_only = orthogonal_reg_active_codes_only61        self.orthogonal_reg_max_codes = orthogonal_reg_max_codes62        self.vq = ResidualVectorQuantization(63            dim=self.dimension,64            codebook_size=self.bins,65            num_quantizers=self.n_q,66            decay=self.decay,67            kmeans_init=self.kmeans_init,68            kmeans_iters=self.kmeans_iters,69            threshold_ema_dead_code=self.threshold_ema_dead_code,70            orthogonal_reg_weight=self.orthogonal_reg_weight,71            orthogonal_reg_active_codes_only=self.orthogonal_reg_active_codes_only,72            orthogonal_reg_max_codes=self.orthogonal_reg_max_codes,73            channels_last=False74        )75 76    def forward(self, x: torch.Tensor, frame_rate: int):77        n_q = self.n_q78        if self.training and self.q_dropout:79            n_q = int(torch.randint(1, self.n_q + 1, (1,)).item())80        bw_per_q = math.log2(self.bins) * frame_rate / 100081        quantized, codes, commit_loss = self.vq(x, n_q=n_q)82        codes = codes.transpose(0, 1)83        # codes is [B, K, T], with T frames, K nb of codebooks.84        bw = torch.tensor(n_q * bw_per_q).to(x)85        return QuantizedResult(quantized, codes, bw, penalty=torch.mean(commit_loss))86 87    def encode(self, x: torch.Tensor) -> torch.Tensor:88        """Encode a given input tensor with the specified frame rate at the given bandwidth.89        The RVQ encode method sets the appropriate number of quantizer to use90        and returns indices for each quantizer.91        """92        n_q = self.n_q93        codes = self.vq.encode(x, n_q=n_q)94        codes = codes.transpose(0, 1)95        # codes is [B, K, T], with T frames, K nb of codebooks.96        return codes97 98    def decode(self, codes: torch.Tensor) -> torch.Tensor:99        """Decode the given codes to the quantized representation."""100        # codes is [B, K, T], with T frames, K nb of codebooks, vq.decode expects [K, B, T].101        codes = codes.transpose(0, 1)102        quantized = self.vq.decode(codes)103        return quantized104 105    @property106    def total_codebooks(self):107        return self.max_n_q108 109    @property110    def num_codebooks(self):111        return self.n_q112 113    def set_num_codebooks(self, n: int):114        assert n > 0 and n <= self.max_n_q115        self.n_q = n116