goathead777/Zero_Shot_Inference

1# Copyright (c) Meta Platforms, Inc. and affiliates.2# All rights reserved.3#4# This source code is licensed under the license found in the5# LICENSE file in the root directory of this source tree.6 7"""Residual vector quantizer implementation."""8 9from dataclasses import dataclass, field10import math11import typing as tp12 13import torch14from torch import nn15 16from module.core_vq import ResidualVectorQuantization17 18 19@dataclass20class QuantizedResult:21    quantized: torch.Tensor22    codes: torch.Tensor23    bandwidth: torch.Tensor  # bandwidth in kb/s used, per batch item.24    penalty: tp.Optional[torch.Tensor] = None25    metrics: dict = field(default_factory=dict)26 27 28class ResidualVectorQuantizer(nn.Module):29    """Residual Vector Quantizer.30    Args:31        dimension (int): Dimension of the codebooks.32        n_q (int): Number of residual vector quantizers used.33        bins (int): Codebook size.34        decay (float): Decay for exponential moving average over the codebooks.35        kmeans_init (bool): Whether to use kmeans to initialize the codebooks.36        kmeans_iters (int): Number of iterations used for kmeans initialization.37        threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes38            that have an exponential moving average cluster size less than the specified threshold with39            randomly selected vector from the current batch.40    """41 42    def __init__(43        self,44        dimension: int = 256,45        n_q: int = 8,46        bins: int = 1024,47        decay: float = 0.99,48        kmeans_init: bool = True,49        kmeans_iters: int = 50,50        threshold_ema_dead_code: int = 2,51    ):52        super().__init__()53        self.n_q = n_q54        self.dimension = dimension55        self.bins = bins56        self.decay = decay57        self.kmeans_init = kmeans_init58        self.kmeans_iters = kmeans_iters59        self.threshold_ema_dead_code = threshold_ema_dead_code60        self.vq = ResidualVectorQuantization(61            dim=self.dimension,62            codebook_size=self.bins,63            num_quantizers=self.n_q,64            decay=self.decay,65            kmeans_init=self.kmeans_init,66            kmeans_iters=self.kmeans_iters,67            threshold_ema_dead_code=self.threshold_ema_dead_code,68        )69 70    def forward(71        self,72        x: torch.Tensor,73        n_q: tp.Optional[int] = None,74        layers: tp.Optional[list] = None,75    ) -> QuantizedResult:76        """Residual vector quantization on the given input tensor.77        Args:78            x (torch.Tensor): Input tensor.79            n_q (int): Number of quantizer used to quantize. Default: All quantizers.80            layers (list): Layer that need to return quantized. Defalt: None.81        Returns:82            QuantizedResult:83                The quantized (or approximately quantized) representation with84                the associated numbert quantizers and layer quantized required to return.85        """86        n_q = n_q if n_q else self.n_q87        if layers and max(layers) >= n_q:88            raise ValueError(89                f"Last layer index in layers: A {max(layers)}. Number of quantizers in RVQ: B {self.n_q}. A must less than B."90            )91        quantized, codes, commit_loss, quantized_list = self.vq(92            x, n_q=n_q, layers=layers93        )94        return quantized, codes, torch.mean(commit_loss), quantized_list95 96    def encode(97        self, x: torch.Tensor, n_q: tp.Optional[int] = None, st: tp.Optional[int] = None98    ) -> torch.Tensor:99        """Encode a given input tensor with the specified sample rate at the given bandwidth.100        The RVQ encode method sets the appropriate number of quantizer to use101        and returns indices for each quantizer.102        Args:103            x (torch.Tensor): Input tensor.104            n_q (int): Number of quantizer used to quantize. Default: All quantizers.105            st (int): Start to encode input from which layers. Default: 0.106        """107        n_q = n_q if n_q else self.n_q108        st = st or 0109        codes = self.vq.encode(x, n_q=n_q, st=st)110        return codes111 112    def decode(self, codes: torch.Tensor, st: int = 0) -> torch.Tensor:113        """Decode the given codes to the quantized representation.114        Args:115            codes (torch.Tensor): Input indices for each quantizer.116            st (int): Start to decode input codes from which layers. Default: 0.117        """118        quantized = self.vq.decode(codes, st=st)119        return quantized120