diffusers/community-pipelines-mirror

1from typing import List, Optional, Tuple, Union2 3import torch4 5from diffusers import DiffusionPipeline6from diffusers.configuration_utils import ConfigMixin7from diffusers.pipelines.pipeline_utils import ImagePipelineOutput8from diffusers.schedulers.scheduling_utils import SchedulerMixin9 10 11class IADBScheduler(SchedulerMixin, ConfigMixin):12    """13    IADBScheduler is a scheduler for the Iterative α-(de)Blending denoising method. It is simple and minimalist.14 15    For more details, see the original paper: https://arxiv.org/abs/2305.03486 and the blog post: https://ggx-research.github.io/publication/2023/05/10/publication-iadb.html16    """17 18    def step(19        self,20        model_output: torch.Tensor,21        timestep: int,22        x_alpha: torch.Tensor,23    ) -> torch.Tensor:24        """25        Predict the sample at the previous timestep by reversing the ODE. Core function to propagate the diffusion26        process from the learned model outputs (most often the predicted noise).27 28        Args:29            model_output (`torch.Tensor`): direct output from learned diffusion model. It is the direction from x0 to x1.30            timestep (`float`): current timestep in the diffusion chain.31            x_alpha (`torch.Tensor`): x_alpha sample for the current timestep32 33        Returns:34            `torch.Tensor`: the sample at the previous timestep35 36        """37        if self.num_inference_steps is None:38            raise ValueError(39                "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"40            )41 42        alpha = timestep / self.num_inference_steps43        alpha_next = (timestep + 1) / self.num_inference_steps44 45        d = model_output46 47        x_alpha = x_alpha + (alpha_next - alpha) * d48 49        return x_alpha50 51    def set_timesteps(self, num_inference_steps: int):52        self.num_inference_steps = num_inference_steps53 54    def add_noise(55        self,56        original_samples: torch.Tensor,57        noise: torch.Tensor,58        alpha: torch.Tensor,59    ) -> torch.Tensor:60        return original_samples * alpha + noise * (1 - alpha)61 62    def __len__(self):63        return self.config.num_train_timesteps64 65 66class IADBPipeline(DiffusionPipeline):67    r"""68    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the69    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)70 71    Parameters:72        unet ([`UNet2DModel`]): U-Net architecture to denoise the encoded image.73        scheduler ([`SchedulerMixin`]):74            A scheduler to be used in combination with `unet` to denoise the encoded image. Can be one of75            [`DDPMScheduler`], or [`DDIMScheduler`].76    """77 78    def __init__(self, unet, scheduler):79        super().__init__()80 81        self.register_modules(unet=unet, scheduler=scheduler)82 83    @torch.no_grad()84    def __call__(85        self,86        batch_size: int = 1,87        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,88        num_inference_steps: int = 50,89        output_type: Optional[str] = "pil",90        return_dict: bool = True,91    ) -> Union[ImagePipelineOutput, Tuple]:92        r"""93        Args:94            batch_size (`int`, *optional*, defaults to 1):95                The number of images to generate.96            num_inference_steps (`int`, *optional*, defaults to 50):97                The number of denoising steps. More denoising steps usually lead to a higher quality image at the98                expense of slower inference.99            output_type (`str`, *optional*, defaults to `"pil"`):100                The output format of the generate image. Choose between101                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.102            return_dict (`bool`, *optional*, defaults to `True`):103                Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple.104 105        Returns:106            [`~pipelines.ImagePipelineOutput`] or `tuple`: [`~pipelines.utils.ImagePipelineOutput`] if `return_dict` is107            True, otherwise a `tuple. When returning a tuple, the first element is a list with the generated images.108        """109 110        # Sample gaussian noise to begin loop111        if isinstance(self.unet.config.sample_size, int):112            image_shape = (113                batch_size,114                self.unet.config.in_channels,115                self.unet.config.sample_size,116                self.unet.config.sample_size,117            )118        else:119            image_shape = (batch_size, self.unet.config.in_channels, *self.unet.config.sample_size)120 121        if isinstance(generator, list) and len(generator) != batch_size:122            raise ValueError(123                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"124                f" size of {batch_size}. Make sure the batch size matches the length of the generators."125            )126 127        image = torch.randn(image_shape, generator=generator, device=self.device, dtype=self.unet.dtype)128 129        # set step values130        self.scheduler.set_timesteps(num_inference_steps)131        x_alpha = image.clone()132        for t in self.progress_bar(range(num_inference_steps)):133            alpha = t / num_inference_steps134 135            # 1. predict noise model_output136            model_output = self.unet(x_alpha, torch.tensor(alpha, device=x_alpha.device)).sample137 138            # 2. step139            x_alpha = self.scheduler.step(model_output, t, x_alpha)140 141        image = (x_alpha * 0.5 + 0.5).clamp(0, 1)142        image = image.cpu().permute(0, 2, 3, 1).numpy()143        if output_type == "pil":144            image = self.numpy_to_pil(image)145 146        if not return_dict:147            return (image,)148 149        return ImagePipelineOutput(images=image)150