diffusers/community-pipelines-mirror

1# Copyright 2024 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15from dataclasses import dataclass16from typing import Any, Callable, Dict, List, Optional, Tuple, Union17 18import numpy as np19import PIL.Image20import torch21import torch.nn.functional as F22import torchvision.transforms as T23from gmflow.gmflow import GMFlow24from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer25 26from diffusers.image_processor import VaeImageProcessor27from diffusers.models import AutoencoderKL, ControlNetModel, UNet2DConditionModel28from diffusers.models.attention_processor import Attention, AttnProcessor29from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel30from diffusers.pipelines.controlnet.pipeline_controlnet_img2img import StableDiffusionControlNetImg2ImgPipeline31from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker32from diffusers.schedulers import KarrasDiffusionSchedulers33from diffusers.utils import BaseOutput, deprecate, logging34from diffusers.utils.torch_utils import is_compiled_module, randn_tensor35 36 37logger = logging.get_logger(__name__)  # pylint: disable=invalid-name38 39 40def coords_grid(b, h, w, homogeneous=False, device=None):41    y, x = torch.meshgrid(torch.arange(h), torch.arange(w))  # [H, W]42 43    stacks = [x, y]44 45    if homogeneous:46        ones = torch.ones_like(x)  # [H, W]47        stacks.append(ones)48 49    grid = torch.stack(stacks, dim=0).float()  # [2, H, W] or [3, H, W]50 51    grid = grid[None].repeat(b, 1, 1, 1)  # [B, 2, H, W] or [B, 3, H, W]52 53    if device is not None:54        grid = grid.to(device)55 56    return grid57 58 59def bilinear_sample(img, sample_coords, mode="bilinear", padding_mode="zeros", return_mask=False):60    # img: [B, C, H, W]61    # sample_coords: [B, 2, H, W] in image scale62    if sample_coords.size(1) != 2:  # [B, H, W, 2]63        sample_coords = sample_coords.permute(0, 3, 1, 2)64 65    b, _, h, w = sample_coords.shape66 67    # Normalize to [-1, 1]68    x_grid = 2 * sample_coords[:, 0] / (w - 1) - 169    y_grid = 2 * sample_coords[:, 1] / (h - 1) - 170 71    grid = torch.stack([x_grid, y_grid], dim=-1)  # [B, H, W, 2]72 73    img = F.grid_sample(img, grid, mode=mode, padding_mode=padding_mode, align_corners=True)74 75    if return_mask:76        mask = (x_grid >= -1) & (y_grid >= -1) & (x_grid <= 1) & (y_grid <= 1)  # [B, H, W]77 78        return img, mask79 80    return img81 82 83def flow_warp(feature, flow, mask=False, mode="bilinear", padding_mode="zeros"):84    b, c, h, w = feature.size()85    assert flow.size(1) == 286 87    grid = coords_grid(b, h, w).to(flow.device) + flow  # [B, 2, H, W]88    grid = grid.to(feature.dtype)89    return bilinear_sample(feature, grid, mode=mode, padding_mode=padding_mode, return_mask=mask)90 91 92def forward_backward_consistency_check(fwd_flow, bwd_flow, alpha=0.01, beta=0.5):93    # fwd_flow, bwd_flow: [B, 2, H, W]94    # alpha and beta values are following UnFlow95    # (https://arxiv.org/abs/1711.07837)96    assert fwd_flow.dim() == 4 and bwd_flow.dim() == 497    assert fwd_flow.size(1) == 2 and bwd_flow.size(1) == 298    flow_mag = torch.norm(fwd_flow, dim=1) + torch.norm(bwd_flow, dim=1)  # [B, H, W]99 100    warped_bwd_flow = flow_warp(bwd_flow, fwd_flow)  # [B, 2, H, W]101    warped_fwd_flow = flow_warp(fwd_flow, bwd_flow)  # [B, 2, H, W]102 103    diff_fwd = torch.norm(fwd_flow + warped_bwd_flow, dim=1)  # [B, H, W]104    diff_bwd = torch.norm(bwd_flow + warped_fwd_flow, dim=1)105 106    threshold = alpha * flow_mag + beta107 108    fwd_occ = (diff_fwd > threshold).float()  # [B, H, W]109    bwd_occ = (diff_bwd > threshold).float()110 111    return fwd_occ, bwd_occ112 113 114@torch.no_grad()115def get_warped_and_mask(flow_model, image1, image2, image3=None, pixel_consistency=False, device=None):116    if image3 is None:117        image3 = image1118    padder = InputPadder(image1.shape, padding_factor=8)119    image1, image2 = padder.pad(image1[None].to(device), image2[None].to(device))120    results_dict = flow_model(121        image1, image2, attn_splits_list=[2], corr_radius_list=[-1], prop_radius_list=[-1], pred_bidir_flow=True122    )123    flow_pr = results_dict["flow_preds"][-1]  # [B, 2, H, W]124    fwd_flow = padder.unpad(flow_pr[0]).unsqueeze(0)  # [1, 2, H, W]125    bwd_flow = padder.unpad(flow_pr[1]).unsqueeze(0)  # [1, 2, H, W]126    fwd_occ, bwd_occ = forward_backward_consistency_check(fwd_flow, bwd_flow)  # [1, H, W] float127    if pixel_consistency:128        warped_image1 = flow_warp(image1, bwd_flow)129        bwd_occ = torch.clamp(130            bwd_occ + (abs(image2 - warped_image1).mean(dim=1) > 255 * 0.25).float(), 0, 1131        ).unsqueeze(0)132    warped_results = flow_warp(image3, bwd_flow)133    return warped_results, bwd_occ, bwd_flow134 135 136blur = T.GaussianBlur(kernel_size=(9, 9), sigma=(18, 18))137 138 139@dataclass140class TextToVideoSDPipelineOutput(BaseOutput):141    """142    Output class for text-to-video pipelines.143 144    Args:145        frames (`List[np.ndarray]` or `torch.Tensor`)146            List of denoised frames (essentially images) as NumPy arrays of shape `(height, width, num_channels)` or as147            a `torch` tensor. The length of the list denotes the video length (the number of frames).148    """149 150    frames: Union[List[np.ndarray], torch.Tensor]151 152 153@torch.no_grad()154def find_flat_region(mask):155    device = mask.device156    kernel_x = torch.Tensor([[-1, 0, 1], [-1, 0, 1], [-1, 0, 1]]).unsqueeze(0).unsqueeze(0).to(device)157    kernel_y = torch.Tensor([[-1, -1, -1], [0, 0, 0], [1, 1, 1]]).unsqueeze(0).unsqueeze(0).to(device)158    mask_ = F.pad(mask.unsqueeze(0), (1, 1, 1, 1), mode="replicate")159 160    grad_x = torch.nn.functional.conv2d(mask_, kernel_x)161    grad_y = torch.nn.functional.conv2d(mask_, kernel_y)162    return ((abs(grad_x) + abs(grad_y)) == 0).float()[0]163 164 165class AttnState:166    STORE = 0167    LOAD = 1168    LOAD_AND_STORE_PREV = 2169 170    def __init__(self):171        self.reset()172 173    @property174    def state(self):175        return self.__state176 177    @property178    def timestep(self):179        return self.__timestep180 181    def set_timestep(self, t):182        self.__timestep = t183 184    def reset(self):185        self.__state = AttnState.STORE186        self.__timestep = 0187 188    def to_load(self):189        self.__state = AttnState.LOAD190 191    def to_load_and_store_prev(self):192        self.__state = AttnState.LOAD_AND_STORE_PREV193 194 195class CrossFrameAttnProcessor(AttnProcessor):196    """197    Cross frame attention processor. Each frame attends the first frame and previous frame.198 199    Args:200        attn_state: Whether the model is processing the first frame or an intermediate frame201    """202 203    def __init__(self, attn_state: AttnState):204        super().__init__()205        self.attn_state = attn_state206        self.first_maps = {}207        self.prev_maps = {}208 209    def __call__(self, attn: Attention, hidden_states, encoder_hidden_states=None, attention_mask=None, temb=None):210        # Is self attention211        if encoder_hidden_states is None:212            t = self.attn_state.timestep213            if self.attn_state.state == AttnState.STORE:214                self.first_maps[t] = hidden_states.detach()215                self.prev_maps[t] = hidden_states.detach()216                res = super().__call__(attn, hidden_states, encoder_hidden_states, attention_mask, temb)217            else:218                if self.attn_state.state == AttnState.LOAD_AND_STORE_PREV:219                    tmp = hidden_states.detach()220                cross_map = torch.cat((self.first_maps[t], self.prev_maps[t]), dim=1)221                res = super().__call__(attn, hidden_states, cross_map, attention_mask, temb)222                if self.attn_state.state == AttnState.LOAD_AND_STORE_PREV:223                    self.prev_maps[t] = tmp224        else:225            res = super().__call__(attn, hidden_states, encoder_hidden_states, attention_mask, temb)226 227        return res228 229 230def prepare_image(image):231    if isinstance(image, torch.Tensor):232        # Batch single image233        if image.ndim == 3:234            image = image.unsqueeze(0)235 236        image = image.to(dtype=torch.float32)237    else:238        # preprocess image239        if isinstance(image, (PIL.Image.Image, np.ndarray)):240            image = [image]241 242        if isinstance(image, list) and isinstance(image[0], PIL.Image.Image):243            image = [np.array(i.convert("RGB"))[None, :] for i in image]244            image = np.concatenate(image, axis=0)245        elif isinstance(image, list) and isinstance(image[0], np.ndarray):246            image = np.concatenate([i[None, :] for i in image], axis=0)247 248        image = image.transpose(0, 3, 1, 2)249        image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0250 251    return image252 253 254class RerenderAVideoPipeline(StableDiffusionControlNetImg2ImgPipeline):255    r"""256    Pipeline for video-to-video translation using Stable Diffusion with Rerender Algorithm.257 258    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the259    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)260 261    In addition the pipeline inherits the following loading methods:262        - *Textual-Inversion*: [`loaders.TextualInversionLoaderMixin.load_textual_inversion`]263 264    Args:265        vae ([`AutoencoderKL`]):266            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.267        text_encoder ([`CLIPTextModel`]):268            Frozen text-encoder. Stable Diffusion uses the text portion of269            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically270            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.271        tokenizer (`CLIPTokenizer`):272            Tokenizer of class273            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).274        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.275        controlnet ([`ControlNetModel`] or `List[ControlNetModel]`):276            Provides additional conditioning to the unet during the denoising process. If you set multiple ControlNets277            as a list, the outputs from each ControlNet are added together to create one combined additional278            conditioning.279        scheduler ([`SchedulerMixin`]):280            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of281            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].282        safety_checker ([`StableDiffusionSafetyChecker`]):283            Classification module that estimates whether generated images could be considered offensive or harmful.284            Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details.285        feature_extractor ([`CLIPImageProcessor`]):286            Model that extracts features from generated images to be used as inputs for the `safety_checker`.287    """288 289    _optional_components = ["safety_checker", "feature_extractor"]290 291    def __init__(292        self,293        vae: AutoencoderKL,294        text_encoder: CLIPTextModel,295        tokenizer: CLIPTokenizer,296        unet: UNet2DConditionModel,297        controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel],298        scheduler: KarrasDiffusionSchedulers,299        safety_checker: StableDiffusionSafetyChecker,300        feature_extractor: CLIPImageProcessor,301        image_encoder=None,302        requires_safety_checker: bool = True,303        device=None,304    ):305        super().__init__(306            vae,307            text_encoder,308            tokenizer,309            unet,310            controlnet,311            scheduler,312            safety_checker,313            feature_extractor,314            image_encoder,315            requires_safety_checker,316        )317        self.to(device)318 319        if safety_checker is None and requires_safety_checker:320            logger.warning(321                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"322                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"323                " results in services or applications open to the public. Both the diffusers team and Hugging Face"324                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"325                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"326                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."327            )328 329        if safety_checker is not None and feature_extractor is None:330            raise ValueError(331                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"332                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."333            )334 335        if isinstance(controlnet, (list, tuple)):336            controlnet = MultiControlNetModel(controlnet)337 338        self.register_modules(339            vae=vae,340            text_encoder=text_encoder,341            tokenizer=tokenizer,342            unet=unet,343            controlnet=controlnet,344            scheduler=scheduler,345            safety_checker=safety_checker,346            feature_extractor=feature_extractor,347        )348        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)349        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True)350        self.control_image_processor = VaeImageProcessor(351            vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True, do_normalize=False352        )353        self.register_to_config(requires_safety_checker=requires_safety_checker)354        self.attn_state = AttnState()355        attn_processor_dict = {}356        for k in unet.attn_processors.keys():357            if k.startswith("up"):358                attn_processor_dict[k] = CrossFrameAttnProcessor(self.attn_state)359            else:360                attn_processor_dict[k] = AttnProcessor()361 362        self.unet.set_attn_processor(attn_processor_dict)363 364        flow_model = GMFlow(365            feature_channels=128,366            num_scales=1,367            upsample_factor=8,368            num_head=1,369            attention_type="swin",370            ffn_dim_expansion=4,371            num_transformer_layers=6,372        ).to(self.device)373 374        checkpoint = torch.utils.model_zoo.load_url(375            "https://huggingface.co/Anonymous-sub/Rerender/resolve/main/models/gmflow_sintel-0c07dcb3.pth",376            map_location=lambda storage, loc: storage,377        )378        weights = checkpoint["model"] if "model" in checkpoint else checkpoint379        flow_model.load_state_dict(weights, strict=False)380        flow_model.eval()381        self.flow_model = flow_model382 383    # Modified from src/diffusers/pipelines/controlnet/pipeline_controlnet.StableDiffusionControlNetImg2ImgPipeline.check_inputs384    def check_inputs(385        self,386        prompt,387        callback_steps,388        negative_prompt=None,389        prompt_embeds=None,390        negative_prompt_embeds=None,391        controlnet_conditioning_scale=1.0,392        control_guidance_start=0.0,393        control_guidance_end=1.0,394    ):395        if (callback_steps is None) or (396            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)397        ):398            raise ValueError(399                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"400                f" {type(callback_steps)}."401            )402 403        if prompt is not None and prompt_embeds is not None:404            raise ValueError(405                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"406                " only forward one of the two."407            )408        elif prompt is None and prompt_embeds is None:409            raise ValueError(410                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."411            )412        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):413            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")414 415        if negative_prompt is not None and negative_prompt_embeds is not None:416            raise ValueError(417                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"418                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."419            )420 421        if prompt_embeds is not None and negative_prompt_embeds is not None:422            if prompt_embeds.shape != negative_prompt_embeds.shape:423                raise ValueError(424                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"425                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"426                    f" {negative_prompt_embeds.shape}."427                )428 429        # `prompt` needs more sophisticated handling when there are multiple430        # conditionings.431        if isinstance(self.controlnet, MultiControlNetModel):432            if isinstance(prompt, list):433                logger.warning(434                    f"You have {len(self.controlnet.nets)} ControlNets and you have passed {len(prompt)}"435                    " prompts. The conditionings will be fixed across the prompts."436                )437 438        is_compiled = hasattr(F, "scaled_dot_product_attention") and isinstance(439            self.controlnet, torch._dynamo.eval_frame.OptimizedModule440        )441 442        # Check `controlnet_conditioning_scale`443        if (444            isinstance(self.controlnet, ControlNetModel)445            or is_compiled446            and isinstance(self.controlnet._orig_mod, ControlNetModel)447        ):448            if not isinstance(controlnet_conditioning_scale, float):449                raise TypeError("For single controlnet: `controlnet_conditioning_scale` must be type `float`.")450        elif (451            isinstance(self.controlnet, MultiControlNetModel)452            or is_compiled453            and isinstance(self.controlnet._orig_mod, MultiControlNetModel)454        ):455            if isinstance(controlnet_conditioning_scale, list):456                if any(isinstance(i, list) for i in controlnet_conditioning_scale):457                    raise ValueError("A single batch of multiple conditionings are supported at the moment.")458            elif isinstance(controlnet_conditioning_scale, list) and len(controlnet_conditioning_scale) != len(459                self.controlnet.nets460            ):461                raise ValueError(462                    "For multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have"463                    " the same length as the number of controlnets"464                )465        else:466            assert False467 468        if len(control_guidance_start) != len(control_guidance_end):469            raise ValueError(470                f"`control_guidance_start` has {len(control_guidance_start)} elements, but `control_guidance_end` has {len(control_guidance_end)} elements. Make sure to provide the same number of elements to each list."471            )472 473        if isinstance(self.controlnet, MultiControlNetModel):474            if len(control_guidance_start) != len(self.controlnet.nets):475                raise ValueError(476                    f"`control_guidance_start`: {control_guidance_start} has {len(control_guidance_start)} elements but there are {len(self.controlnet.nets)} controlnets available. Make sure to provide {len(self.controlnet.nets)}."477                )478 479        for start, end in zip(control_guidance_start, control_guidance_end):480            if start >= end:481                raise ValueError(482                    f"control guidance start: {start} cannot be larger or equal to control guidance end: {end}."483                )484            if start < 0.0:485                raise ValueError(f"control guidance start: {start} can't be smaller than 0.")486            if end > 1.0:487                raise ValueError(f"control guidance end: {end} can't be larger than 1.0.")488 489    # Copied from diffusers.pipelines.controlnet.pipeline_controlnet.StableDiffusionControlNetPipeline.prepare_image490    def prepare_control_image(491        self,492        image,493        width,494        height,495        batch_size,496        num_images_per_prompt,497        device,498        dtype,499        do_classifier_free_guidance=False,500        guess_mode=False,501    ):502        image = self.control_image_processor.preprocess(image, height=height, width=width).to(dtype=torch.float32)503        image_batch_size = image.shape[0]504 505        if image_batch_size == 1:506            repeat_by = batch_size507        else:508            # image batch size is the same as prompt batch size509            repeat_by = num_images_per_prompt510 511        image = image.repeat_interleave(repeat_by, dim=0)512 513        image = image.to(device=device, dtype=dtype)514 515        if do_classifier_free_guidance and not guess_mode:516            image = torch.cat([image] * 2)517 518        return image519 520    # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps521    def get_timesteps(self, num_inference_steps, strength, device):522        # get the original timestep using init_timestep523        init_timestep = min(int(num_inference_steps * strength), num_inference_steps)524 525        t_start = max(num_inference_steps - init_timestep, 0)526        timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]527 528        return timesteps, num_inference_steps - t_start529 530    # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.prepare_latents531    def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None):532        if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)):533            raise ValueError(534                f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}"535            )536 537        image = image.to(device=device, dtype=dtype)538 539        batch_size = batch_size * num_images_per_prompt540 541        if image.shape[1] == 4:542            init_latents = image543 544        else:545            if isinstance(generator, list) and len(generator) != batch_size:546                raise ValueError(547                    f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"548                    f" size of {batch_size}. Make sure the batch size matches the length of the generators."549                )550 551            elif isinstance(generator, list):552                init_latents = [553                    self.vae.encode(image[i : i + 1]).latent_dist.sample(generator[i]) for i in range(batch_size)554                ]555                init_latents = torch.cat(init_latents, dim=0)556            else:557                init_latents = self.vae.encode(image).latent_dist.sample(generator)558 559            init_latents = self.vae.config.scaling_factor * init_latents560 561        if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0:562            # expand init_latents for batch_size563            deprecation_message = (564                f"You have passed {batch_size} text prompts (`prompt`), but only {init_latents.shape[0]} initial"565                " images (`image`). Initial images are now duplicating to match the number of text prompts. Note"566                " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update"567                " your script to pass as many initial images as text prompts to suppress this warning."568            )569            deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False)570            additional_image_per_prompt = batch_size // init_latents.shape[0]571            init_latents = torch.cat([init_latents] * additional_image_per_prompt, dim=0)572        elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0:573            raise ValueError(574                f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts."575            )576        else:577            init_latents = torch.cat([init_latents], dim=0)578 579        shape = init_latents.shape580        noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)581 582        # get latents583        init_latents = self.scheduler.add_noise(init_latents, noise, timestep)584        latents = init_latents585 586        return latents587 588    @torch.no_grad()589    def __call__(590        self,591        prompt: Union[str, List[str]] = None,592        frames: Union[List[np.ndarray], torch.Tensor] = None,593        control_frames: Union[List[np.ndarray], torch.Tensor] = None,594        strength: float = 0.8,595        num_inference_steps: int = 50,596        guidance_scale: float = 7.5,597        negative_prompt: Optional[Union[str, List[str]]] = None,598        eta: float = 0.0,599        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,600        latents: Optional[torch.Tensor] = None,601        prompt_embeds: Optional[torch.Tensor] = None,602        negative_prompt_embeds: Optional[torch.Tensor] = None,603        output_type: Optional[str] = "pil",604        return_dict: bool = True,605        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,606        callback_steps: int = 1,607        cross_attention_kwargs: Optional[Dict[str, Any]] = None,608        controlnet_conditioning_scale: Union[float, List[float]] = 0.8,609        guess_mode: bool = False,610        control_guidance_start: Union[float, List[float]] = 0.0,611        control_guidance_end: Union[float, List[float]] = 1.0,612        warp_start: Union[float, List[float]] = 0.0,613        warp_end: Union[float, List[float]] = 0.3,614        mask_start: Union[float, List[float]] = 0.5,615        mask_end: Union[float, List[float]] = 0.8,616        smooth_boundary: bool = True,617        mask_strength: Union[float, List[float]] = 0.5,618        inner_strength: Union[float, List[float]] = 0.9,619    ):620        r"""621        Function invoked when calling the pipeline for generation.622 623        Args:624            prompt (`str` or `List[str]`, *optional*):625                The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.626                instead.627            frames (`List[np.ndarray]` or `torch.Tensor`): The input images to be used as the starting point for the image generation process.628            control_frames (`List[np.ndarray]` or `torch.Tensor`): The ControlNet input images condition to provide guidance to the `unet` for generation.629            strength ('float'): SDEdit strength.630            num_inference_steps (`int`, *optional*, defaults to 50):631                The number of denoising steps. More denoising steps usually lead to a higher quality image at the632                expense of slower inference.633            guidance_scale (`float`, *optional*, defaults to 7.5):634                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).635                `guidance_scale` is defined as `w` of equation 2. of [Imagen636                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >637                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,638                usually at the expense of lower image quality.639            negative_prompt (`str` or `List[str]`, *optional*):640                The prompt or prompts not to guide the image generation. If not defined, one has to pass641                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is642                less than `1`).643            eta (`float`, *optional*, defaults to 0.0):644                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to645                [`schedulers.DDIMScheduler`], will be ignored for others.646            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):647                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)648                to make generation deterministic.649            latents (`torch.Tensor`, *optional*):650                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image651                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents652                tensor will ge generated by sampling using the supplied random `generator`.653            prompt_embeds (`torch.Tensor`, *optional*):654                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not655                provided, text embeddings will be generated from `prompt` input argument.656            negative_prompt_embeds (`torch.Tensor`, *optional*):657                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt658                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input659                argument.660            output_type (`str`, *optional*, defaults to `"pil"`):661                The output format of the generate image. Choose between662                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.663            return_dict (`bool`, *optional*, defaults to `True`):664                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a665                plain tuple.666            callback (`Callable`, *optional*):667                A function that will be called every `callback_steps` steps during inference. The function will be668                called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.669            callback_steps (`int`, *optional*, defaults to 1):670                The frequency at which the `callback` function will be called. If not specified, the callback will be671                called at every step.672            cross_attention_kwargs (`dict`, *optional*):673                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under674                `self.processor` in675                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).676            controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0):677                The outputs of the controlnet are multiplied by `controlnet_conditioning_scale` before they are added678                to the residual in the original unet. If multiple ControlNets are specified in init, you can set the679                corresponding scale as a list. Note that by default, we use a smaller conditioning scale for inpainting680                than for [`~StableDiffusionControlNetPipeline.__call__`].681            guess_mode (`bool`, *optional*, defaults to `False`):682                In this mode, the ControlNet encoder will try best to recognize the content of the input image even if683                you remove all prompts. The `guidance_scale` between 3.0 and 5.0 is recommended.684            control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0):685                The percentage of total steps at which the controlnet starts applying.686            control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0):687                The percentage of total steps at which the controlnet stops applying.688            warp_start (`float`): Shape-aware fusion start timestep.689            warp_end (`float`): Shape-aware fusion end timestep.690            mask_start (`float`): Pixel-aware fusion start timestep.691            mask_end (`float`):Pixel-aware fusion end timestep.692            smooth_boundary (`bool`): Smooth fusion boundary. Set `True` to prevent artifacts at boundary.693            mask_strength (`float`): Pixel-aware fusion strength.694            inner_strength (`float`): Pixel-aware fusion detail level.695 696        Examples:697 698        Returns:699            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:700            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.701            When returning a tuple, the first element is a list with the generated images, and the second element is a702            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"703            (nsfw) content, according to the `safety_checker`.704        """705        controlnet = self.controlnet._orig_mod if is_compiled_module(self.controlnet) else self.controlnet706 707        # align format for control guidance708        if not isinstance(control_guidance_start, list) and isinstance(control_guidance_end, list):709            control_guidance_start = len(control_guidance_end) * [control_guidance_start]710        elif not isinstance(control_guidance_end, list) and isinstance(control_guidance_start, list):711            control_guidance_end = len(control_guidance_start) * [control_guidance_end]712        elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list):713            mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1714            control_guidance_start, control_guidance_end = (715                mult * [control_guidance_start],716                mult * [control_guidance_end],717            )718 719        # 1. Check inputs. Raise error if not correct720        self.check_inputs(721            prompt,722            callback_steps,723            negative_prompt,724            prompt_embeds,725            negative_prompt_embeds,726            controlnet_conditioning_scale,727            control_guidance_start,728            control_guidance_end,729        )730 731        # 2. Define call parameters732        # Currently we only support 1 prompt733        if prompt is not None and isinstance(prompt, str):734            batch_size = 1735        elif prompt is not None and isinstance(prompt, list):736            assert False737        else:738            assert False739        num_images_per_prompt = 1740 741        device = self._execution_device742        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)743        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`744        # corresponds to doing no classifier free guidance.745        do_classifier_free_guidance = guidance_scale > 1.0746 747        if isinstance(controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float):748            controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnet.nets)749 750        global_pool_conditions = (751            controlnet.config.global_pool_conditions752            if isinstance(controlnet, ControlNetModel)753            else controlnet.nets[0].config.global_pool_conditions754        )755        guess_mode = guess_mode or global_pool_conditions756 757        # 3. Encode input prompt758        text_encoder_lora_scale = (759            cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None760        )761        prompt_embeds = self._encode_prompt(762            prompt,763            device,764            num_images_per_prompt,765            do_classifier_free_guidance,766            negative_prompt,767            prompt_embeds=prompt_embeds,768            negative_prompt_embeds=negative_prompt_embeds,769            lora_scale=text_encoder_lora_scale,770        )771 772        # 4. Process the first frame773        height, width = None, None774        output_frames = []775        self.attn_state.reset()776 777        # 4.1 prepare frames778        image = self.image_processor.preprocess(frames[0]).to(dtype=torch.float32)779        first_image = image[0]  # C, H, W780 781        # 4.2 Prepare controlnet_conditioning_image782        # Currently we only support single control783        if isinstance(controlnet, ControlNetModel):784            control_image = self.prepare_control_image(785                image=control_frames[0],786                width=width,787                height=height,788                batch_size=batch_size,789                num_images_per_prompt=1,790                device=device,791                dtype=controlnet.dtype,792                do_classifier_free_guidance=do_classifier_free_guidance,793                guess_mode=guess_mode,794            )795        else:796            assert False797 798        # 4.3 Prepare timesteps799        self.scheduler.set_timesteps(num_inference_steps, device=device)800        timesteps, cur_num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)801        latent_timestep = timesteps[:1].repeat(batch_size)802 803        # 4.4 Prepare latent variables804        latents = self.prepare_latents(805            image,806            latent_timestep,807            batch_size,808            num_images_per_prompt,809            prompt_embeds.dtype,810            device,811            generator,812        )813 814        # 4.5 Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline815        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)816 817        # 4.6 Create tensor stating which controlnets to keep818        controlnet_keep = []819        for i in range(len(timesteps)):820            keeps = [821                1.0 - float(i / len(timesteps) < s or (i + 1) / len(timesteps) > e)822                for s, e in zip(control_guidance_start, control_guidance_end)823            ]824            controlnet_keep.append(keeps[0] if isinstance(controlnet, ControlNetModel) else keeps)825 826        first_x0_list = []827 828        # 4.7 Denoising loop829        num_warmup_steps = len(timesteps) - cur_num_inference_steps * self.scheduler.order830        with self.progress_bar(total=cur_num_inference_steps) as progress_bar:831            for i, t in enumerate(timesteps):832                self.attn_state.set_timestep(t.item())833 834                # expand the latents if we are doing classifier free guidance835                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents836                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)837 838                # controlnet(s) inference839                if guess_mode and do_classifier_free_guidance:840                    # Infer ControlNet only for the conditional batch.841                    control_model_input = latents842                    control_model_input = self.scheduler.scale_model_input(control_model_input, t)843                    controlnet_prompt_embeds = prompt_embeds.chunk(2)[1]844                else:845                    control_model_input = latent_model_input846                    controlnet_prompt_embeds = prompt_embeds847 848                if isinstance(controlnet_keep[i], list):849                    cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])]850                else:851                    controlnet_cond_scale = controlnet_conditioning_scale852                    if isinstance(controlnet_cond_scale, list):853                        controlnet_cond_scale = controlnet_cond_scale[0]854                    cond_scale = controlnet_cond_scale * controlnet_keep[i]855 856                down_block_res_samples, mid_block_res_sample = self.controlnet(857                    control_model_input,858                    t,859                    encoder_hidden_states=controlnet_prompt_embeds,860                    controlnet_cond=control_image,861                    conditioning_scale=cond_scale,862                    guess_mode=guess_mode,863                    return_dict=False,864                )865 866                if guess_mode and do_classifier_free_guidance:867                    # Infered ControlNet only for the conditional batch.868                    # To apply the output of ControlNet to both the unconditional and conditional batches,869                    # add 0 to the unconditional batch to keep it unchanged.870                    down_block_res_samples = [torch.cat([torch.zeros_like(d), d]) for d in down_block_res_samples]871                    mid_block_res_sample = torch.cat([torch.zeros_like(mid_block_res_sample), mid_block_res_sample])872 873                # predict the noise residual874                noise_pred = self.unet(875                    latent_model_input,876                    t,877                    encoder_hidden_states=prompt_embeds,878                    cross_attention_kwargs=cross_attention_kwargs,879                    down_block_additional_residuals=down_block_res_samples,880                    mid_block_additional_residual=mid_block_res_sample,881                    return_dict=False,882                )[0]883 884                # perform guidance885                if do_classifier_free_guidance:886                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)887                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)888 889                alpha_prod_t = self.scheduler.alphas_cumprod[t]890                beta_prod_t = 1 - alpha_prod_t891                pred_x0 = (latents - beta_prod_t ** (0.5) * noise_pred) / alpha_prod_t ** (0.5)892                first_x0 = pred_x0.detach()893                first_x0_list.append(first_x0)894 895                # compute the previous noisy sample x_t -> x_t-1896                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]897 898                # call the callback, if provided899                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):900                    progress_bar.update()901                    if callback is not None and i % callback_steps == 0:902                        callback(i, t, latents)903 904        if not output_type == "latent":905            image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]906        else:907            image = latents908 909        first_result = image910        prev_result = image911        do_denormalize = [True] * image.shape[0]912        image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)913 914        output_frames.append(image[0])915 916        # 5. Process each frame917        for idx in range(1, len(frames)):918            image = frames[idx]919            prev_image = frames[idx - 1]920            control_image = control_frames[idx]921            # 5.1 prepare frames922            image = self.image_processor.preprocess(image).to(dtype=torch.float32)923            prev_image = self.image_processor.preprocess(prev_image).to(dtype=torch.float32)924 925            warped_0, bwd_occ_0, bwd_flow_0 = get_warped_and_mask(926                self.flow_model, first_image, image[0], first_result, False, self.device927            )928            blend_mask_0 = blur(F.max_pool2d(bwd_occ_0, kernel_size=9, stride=1, padding=4))929            blend_mask_0 = torch.clamp(blend_mask_0 + bwd_occ_0, 0, 1)930 931            warped_pre, bwd_occ_pre, bwd_flow_pre = get_warped_and_mask(932                self.flow_model, prev_image[0], image[0], prev_result, False, self.device933            )934            blend_mask_pre = blur(F.max_pool2d(bwd_occ_pre, kernel_size=9, stride=1, padding=4))935            blend_mask_pre = torch.clamp(blend_mask_pre + bwd_occ_pre, 0, 1)936 937            warp_mask = 1 - F.max_pool2d(blend_mask_0, kernel_size=8)938            warp_flow = F.interpolate(bwd_flow_0 / 8.0, scale_factor=1.0 / 8, mode="bilinear")939 940            # 5.2 Prepare controlnet_conditioning_image941            # Currently we only support single control942            if isinstance(controlnet, ControlNetModel):943                control_image = self.prepare_control_image(944                    image=control_image,945                    width=width,946                    height=height,947                    batch_size=batch_size,948                    num_images_per_prompt=1,949                    device=device,950                    dtype=controlnet.dtype,951                    do_classifier_free_guidance=do_classifier_free_guidance,952                    guess_mode=guess_mode,953                )954            else:955                assert False956 957            # 5.3 Prepare timesteps958            self.scheduler.set_timesteps(num_inference_steps, device=device)959            timesteps, cur_num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)960            latent_timestep = timesteps[:1].repeat(batch_size)961 962            skip_t = int(num_inference_steps * (1 - strength))963            warp_start_t = int(warp_start * num_inference_steps)964            warp_end_t = int(warp_end * num_inference_steps)965            mask_start_t = int(mask_start * num_inference_steps)966            mask_end_t = int(mask_end * num_inference_steps)967 968            # 5.4 Prepare latent variables969            init_latents = self.prepare_latents(970                image,971                latent_timestep,972                batch_size,973                num_images_per_prompt,974                prompt_embeds.dtype,975                device,976                generator,977            )978 979            # 5.5 Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline980            extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)981 982            # 5.6 Create tensor stating which controlnets to keep983            controlnet_keep = []984            for i in range(len(timesteps)):985                keeps = [986                    1.0 - float(i / len(timesteps) < s or (i + 1) / len(timesteps) > e)987                    for s, e in zip(control_guidance_start, control_guidance_end)988                ]989                controlnet_keep.append(keeps[0] if isinstance(controlnet, ControlNetModel) else keeps)990 991            # 5.7 Denoising loop992            num_warmup_steps = len(timesteps) - cur_num_inference_steps * self.scheduler.order993 994            def denoising_loop(latents, mask=None, xtrg=None, noise_rescale=None):995                dir_xt = 0996                latents_dtype = latents.dtype997                with self.progress_bar(total=cur_num_inference_steps) as progress_bar:998                    for i, t in enumerate(timesteps):999                        self.attn_state.set_timestep(t.item())1000                        if i + skip_t >= mask_start_t and i + skip_t <= mask_end_t and xtrg is not None:1001                            rescale = torch.maximum(1.0 - mask, (1 - mask**2) ** 0.5 * inner_strength)1002                            if noise_rescale is not None:1003                                rescale = (1.0 - mask) * (1 - noise_rescale) + rescale * noise_rescale1004                            noise = randn_tensor(xtrg.shape, generator=generator, device=device, dtype=xtrg.dtype)1005                            latents_ref = self.scheduler.add_noise(xtrg, noise, t)1006                            latents = latents_ref * mask + (1.0 - mask) * (latents - dir_xt) + rescale * dir_xt1007                            latents = latents.to(latents_dtype)1008 1009                        # expand the latents if we are doing classifier free guidance1010                        latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents1011                        latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)1012 1013                        # controlnet(s) inference1014                        if guess_mode and do_classifier_free_guidance:1015                            # Infer ControlNet only for the conditional batch.1016                            control_model_input = latents1017                            control_model_input = self.scheduler.scale_model_input(control_model_input, t)1018                            controlnet_prompt_embeds = prompt_embeds.chunk(2)[1]1019                        else:1020                            control_model_input = latent_model_input1021                            controlnet_prompt_embeds = prompt_embeds1022 1023                        if isinstance(controlnet_keep[i], list):1024                            cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])]1025                        else:1026                            controlnet_cond_scale = controlnet_conditioning_scale1027                            if isinstance(controlnet_cond_scale, list):1028                                controlnet_cond_scale = controlnet_cond_scale[0]1029                            cond_scale = controlnet_cond_scale * controlnet_keep[i]1030                        down_block_res_samples, mid_block_res_sample = self.controlnet(1031                            control_model_input,1032                            t,1033                            encoder_hidden_states=controlnet_prompt_embeds,1034                            controlnet_cond=control_image,1035                            conditioning_scale=cond_scale,1036                            guess_mode=guess_mode,1037                            return_dict=False,1038                        )1039 1040                        if guess_mode and do_classifier_free_guidance:1041                            # Infered ControlNet only for the conditional batch.1042                            # To apply the output of ControlNet to both the unconditional and conditional batches,1043                            # add 0 to the unconditional batch to keep it unchanged.1044                            down_block_res_samples = [1045                                torch.cat([torch.zeros_like(d), d]) for d in down_block_res_samples1046                            ]1047                            mid_block_res_sample = torch.cat(1048                                [torch.zeros_like(mid_block_res_sample), mid_block_res_sample]1049                            )1050 1051                        # predict the noise residual1052                        noise_pred = self.unet(1053                            latent_model_input,1054                            t,1055                            encoder_hidden_states=prompt_embeds,1056                            cross_attention_kwargs=cross_attention_kwargs,1057                            down_block_additional_residuals=down_block_res_samples,1058                            mid_block_additional_residual=mid_block_res_sample,1059                            return_dict=False,1060                        )[0]1061 1062                        # perform guidance1063                        if do_classifier_free_guidance:1064                            noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)1065                            noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)1066 1067                        # Get pred_x0 from scheduler1068                        alpha_prod_t = self.scheduler.alphas_cumprod[t]1069                        beta_prod_t = 1 - alpha_prod_t1070                        pred_x0 = (latents - beta_prod_t ** (0.5) * noise_pred) / alpha_prod_t ** (0.5)1071 1072                        if i + skip_t >= warp_start_t and i + skip_t <= warp_end_t:1073                            # warp x_01074                            pred_x0 = (1075                                flow_warp(first_x0_list[i], warp_flow, mode="nearest") * warp_mask1076                                + (1 - warp_mask) * pred_x01077                            )1078 1079                            # get x_t from x_01080                            latents = self.scheduler.add_noise(pred_x0, noise_pred, t).to(latents_dtype)1081 1082                        prev_t = t - self.scheduler.config.num_train_timesteps // self.scheduler.num_inference_steps1083                        if i == len(timesteps) - 1:1084                            alpha_t_prev = 1.01085                        else:1086                            alpha_t_prev = self.scheduler.alphas_cumprod[prev_t]1087 1088                        dir_xt = (1.0 - alpha_t_prev) ** 0.5 * noise_pred1089 1090                        # compute the previous noisy sample x_t -> x_t-11091                        latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[1092                            01093                        ]1094 1095                        # call the callback, if provided1096                        if i == len(timesteps) - 1 or (1097                            (i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 01098                        ):1099                            progress_bar.update()1100                            if callback is not None and i % callback_steps == 0:1101                                callback(i, t, latents)1102 1103                    return latents1104 1105            if mask_start_t <= mask_end_t:1106                self.attn_state.to_load()1107            else:1108                self.attn_state.to_load_and_store_prev()1109            latents = denoising_loop(init_latents)1110 1111            if mask_start_t <= mask_end_t:1112                direct_result = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]1113 1114                blend_results = (1 - blend_mask_pre) * warped_pre + blend_mask_pre * direct_result1115                blend_results = (1 - blend_mask_0) * warped_0 + blend_mask_0 * blend_results1116 1117                bwd_occ = 1 - torch.clamp(1 - bwd_occ_pre + 1 - bwd_occ_0, 0, 1)1118                blend_mask = blur(F.max_pool2d(bwd_occ, kernel_size=9, stride=1, padding=4))1119                blend_mask = 1 - torch.clamp(blend_mask + bwd_occ, 0, 1)1120 1121                blend_results = blend_results.to(latents.dtype)1122                xtrg = self.vae.encode(blend_results).latent_dist.sample(generator)1123                xtrg = self.vae.config.scaling_factor * xtrg1124                blend_results_rec = self.vae.decode(xtrg / self.vae.config.scaling_factor, return_dict=False)[0]1125                xtrg_rec = self.vae.encode(blend_results_rec).latent_dist.sample(generator)1126                xtrg_rec = self.vae.config.scaling_factor * xtrg_rec1127                xtrg_ = xtrg + (xtrg - xtrg_rec)1128                blend_results_rec_new = self.vae.decode(xtrg_ / self.vae.config.scaling_factor, return_dict=False)[0]1129                tmp = (abs(blend_results_rec_new - blend_results).mean(dim=1, keepdims=True) > 0.25).float()1130 1131                mask_x = F.max_pool2d(1132                    (F.interpolate(tmp, scale_factor=1 / 8.0, mode="bilinear") > 0).float(),1133                    kernel_size=3,1134                    stride=1,1135                    padding=1,1136                )1137 1138                mask = 1 - F.max_pool2d(1 - blend_mask, kernel_size=8)  # * (1-mask_x)1139 1140                if smooth_boundary:1141                    noise_rescale = find_flat_region(mask)1142                else:1143                    noise_rescale = torch.ones_like(mask)1144 1145                xtrg = (xtrg + (1 - mask_x) * (xtrg - xtrg_rec)) * mask1146                xtrg = xtrg.to(latents.dtype)1147 1148                self.scheduler.set_timesteps(num_inference_steps, device=device)1149                timesteps, cur_num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)1150 1151                self.attn_state.to_load_and_store_prev()1152                latents = denoising_loop(init_latents, mask * mask_strength, xtrg, noise_rescale)1153 1154            if not output_type == "latent":1155                image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]1156            else:1157                image = latents1158 1159            prev_result = image1160 1161            do_denormalize = [True] * image.shape[0]1162            image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)1163 1164            output_frames.append(image[0])1165 1166        # Offload last model to CPU1167        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:1168            self.final_offload_hook.offload()1169 1170        if not return_dict:1171            return output_frames1172 1173        return TextToVideoSDPipelineOutput(frames=output_frames)1174 1175 1176class InputPadder:1177    """Pads images such that dimensions are divisible by 8"""1178 1179    def __init__(self, dims, mode="sintel", padding_factor=8):1180        self.ht, self.wd = dims[-2:]1181        pad_ht = (((self.ht // padding_factor) + 1) * padding_factor - self.ht) % padding_factor1182        pad_wd = (((self.wd // padding_factor) + 1) * padding_factor - self.wd) % padding_factor1183        if mode == "sintel":1184            self._pad = [pad_wd // 2, pad_wd - pad_wd // 2, pad_ht // 2, pad_ht - pad_ht // 2]1185        else:1186            self._pad = [pad_wd // 2, pad_wd - pad_wd // 2, 0, pad_ht]1187 1188    def pad(self, *inputs):1189        return [F.pad(x, self._pad, mode="replicate") for x in inputs]1190 1191    def unpad(self, x):1192        ht, wd = x.shape[-2:]1193        c = [self._pad[2], ht - self._pad[3], self._pad[0], wd - self._pad[1]]1194        return x[..., c[0] : c[1], c[2] : c[3]]1195