diffusers/community-pipelines-mirror

1# Based on stable_diffusion_xl_reference.py and stable_diffusion_controlnet_reference.py2 3import inspect4from typing import Any, Callable, Dict, List, Optional, Tuple, Union5 6import numpy as np7import PIL.Image8import torch9 10from diffusers import StableDiffusionXLControlNetPipeline11from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback12from diffusers.image_processor import PipelineImageInput13from diffusers.models import ControlNetModel14from diffusers.models.attention import BasicTransformerBlock15from diffusers.models.unets.unet_2d_blocks import CrossAttnDownBlock2D, CrossAttnUpBlock2D, DownBlock2D, UpBlock2D16from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel17from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput18from diffusers.utils import PIL_INTERPOLATION, deprecate, logging, replace_example_docstring19from diffusers.utils.torch_utils import is_compiled_module, is_torch_version, randn_tensor20 21 22logger = logging.get_logger(__name__)  # pylint: disable=invalid-name23 24 25EXAMPLE_DOC_STRING = """26    Examples:27        ```py28        >>> # !pip install opencv-python transformers accelerate29        >>> from diffusers import ControlNetModel, AutoencoderKL30        >>> from diffusers.schedulers import UniPCMultistepScheduler31        >>> from diffusers.utils import load_image32        >>> import numpy as np33        >>> import torch34 35        >>> import cv236        >>> from PIL import Image37 38        >>> # download an image for the Canny controlnet39        >>> canny_image = load_image(40        ...     "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl_reference_input_cat.jpg"41        ... )42 43        >>> # download an image for the Reference controlnet44        >>> ref_image = load_image(45        ...     "https://hf.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png"46        ... )47 48        >>> # initialize the models and pipeline49        >>> controlnet_conditioning_scale = 0.5  # recommended for good generalization50        >>> controlnet = ControlNetModel.from_pretrained(51        ...     "diffusers/controlnet-canny-sdxl-1.0", torch_dtype=torch.float1652        ... )53        >>> vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)54        >>> pipe = StableDiffusionXLControlNetReferencePipeline.from_pretrained(55        ...     "stabilityai/stable-diffusion-xl-base-1.0", controlnet=controlnet, vae=vae, torch_dtype=torch.float1656        ... ).to("cuda:0")57 58        >>> pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)59 60        >>> # get canny image61        >>> image = np.array(canny_image)62        >>> image = cv2.Canny(image, 100, 200)63        >>> image = image[:, :, None]64        >>> image = np.concatenate([image, image, image], axis=2)65        >>> canny_image = Image.fromarray(image)66 67        >>> # generate image68        >>> image = pipe(69        ...     prompt="a cat",70        ...     num_inference_steps=20,71        ...     controlnet_conditioning_scale=controlnet_conditioning_scale,72        ...     image=canny_image,73        ...     ref_image=ref_image,74        ...     reference_attn=True,75        ...     reference_adain=True76        ...     style_fidelity=1.0,77        ...     generator=torch.Generator("cuda").manual_seed(42)78        ... ).images[0]79        ```80"""81 82 83def torch_dfs(model: torch.nn.Module):84    result = [model]85    for child in model.children():86        result += torch_dfs(child)87    return result88 89 90# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps91def retrieve_timesteps(92    scheduler,93    num_inference_steps: Optional[int] = None,94    device: Optional[Union[str, torch.device]] = None,95    timesteps: Optional[List[int]] = None,96    sigmas: Optional[List[float]] = None,97    **kwargs,98):99    r"""100    Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles101    custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.102 103    Args:104        scheduler (`SchedulerMixin`):105            The scheduler to get timesteps from.106        num_inference_steps (`int`):107            The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`108            must be `None`.109        device (`str` or `torch.device`, *optional*):110            The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.111        timesteps (`List[int]`, *optional*):112            Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,113            `num_inference_steps` and `sigmas` must be `None`.114        sigmas (`List[float]`, *optional*):115            Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,116            `num_inference_steps` and `timesteps` must be `None`.117 118    Returns:119        `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the120        second element is the number of inference steps.121    """122    if timesteps is not None and sigmas is not None:123        raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")124    if timesteps is not None:125        accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())126        if not accepts_timesteps:127            raise ValueError(128                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"129                f" timestep schedules. Please check whether you are using the correct scheduler."130            )131        scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)132        timesteps = scheduler.timesteps133        num_inference_steps = len(timesteps)134    elif sigmas is not None:135        accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())136        if not accept_sigmas:137            raise ValueError(138                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"139                f" sigmas schedules. Please check whether you are using the correct scheduler."140            )141        scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)142        timesteps = scheduler.timesteps143        num_inference_steps = len(timesteps)144    else:145        scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)146        timesteps = scheduler.timesteps147    return timesteps, num_inference_steps148 149 150class StableDiffusionXLControlNetReferencePipeline(StableDiffusionXLControlNetPipeline):151    r"""152    Pipeline for text-to-image generation using Stable Diffusion XL with ControlNet guidance.153 154    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods155    implemented for all pipelines (downloading, saving, running on a particular device, etc.).156 157    The pipeline also inherits the following loading methods:158        - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings159        - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights160        - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights161        - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files162        - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters163 164    Args:165        vae ([`AutoencoderKL`]):166            Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.167        text_encoder ([`~transformers.CLIPTextModel`]):168            Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).169        text_encoder_2 ([`~transformers.CLIPTextModelWithProjection`]):170            Second frozen text-encoder171            ([laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)).172        tokenizer ([`~transformers.CLIPTokenizer`]):173            A `CLIPTokenizer` to tokenize text.174        tokenizer_2 ([`~transformers.CLIPTokenizer`]):175            A `CLIPTokenizer` to tokenize text.176        unet ([`UNet2DConditionModel`]):177            A `UNet2DConditionModel` to denoise the encoded image latents.178        controlnet ([`ControlNetModel`] or `List[ControlNetModel]`):179            Provides additional conditioning to the `unet` during the denoising process. If you set multiple180            ControlNets as a list, the outputs from each ControlNet are added together to create one combined181            additional conditioning.182        scheduler ([`SchedulerMixin`]):183            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of184            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].185        force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`):186            Whether the negative prompt embeddings should always be set to 0. Also see the config of187            `stabilityai/stable-diffusion-xl-base-1-0`.188        add_watermarker (`bool`, *optional*):189            Whether to use the [invisible_watermark](https://github.com/ShieldMnt/invisible-watermark/) library to190            watermark output images. If not defined, it defaults to `True` if the package is installed; otherwise no191            watermarker is used.192    """193 194    def prepare_ref_latents(self, refimage, batch_size, dtype, device, generator, do_classifier_free_guidance):195        refimage = refimage.to(device=device)196        needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast197        if needs_upcasting:198            self.upcast_vae()199            refimage = refimage.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)200        if refimage.dtype != self.vae.dtype:201            refimage = refimage.to(dtype=self.vae.dtype)202        # encode the mask image into latents space so we can concatenate it to the latents203        if isinstance(generator, list):204            ref_image_latents = [205                self.vae.encode(refimage[i : i + 1]).latent_dist.sample(generator=generator[i])206                for i in range(batch_size)207            ]208            ref_image_latents = torch.cat(ref_image_latents, dim=0)209        else:210            ref_image_latents = self.vae.encode(refimage).latent_dist.sample(generator=generator)211        ref_image_latents = self.vae.config.scaling_factor * ref_image_latents212 213        # duplicate mask and ref_image_latents for each generation per prompt, using mps friendly method214        if ref_image_latents.shape[0] < batch_size:215            if not batch_size % ref_image_latents.shape[0] == 0:216                raise ValueError(217                    "The passed images and the required batch size don't match. Images are supposed to be duplicated"218                    f" to a total batch size of {batch_size}, but {ref_image_latents.shape[0]} images were passed."219                    " Make sure the number of images that you pass is divisible by the total requested batch size."220                )221            ref_image_latents = ref_image_latents.repeat(batch_size // ref_image_latents.shape[0], 1, 1, 1)222 223        ref_image_latents = torch.cat([ref_image_latents] * 2) if do_classifier_free_guidance else ref_image_latents224 225        # aligning device to prevent device errors when concating it with the latent model input226        ref_image_latents = ref_image_latents.to(device=device, dtype=dtype)227 228        # cast back to fp16 if needed229        if needs_upcasting:230            self.vae.to(dtype=torch.float16)231 232        return ref_image_latents233 234    def prepare_ref_image(235        self,236        image,237        width,238        height,239        batch_size,240        num_images_per_prompt,241        device,242        dtype,243        do_classifier_free_guidance=False,244        guess_mode=False,245    ):246        if not isinstance(image, torch.Tensor):247            if isinstance(image, PIL.Image.Image):248                image = [image]249 250            if isinstance(image[0], PIL.Image.Image):251                images = []252 253                for image_ in image:254                    image_ = image_.convert("RGB")255                    image_ = image_.resize((width, height), resample=PIL_INTERPOLATION["lanczos"])256                    image_ = np.array(image_)257                    image_ = image_[None, :]258                    images.append(image_)259 260                image = images261 262                image = np.concatenate(image, axis=0)263                image = np.array(image).astype(np.float32) / 255.0264                image = (image - 0.5) / 0.5265                image = image.transpose(0, 3, 1, 2)266                image = torch.from_numpy(image)267 268            elif isinstance(image[0], torch.Tensor):269                image = torch.stack(image, dim=0)270 271        image_batch_size = image.shape[0]272 273        if image_batch_size == 1:274            repeat_by = batch_size275        else:276            repeat_by = num_images_per_prompt277 278        image = image.repeat_interleave(repeat_by, dim=0)279 280        image = image.to(device=device, dtype=dtype)281 282        if do_classifier_free_guidance and not guess_mode:283            image = torch.cat([image] * 2)284 285        return image286 287    def check_ref_inputs(288        self,289        ref_image,290        reference_guidance_start,291        reference_guidance_end,292        style_fidelity,293        reference_attn,294        reference_adain,295    ):296        ref_image_is_pil = isinstance(ref_image, PIL.Image.Image)297        ref_image_is_tensor = isinstance(ref_image, torch.Tensor)298 299        if not ref_image_is_pil and not ref_image_is_tensor:300            raise TypeError(301                f"ref image must be passed and be one of PIL image or torch tensor, but is {type(ref_image)}"302            )303 304        if not reference_attn and not reference_adain:305            raise ValueError("`reference_attn` or `reference_adain` must be True.")306 307        if style_fidelity < 0.0:308            raise ValueError(f"style fidelity: {style_fidelity} can't be smaller than 0.")309        if style_fidelity > 1.0:310            raise ValueError(f"style fidelity: {style_fidelity} can't be larger than 1.0.")311 312        if reference_guidance_start >= reference_guidance_end:313            raise ValueError(314                f"reference guidance start: {reference_guidance_start} cannot be larger or equal to reference guidance end: {reference_guidance_end}."315            )316        if reference_guidance_start < 0.0:317            raise ValueError(f"reference guidance start: {reference_guidance_start} can't be smaller than 0.")318        if reference_guidance_end > 1.0:319            raise ValueError(f"reference guidance end: {reference_guidance_end} can't be larger than 1.0.")320 321    @torch.no_grad()322    @replace_example_docstring(EXAMPLE_DOC_STRING)323    def __call__(324        self,325        prompt: Union[str, List[str]] = None,326        prompt_2: Optional[Union[str, List[str]]] = None,327        image: PipelineImageInput = None,328        ref_image: Union[torch.Tensor, PIL.Image.Image] = None,329        height: Optional[int] = None,330        width: Optional[int] = None,331        num_inference_steps: int = 50,332        timesteps: List[int] = None,333        sigmas: List[float] = None,334        denoising_end: Optional[float] = None,335        guidance_scale: float = 5.0,336        negative_prompt: Optional[Union[str, List[str]]] = None,337        negative_prompt_2: Optional[Union[str, List[str]]] = None,338        num_images_per_prompt: Optional[int] = 1,339        eta: float = 0.0,340        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,341        latents: Optional[torch.Tensor] = None,342        prompt_embeds: Optional[torch.Tensor] = None,343        negative_prompt_embeds: Optional[torch.Tensor] = None,344        pooled_prompt_embeds: Optional[torch.Tensor] = None,345        negative_pooled_prompt_embeds: Optional[torch.Tensor] = None,346        ip_adapter_image: Optional[PipelineImageInput] = None,347        ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None,348        output_type: Optional[str] = "pil",349        return_dict: bool = True,350        cross_attention_kwargs: Optional[Dict[str, Any]] = None,351        controlnet_conditioning_scale: Union[float, List[float]] = 1.0,352        guess_mode: bool = False,353        control_guidance_start: Union[float, List[float]] = 0.0,354        control_guidance_end: Union[float, List[float]] = 1.0,355        original_size: Tuple[int, int] = None,356        crops_coords_top_left: Tuple[int, int] = (0, 0),357        target_size: Tuple[int, int] = None,358        negative_original_size: Optional[Tuple[int, int]] = None,359        negative_crops_coords_top_left: Tuple[int, int] = (0, 0),360        negative_target_size: Optional[Tuple[int, int]] = None,361        clip_skip: Optional[int] = None,362        callback_on_step_end: Optional[363            Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks]364        ] = None,365        callback_on_step_end_tensor_inputs: List[str] = ["latents"],366        attention_auto_machine_weight: float = 1.0,367        gn_auto_machine_weight: float = 1.0,368        reference_guidance_start: float = 0.0,369        reference_guidance_end: float = 1.0,370        style_fidelity: float = 0.5,371        reference_attn: bool = True,372        reference_adain: bool = True,373        **kwargs,374    ):375        r"""376        The call function to the pipeline for generation.377 378        Args:379            prompt (`str` or `List[str]`, *optional*):380                The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.381            prompt_2 (`str` or `List[str]`, *optional*):382                The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is383                used in both text-encoders.384            image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,:385                    `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`):386                The ControlNet input condition to provide guidance to the `unet` for generation. If the type is387                specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted388                as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or389                width are passed, `image` is resized accordingly. If multiple ControlNets are specified in `init`,390                images must be passed as a list such that each element of the list can be correctly batched for input391                to a single ControlNet.392            ref_image (`torch.Tensor`, `PIL.Image.Image`):393                The Reference Control input condition. Reference Control uses this input condition to generate guidance to Unet. If394                the type is specified as `Torch.Tensor`, it is passed to Reference Control as is. `PIL.Image.Image` can395                also be accepted as an image.396            height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):397                The height in pixels of the generated image. Anything below 512 pixels won't work well for398                [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)399                and checkpoints that are not specifically fine-tuned on low resolutions.400            width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):401                The width in pixels of the generated image. Anything below 512 pixels won't work well for402                [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)403                and checkpoints that are not specifically fine-tuned on low resolutions.404            num_inference_steps (`int`, *optional*, defaults to 50):405                The number of denoising steps. More denoising steps usually lead to a higher quality image at the406                expense of slower inference.407            timesteps (`List[int]`, *optional*):408                Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument409                in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is410                passed will be used. Must be in descending order.411            sigmas (`List[float]`, *optional*):412                Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in413                their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed414                will be used.415            denoising_end (`float`, *optional*):416                When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be417                completed before it is intentionally prematurely terminated. As a result, the returned sample will418                still retain a substantial amount of noise as determined by the discrete timesteps selected by the419                scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a420                "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image421                Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output)422            guidance_scale (`float`, *optional*, defaults to 5.0):423                A higher guidance scale value encourages the model to generate images closely linked to the text424                `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.425            negative_prompt (`str` or `List[str]`, *optional*):426                The prompt or prompts to guide what to not include in image generation. If not defined, you need to427                pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).428            negative_prompt_2 (`str` or `List[str]`, *optional*):429                The prompt or prompts to guide what to not include in image generation. This is sent to `tokenizer_2`430                and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders.431            num_images_per_prompt (`int`, *optional*, defaults to 1):432                The number of images to generate per prompt.433            eta (`float`, *optional*, defaults to 0.0):434                Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies435                to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.436            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):437                A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make438                generation deterministic.439            latents (`torch.Tensor`, *optional*):440                Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image441                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents442                tensor is generated by sampling using the supplied random `generator`.443            prompt_embeds (`torch.Tensor`, *optional*):444                Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not445                provided, text embeddings are generated from the `prompt` input argument.446            negative_prompt_embeds (`torch.Tensor`, *optional*):447                Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If448                not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.449            pooled_prompt_embeds (`torch.Tensor`, *optional*):450                Pre-generated pooled text embeddings. Can be used to easily tweak text inputs (prompt weighting). If451                not provided, pooled text embeddings are generated from `prompt` input argument.452            negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):453                Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs (prompt454                weighting). If not provided, pooled `negative_prompt_embeds` are generated from `negative_prompt` input455                argument.456            ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.457            ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*):458                Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of459                IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should460                contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not461                provided, embeddings are computed from the `ip_adapter_image` input argument.462            output_type (`str`, *optional*, defaults to `"pil"`):463                The output format of the generated image. Choose between `PIL.Image` or `np.array`.464            return_dict (`bool`, *optional*, defaults to `True`):465                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a466                plain tuple.467            cross_attention_kwargs (`dict`, *optional*):468                A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in469                [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).470            controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0):471                The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added472                to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set473                the corresponding scale as a list.474            guess_mode (`bool`, *optional*, defaults to `False`):475                The ControlNet encoder tries to recognize the content of the input image even if you remove all476                prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended.477            control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0):478                The percentage of total steps at which the ControlNet starts applying.479            control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0):480                The percentage of total steps at which the ControlNet stops applying.481            original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):482                If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.483                `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as484                explained in section 2.2 of485                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).486            crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):487                `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position488                `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting489                `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of490                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).491            target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):492                For most cases, `target_size` should be set to the desired height and width of the generated image. If493                not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in494                section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).495            negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):496                To negatively condition the generation process based on a specific image resolution. Part of SDXL's497                micro-conditioning as explained in section 2.2 of498                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more499                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.500            negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):501                To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's502                micro-conditioning as explained in section 2.2 of503                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more504                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.505            negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):506                To negatively condition the generation process based on a target image resolution. It should be as same507                as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of508                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more509                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.510            clip_skip (`int`, *optional*):511                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that512                the output of the pre-final layer will be used for computing the prompt embeddings.513            callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):514                A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of515                each denoising step during the inference. with the following arguments: `callback_on_step_end(self:516                DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a517                list of all tensors as specified by `callback_on_step_end_tensor_inputs`.518            callback_on_step_end_tensor_inputs (`List`, *optional*):519                The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list520                will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the521                `._callback_tensor_inputs` attribute of your pipeline class.522            attention_auto_machine_weight (`float`):523                Weight of using reference query for self attention's context.524                If attention_auto_machine_weight=1.0, use reference query for all self attention's context.525            gn_auto_machine_weight (`float`):526                Weight of using reference adain. If gn_auto_machine_weight=2.0, use all reference adain plugins.527            reference_guidance_start (`float`, *optional*, defaults to 0.0):528                The percentage of total steps at which the reference ControlNet starts applying.529            reference_guidance_end (`float`, *optional*, defaults to 1.0):530                The percentage of total steps at which the reference ControlNet stops applying.531            style_fidelity (`float`):532                style fidelity of ref_uncond_xt. If style_fidelity=1.0, control more important,533                elif style_fidelity=0.0, prompt more important, else balanced.534            reference_attn (`bool`):535                Whether to use reference query for self attention's context.536            reference_adain (`bool`):537                Whether to use reference adain.538 539        Examples:540 541        Returns:542            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:543                If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned,544                otherwise a `tuple` is returned containing the output images.545        """546 547        callback = kwargs.pop("callback", None)548        callback_steps = kwargs.pop("callback_steps", None)549 550        if callback is not None:551            deprecate(552                "callback",553                "1.0.0",554                "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",555            )556        if callback_steps is not None:557            deprecate(558                "callback_steps",559                "1.0.0",560                "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",561            )562 563        if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):564            callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs565 566        controlnet = self.controlnet._orig_mod if is_compiled_module(self.controlnet) else self.controlnet567 568        # align format for control guidance569        if not isinstance(control_guidance_start, list) and isinstance(control_guidance_end, list):570            control_guidance_start = len(control_guidance_end) * [control_guidance_start]571        elif not isinstance(control_guidance_end, list) and isinstance(control_guidance_start, list):572            control_guidance_end = len(control_guidance_start) * [control_guidance_end]573        elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list):574            mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1575            control_guidance_start, control_guidance_end = (576                mult * [control_guidance_start],577                mult * [control_guidance_end],578            )579 580        # 1. Check inputs. Raise error if not correct581        self.check_inputs(582            prompt,583            prompt_2,584            image,585            callback_steps,586            negative_prompt,587            negative_prompt_2,588            prompt_embeds,589            negative_prompt_embeds,590            pooled_prompt_embeds,591            ip_adapter_image,592            ip_adapter_image_embeds,593            negative_pooled_prompt_embeds,594            controlnet_conditioning_scale,595            control_guidance_start,596            control_guidance_end,597            callback_on_step_end_tensor_inputs,598        )599 600        self.check_ref_inputs(601            ref_image,602            reference_guidance_start,603            reference_guidance_end,604            style_fidelity,605            reference_attn,606            reference_adain,607        )608 609        self._guidance_scale = guidance_scale610        self._clip_skip = clip_skip611        self._cross_attention_kwargs = cross_attention_kwargs612        self._denoising_end = denoising_end613        self._interrupt = False614 615        # 2. Define call parameters616        if prompt is not None and isinstance(prompt, str):617            batch_size = 1618        elif prompt is not None and isinstance(prompt, list):619            batch_size = len(prompt)620        else:621            batch_size = prompt_embeds.shape[0]622 623        device = self._execution_device624 625        if isinstance(controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float):626            controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnet.nets)627 628        global_pool_conditions = (629            controlnet.config.global_pool_conditions630            if isinstance(controlnet, ControlNetModel)631            else controlnet.nets[0].config.global_pool_conditions632        )633        guess_mode = guess_mode or global_pool_conditions634 635        # 3.1 Encode input prompt636        text_encoder_lora_scale = (637            self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None638        )639        (640            prompt_embeds,641            negative_prompt_embeds,642            pooled_prompt_embeds,643            negative_pooled_prompt_embeds,644        ) = self.encode_prompt(645            prompt,646            prompt_2,647            device,648            num_images_per_prompt,649            self.do_classifier_free_guidance,650            negative_prompt,651            negative_prompt_2,652            prompt_embeds=prompt_embeds,653            negative_prompt_embeds=negative_prompt_embeds,654            pooled_prompt_embeds=pooled_prompt_embeds,655            negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,656            lora_scale=text_encoder_lora_scale,657            clip_skip=self.clip_skip,658        )659 660        # 3.2 Encode ip_adapter_image661        if ip_adapter_image is not None or ip_adapter_image_embeds is not None:662            image_embeds = self.prepare_ip_adapter_image_embeds(663                ip_adapter_image,664                ip_adapter_image_embeds,665                device,666                batch_size * num_images_per_prompt,667                self.do_classifier_free_guidance,668            )669 670        # 4. Prepare image671        if isinstance(controlnet, ControlNetModel):672            image = self.prepare_image(673                image=image,674                width=width,675                height=height,676                batch_size=batch_size * num_images_per_prompt,677                num_images_per_prompt=num_images_per_prompt,678                device=device,679                dtype=controlnet.dtype,680                do_classifier_free_guidance=self.do_classifier_free_guidance,681                guess_mode=guess_mode,682            )683            height, width = image.shape[-2:]684        elif isinstance(controlnet, MultiControlNetModel):685            images = []686 687            for image_ in image:688                image_ = self.prepare_image(689                    image=image_,690                    width=width,691                    height=height,692                    batch_size=batch_size * num_images_per_prompt,693                    num_images_per_prompt=num_images_per_prompt,694                    device=device,695                    dtype=controlnet.dtype,696                    do_classifier_free_guidance=self.do_classifier_free_guidance,697                    guess_mode=guess_mode,698                )699 700                images.append(image_)701 702            image = images703            height, width = image[0].shape[-2:]704        else:705            assert False706 707        # 5. Preprocess reference image708        ref_image = self.prepare_ref_image(709            image=ref_image,710            width=width,711            height=height,712            batch_size=batch_size * num_images_per_prompt,713            num_images_per_prompt=num_images_per_prompt,714            device=device,715            dtype=prompt_embeds.dtype,716        )717 718        # 6. Prepare timesteps719        timesteps, num_inference_steps = retrieve_timesteps(720            self.scheduler, num_inference_steps, device, timesteps, sigmas721        )722        self._num_timesteps = len(timesteps)723 724        # 7. Prepare latent variables725        num_channels_latents = self.unet.config.in_channels726        latents = self.prepare_latents(727            batch_size * num_images_per_prompt,728            num_channels_latents,729            height,730            width,731            prompt_embeds.dtype,732            device,733            generator,734            latents,735        )736 737        # 7.5 Optionally get Guidance Scale Embedding738        timestep_cond = None739        if self.unet.config.time_cond_proj_dim is not None:740            guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)741            timestep_cond = self.get_guidance_scale_embedding(742                guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim743            ).to(device=device, dtype=latents.dtype)744 745        # 8. Prepare reference latent variables746        ref_image_latents = self.prepare_ref_latents(747            ref_image,748            batch_size * num_images_per_prompt,749            prompt_embeds.dtype,750            device,751            generator,752            self.do_classifier_free_guidance,753        )754 755        # 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline756        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)757 758        # 9.1 Create tensor stating which controlnets to keep759        controlnet_keep = []760        reference_keeps = []761        for i in range(len(timesteps)):762            keeps = [763                1.0 - float(i / len(timesteps) < s or (i + 1) / len(timesteps) > e)764                for s, e in zip(control_guidance_start, control_guidance_end)765            ]766            controlnet_keep.append(keeps[0] if isinstance(controlnet, ControlNetModel) else keeps)767            reference_keep = 1.0 - float(768                i / len(timesteps) < reference_guidance_start or (i + 1) / len(timesteps) > reference_guidance_end769            )770            reference_keeps.append(reference_keep)771 772        # 9.2 Modify self attention and group norm773        MODE = "write"774        uc_mask = (775            torch.Tensor([1] * batch_size * num_images_per_prompt + [0] * batch_size * num_images_per_prompt)776            .type_as(ref_image_latents)777            .bool()778        )779 780        do_classifier_free_guidance = self.do_classifier_free_guidance781 782        def hacked_basic_transformer_inner_forward(783            self,784            hidden_states: torch.Tensor,785            attention_mask: Optional[torch.Tensor] = None,786            encoder_hidden_states: Optional[torch.Tensor] = None,787            encoder_attention_mask: Optional[torch.Tensor] = None,788            timestep: Optional[torch.LongTensor] = None,789            cross_attention_kwargs: Dict[str, Any] = None,790            class_labels: Optional[torch.LongTensor] = None,791        ):792            if self.use_ada_layer_norm:793                norm_hidden_states = self.norm1(hidden_states, timestep)794            elif self.use_ada_layer_norm_zero:795                norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(796                    hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype797                )798            else:799                norm_hidden_states = self.norm1(hidden_states)800 801            # 1. Self-Attention802            cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {}803            if self.only_cross_attention:804                attn_output = self.attn1(805                    norm_hidden_states,806                    encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,807                    attention_mask=attention_mask,808                    **cross_attention_kwargs,809                )810            else:811                if MODE == "write":812                    self.bank.append(norm_hidden_states.detach().clone())813                    attn_output = self.attn1(814                        norm_hidden_states,815                        encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,816                        attention_mask=attention_mask,817                        **cross_attention_kwargs,818                    )819                if MODE == "read":820                    if attention_auto_machine_weight > self.attn_weight:821                        attn_output_uc = self.attn1(822                            norm_hidden_states,823                            encoder_hidden_states=torch.cat([norm_hidden_states] + self.bank, dim=1),824                            # attention_mask=attention_mask,825                            **cross_attention_kwargs,826                        )827                        attn_output_c = attn_output_uc.clone()828                        if do_classifier_free_guidance and style_fidelity > 0:829                            attn_output_c[uc_mask] = self.attn1(830                                norm_hidden_states[uc_mask],831                                encoder_hidden_states=norm_hidden_states[uc_mask],832                                **cross_attention_kwargs,833                            )834                        attn_output = style_fidelity * attn_output_c + (1.0 - style_fidelity) * attn_output_uc835                        self.bank.clear()836                    else:837                        attn_output = self.attn1(838                            norm_hidden_states,839                            encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,840                            attention_mask=attention_mask,841                            **cross_attention_kwargs,842                        )843            if self.use_ada_layer_norm_zero:844                attn_output = gate_msa.unsqueeze(1) * attn_output845            hidden_states = attn_output + hidden_states846 847            if self.attn2 is not None:848                norm_hidden_states = (849                    self.norm2(hidden_states, timestep) if self.use_ada_layer_norm else self.norm2(hidden_states)850                )851 852                # 2. Cross-Attention853                attn_output = self.attn2(854                    norm_hidden_states,855                    encoder_hidden_states=encoder_hidden_states,856                    attention_mask=encoder_attention_mask,857                    **cross_attention_kwargs,858                )859                hidden_states = attn_output + hidden_states860 861            # 3. Feed-forward862            norm_hidden_states = self.norm3(hidden_states)863 864            if self.use_ada_layer_norm_zero:865                norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]866 867            ff_output = self.ff(norm_hidden_states)868 869            if self.use_ada_layer_norm_zero:870                ff_output = gate_mlp.unsqueeze(1) * ff_output871 872            hidden_states = ff_output + hidden_states873 874            return hidden_states875 876        def hacked_mid_forward(self, *args, **kwargs):877            eps = 1e-6878            x = self.original_forward(*args, **kwargs)879            if MODE == "write":880                if gn_auto_machine_weight >= self.gn_weight:881                    var, mean = torch.var_mean(x, dim=(2, 3), keepdim=True, correction=0)882                    self.mean_bank.append(mean)883                    self.var_bank.append(var)884            if MODE == "read":885                if len(self.mean_bank) > 0 and len(self.var_bank) > 0:886                    var, mean = torch.var_mean(x, dim=(2, 3), keepdim=True, correction=0)887                    std = torch.maximum(var, torch.zeros_like(var) + eps) ** 0.5888                    mean_acc = sum(self.mean_bank) / float(len(self.mean_bank))889                    var_acc = sum(self.var_bank) / float(len(self.var_bank))890                    std_acc = torch.maximum(var_acc, torch.zeros_like(var_acc) + eps) ** 0.5891                    x_uc = (((x - mean) / std) * std_acc) + mean_acc892                    x_c = x_uc.clone()893                    if do_classifier_free_guidance and style_fidelity > 0:894                        x_c[uc_mask] = x[uc_mask]895                    x = style_fidelity * x_c + (1.0 - style_fidelity) * x_uc896                self.mean_bank = []897                self.var_bank = []898            return x899 900        def hack_CrossAttnDownBlock2D_forward(901            self,902            hidden_states: torch.Tensor,903            temb: Optional[torch.Tensor] = None,904            encoder_hidden_states: Optional[torch.Tensor] = None,905            attention_mask: Optional[torch.Tensor] = None,906            cross_attention_kwargs: Optional[Dict[str, Any]] = None,907            encoder_attention_mask: Optional[torch.Tensor] = None,908        ):909            eps = 1e-6910 911            # TODO(Patrick, William) - attention mask is not used912            output_states = ()913 914            for i, (resnet, attn) in enumerate(zip(self.resnets, self.attentions)):915                hidden_states = resnet(hidden_states, temb)916                hidden_states = attn(917                    hidden_states,918                    encoder_hidden_states=encoder_hidden_states,919                    cross_attention_kwargs=cross_attention_kwargs,920                    attention_mask=attention_mask,921                    encoder_attention_mask=encoder_attention_mask,922                    return_dict=False,923                )[0]924                if MODE == "write":925                    if gn_auto_machine_weight >= self.gn_weight:926                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)927                        self.mean_bank.append([mean])928                        self.var_bank.append([var])929                if MODE == "read":930                    if len(self.mean_bank) > 0 and len(self.var_bank) > 0:931                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)932                        std = torch.maximum(var, torch.zeros_like(var) + eps) ** 0.5933                        mean_acc = sum(self.mean_bank[i]) / float(len(self.mean_bank[i]))934                        var_acc = sum(self.var_bank[i]) / float(len(self.var_bank[i]))935                        std_acc = torch.maximum(var_acc, torch.zeros_like(var_acc) + eps) ** 0.5936                        hidden_states_uc = (((hidden_states - mean) / std) * std_acc) + mean_acc937                        hidden_states_c = hidden_states_uc.clone()938                        if do_classifier_free_guidance and style_fidelity > 0:939                            hidden_states_c[uc_mask] = hidden_states[uc_mask]940                        hidden_states = style_fidelity * hidden_states_c + (1.0 - style_fidelity) * hidden_states_uc941 942                output_states = output_states + (hidden_states,)943 944            if MODE == "read":945                self.mean_bank = []946                self.var_bank = []947 948            if self.downsamplers is not None:949                for downsampler in self.downsamplers:950                    hidden_states = downsampler(hidden_states)951 952                output_states = output_states + (hidden_states,)953 954            return hidden_states, output_states955 956        def hacked_DownBlock2D_forward(self, hidden_states, temb=None, *args, **kwargs):957            eps = 1e-6958 959            output_states = ()960 961            for i, resnet in enumerate(self.resnets):962                hidden_states = resnet(hidden_states, temb)963 964                if MODE == "write":965                    if gn_auto_machine_weight >= self.gn_weight:966                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)967                        self.mean_bank.append([mean])968                        self.var_bank.append([var])969                if MODE == "read":970                    if len(self.mean_bank) > 0 and len(self.var_bank) > 0:971                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)972                        std = torch.maximum(var, torch.zeros_like(var) + eps) ** 0.5973                        mean_acc = sum(self.mean_bank[i]) / float(len(self.mean_bank[i]))974                        var_acc = sum(self.var_bank[i]) / float(len(self.var_bank[i]))975                        std_acc = torch.maximum(var_acc, torch.zeros_like(var_acc) + eps) ** 0.5976                        hidden_states_uc = (((hidden_states - mean) / std) * std_acc) + mean_acc977                        hidden_states_c = hidden_states_uc.clone()978                        if do_classifier_free_guidance and style_fidelity > 0:979                            hidden_states_c[uc_mask] = hidden_states[uc_mask]980                        hidden_states = style_fidelity * hidden_states_c + (1.0 - style_fidelity) * hidden_states_uc981 982                output_states = output_states + (hidden_states,)983 984            if MODE == "read":985                self.mean_bank = []986                self.var_bank = []987 988            if self.downsamplers is not None:989                for downsampler in self.downsamplers:990                    hidden_states = downsampler(hidden_states)991 992                output_states = output_states + (hidden_states,)993 994            return hidden_states, output_states995 996        def hacked_CrossAttnUpBlock2D_forward(997            self,998            hidden_states: torch.Tensor,999            res_hidden_states_tuple: Tuple[torch.Tensor, ...],1000            temb: Optional[torch.Tensor] = None,1001            encoder_hidden_states: Optional[torch.Tensor] = None,1002            cross_attention_kwargs: Optional[Dict[str, Any]] = None,1003            upsample_size: Optional[int] = None,1004            attention_mask: Optional[torch.Tensor] = None,1005            encoder_attention_mask: Optional[torch.Tensor] = None,1006        ):1007            eps = 1e-61008            # TODO(Patrick, William) - attention mask is not used1009            for i, (resnet, attn) in enumerate(zip(self.resnets, self.attentions)):1010                # pop res hidden states1011                res_hidden_states = res_hidden_states_tuple[-1]1012                res_hidden_states_tuple = res_hidden_states_tuple[:-1]1013                hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)1014                hidden_states = resnet(hidden_states, temb)1015                hidden_states = attn(1016                    hidden_states,1017                    encoder_hidden_states=encoder_hidden_states,1018                    cross_attention_kwargs=cross_attention_kwargs,1019                    attention_mask=attention_mask,1020                    encoder_attention_mask=encoder_attention_mask,1021                    return_dict=False,1022                )[0]1023 1024                if MODE == "write":1025                    if gn_auto_machine_weight >= self.gn_weight:1026                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)1027                        self.mean_bank.append([mean])1028                        self.var_bank.append([var])1029                if MODE == "read":1030                    if len(self.mean_bank) > 0 and len(self.var_bank) > 0:1031                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)1032                        std = torch.maximum(var, torch.zeros_like(var) + eps) ** 0.51033                        mean_acc = sum(self.mean_bank[i]) / float(len(self.mean_bank[i]))1034                        var_acc = sum(self.var_bank[i]) / float(len(self.var_bank[i]))1035                        std_acc = torch.maximum(var_acc, torch.zeros_like(var_acc) + eps) ** 0.51036                        hidden_states_uc = (((hidden_states - mean) / std) * std_acc) + mean_acc1037                        hidden_states_c = hidden_states_uc.clone()1038                        if do_classifier_free_guidance and style_fidelity > 0:1039                            hidden_states_c[uc_mask] = hidden_states[uc_mask]1040                        hidden_states = style_fidelity * hidden_states_c + (1.0 - style_fidelity) * hidden_states_uc1041 1042            if MODE == "read":1043                self.mean_bank = []1044                self.var_bank = []1045 1046            if self.upsamplers is not None:1047                for upsampler in self.upsamplers:1048                    hidden_states = upsampler(hidden_states, upsample_size)1049 1050            return hidden_states1051 1052        def hacked_UpBlock2D_forward(1053            self, hidden_states, res_hidden_states_tuple, temb=None, upsample_size=None, *args, **kwargs1054        ):1055            eps = 1e-61056            for i, resnet in enumerate(self.resnets):1057                # pop res hidden states1058                res_hidden_states = res_hidden_states_tuple[-1]1059                res_hidden_states_tuple = res_hidden_states_tuple[:-1]1060                hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)1061                hidden_states = resnet(hidden_states, temb)1062 1063                if MODE == "write":1064                    if gn_auto_machine_weight >= self.gn_weight:1065                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)1066                        self.mean_bank.append([mean])1067                        self.var_bank.append([var])1068                if MODE == "read":1069                    if len(self.mean_bank) > 0 and len(self.var_bank) > 0:1070                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)1071                        std = torch.maximum(var, torch.zeros_like(var) + eps) ** 0.51072                        mean_acc = sum(self.mean_bank[i]) / float(len(self.mean_bank[i]))1073                        var_acc = sum(self.var_bank[i]) / float(len(self.var_bank[i]))1074                        std_acc = torch.maximum(var_acc, torch.zeros_like(var_acc) + eps) ** 0.51075                        hidden_states_uc = (((hidden_states - mean) / std) * std_acc) + mean_acc1076                        hidden_states_c = hidden_states_uc.clone()1077                        if do_classifier_free_guidance and style_fidelity > 0:1078                            hidden_states_c[uc_mask] = hidden_states[uc_mask]1079                        hidden_states = style_fidelity * hidden_states_c + (1.0 - style_fidelity) * hidden_states_uc1080 1081            if MODE == "read":1082                self.mean_bank = []1083                self.var_bank = []1084 1085            if self.upsamplers is not None:1086                for upsampler in self.upsamplers:1087                    hidden_states = upsampler(hidden_states, upsample_size)1088 1089            return hidden_states1090 1091        if reference_attn:1092            attn_modules = [module for module in torch_dfs(self.unet) if isinstance(module, BasicTransformerBlock)]1093            attn_modules = sorted(attn_modules, key=lambda x: -x.norm1.normalized_shape[0])1094 1095            for i, module in enumerate(attn_modules):1096                module._original_inner_forward = module.forward1097                module.forward = hacked_basic_transformer_inner_forward.__get__(module, BasicTransformerBlock)1098                module.bank = []1099                module.attn_weight = float(i) / float(len(attn_modules))1100 1101        if reference_adain:1102            gn_modules = [self.unet.mid_block]1103            self.unet.mid_block.gn_weight = 01104 1105            down_blocks = self.unet.down_blocks1106            for w, module in enumerate(down_blocks):1107                module.gn_weight = 1.0 - float(w) / float(len(down_blocks))1108                gn_modules.append(module)1109 1110            up_blocks = self.unet.up_blocks1111            for w, module in enumerate(up_blocks):1112                module.gn_weight = float(w) / float(len(up_blocks))1113                gn_modules.append(module)1114 1115            for i, module in enumerate(gn_modules):1116                if getattr(module, "original_forward", None) is None:1117                    module.original_forward = module.forward1118                if i == 0:1119                    # mid_block1120                    module.forward = hacked_mid_forward.__get__(module, torch.nn.Module)1121                elif isinstance(module, CrossAttnDownBlock2D):1122                    module.forward = hack_CrossAttnDownBlock2D_forward.__get__(module, CrossAttnDownBlock2D)1123                elif isinstance(module, DownBlock2D):1124                    module.forward = hacked_DownBlock2D_forward.__get__(module, DownBlock2D)1125                elif isinstance(module, CrossAttnUpBlock2D):1126                    module.forward = hacked_CrossAttnUpBlock2D_forward.__get__(module, CrossAttnUpBlock2D)1127                elif isinstance(module, UpBlock2D):1128                    module.forward = hacked_UpBlock2D_forward.__get__(module, UpBlock2D)1129                module.mean_bank = []1130                module.var_bank = []1131                module.gn_weight *= 21132 1133        # 9.2 Prepare added time ids & embeddings1134        if isinstance(image, list):1135            original_size = original_size or image[0].shape[-2:]1136        else:1137            original_size = original_size or image.shape[-2:]1138        target_size = target_size or (height, width)1139 1140        add_text_embeds = pooled_prompt_embeds1141        if self.text_encoder_2 is None:1142            text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])1143        else:1144            text_encoder_projection_dim = self.text_encoder_2.config.projection_dim1145 1146        add_time_ids = self._get_add_time_ids(1147            original_size,1148            crops_coords_top_left,1149            target_size,1150            dtype=prompt_embeds.dtype,1151            text_encoder_projection_dim=text_encoder_projection_dim,1152        )1153 1154        if negative_original_size is not None and negative_target_size is not None:1155            negative_add_time_ids = self._get_add_time_ids(1156                negative_original_size,1157                negative_crops_coords_top_left,1158                negative_target_size,1159                dtype=prompt_embeds.dtype,1160                text_encoder_projection_dim=text_encoder_projection_dim,1161            )1162        else:1163            negative_add_time_ids = add_time_ids1164 1165        if self.do_classifier_free_guidance:1166            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)1167            add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)1168            add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0)1169 1170        prompt_embeds = prompt_embeds.to(device)1171        add_text_embeds = add_text_embeds.to(device)1172        add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)1173 1174        # 10. Denoising loop1175        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order1176 1177        # 10.1 Apply denoising_end1178        if (1179            self.denoising_end is not None1180            and isinstance(self.denoising_end, float)1181            and self.denoising_end > 01182            and self.denoising_end < 11183        ):1184            discrete_timestep_cutoff = int(1185                round(1186                    self.scheduler.config.num_train_timesteps1187                    - (self.denoising_end * self.scheduler.config.num_train_timesteps)1188                )1189            )1190            num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))1191            timesteps = timesteps[:num_inference_steps]1192 1193        is_unet_compiled = is_compiled_module(self.unet)1194        is_controlnet_compiled = is_compiled_module(self.controlnet)1195        is_torch_higher_equal_2_1 = is_torch_version(">=", "2.1")1196        with self.progress_bar(total=num_inference_steps) as progress_bar:1197            for i, t in enumerate(timesteps):1198                if self.interrupt:1199                    continue1200 

Showing the first 1,200 of 1369 lines. Download the file for the rest.