diffusers/community-pipelines-mirror

1# Based on stable_diffusion_reference.py2 3import inspect4from typing import Any, Callable, Dict, List, Optional, Tuple, Union5 6import numpy as np7import PIL.Image8import torch9 10from diffusers import StableDiffusionXLPipeline11from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback12from diffusers.image_processor import PipelineImageInput13from diffusers.models.attention import BasicTransformerBlock14from diffusers.models.unets.unet_2d_blocks import CrossAttnDownBlock2D, CrossAttnUpBlock2D, DownBlock2D, UpBlock2D15from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput16from diffusers.utils import PIL_INTERPOLATION, deprecate, is_torch_xla_available, logging, replace_example_docstring17from diffusers.utils.torch_utils import randn_tensor18 19 20if is_torch_xla_available():21    import torch_xla.core.xla_model as xm  # type: ignore22 23    XLA_AVAILABLE = True24else:25    XLA_AVAILABLE = False26 27 28logger = logging.get_logger(__name__)  # pylint: disable=invalid-name29 30EXAMPLE_DOC_STRING = """31    Examples:32        ```py33        >>> import torch34        >>> from diffusers.schedulers import UniPCMultistepScheduler35        >>> from diffusers.utils import load_image36 37        >>> input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl_reference_input_cat.jpg")38 39        >>> pipe = StableDiffusionXLReferencePipeline.from_pretrained(40            "stabilityai/stable-diffusion-xl-base-1.0",41            torch_dtype=torch.float16,42            use_safetensors=True,43            variant="fp16").to('cuda:0')44 45        >>> pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)46        >>> result_img = pipe(ref_image=input_image,47                        prompt="a dog",48                        num_inference_steps=20,49                        reference_attn=True,50                        reference_adain=True).images[0]51 52        >>> result_img.show()53        ```54"""55 56 57def torch_dfs(model: torch.nn.Module):58    result = [model]59    for child in model.children():60        result += torch_dfs(child)61    return result62 63 64# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg65def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):66    """67    Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and68    Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.469    """70    std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)71    std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)72    # rescale the results from guidance (fixes overexposure)73    noise_pred_rescaled = noise_cfg * (std_text / std_cfg)74    # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images75    noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg76    return noise_cfg77 78 79# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps80def retrieve_timesteps(81    scheduler,82    num_inference_steps: Optional[int] = None,83    device: Optional[Union[str, torch.device]] = None,84    timesteps: Optional[List[int]] = None,85    sigmas: Optional[List[float]] = None,86    **kwargs,87):88    r"""89    Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles90    custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.91 92    Args:93        scheduler (`SchedulerMixin`):94            The scheduler to get timesteps from.95        num_inference_steps (`int`):96            The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`97            must be `None`.98        device (`str` or `torch.device`, *optional*):99            The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.100        timesteps (`List[int]`, *optional*):101            Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,102            `num_inference_steps` and `sigmas` must be `None`.103        sigmas (`List[float]`, *optional*):104            Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,105            `num_inference_steps` and `timesteps` must be `None`.106 107    Returns:108        `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the109        second element is the number of inference steps.110    """111    if timesteps is not None and sigmas is not None:112        raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")113    if timesteps is not None:114        accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())115        if not accepts_timesteps:116            raise ValueError(117                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"118                f" timestep schedules. Please check whether you are using the correct scheduler."119            )120        scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)121        timesteps = scheduler.timesteps122        num_inference_steps = len(timesteps)123    elif sigmas is not None:124        accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())125        if not accept_sigmas:126            raise ValueError(127                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"128                f" sigmas schedules. Please check whether you are using the correct scheduler."129            )130        scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)131        timesteps = scheduler.timesteps132        num_inference_steps = len(timesteps)133    else:134        scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)135        timesteps = scheduler.timesteps136    return timesteps, num_inference_steps137 138 139class StableDiffusionXLReferencePipeline(StableDiffusionXLPipeline):140    def prepare_ref_latents(self, refimage, batch_size, dtype, device, generator, do_classifier_free_guidance):141        refimage = refimage.to(device=device)142        if self.vae.dtype == torch.float16 and self.vae.config.force_upcast:143            self.upcast_vae()144            refimage = refimage.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)145        if refimage.dtype != self.vae.dtype:146            refimage = refimage.to(dtype=self.vae.dtype)147        # encode the mask image into latents space so we can concatenate it to the latents148        if isinstance(generator, list):149            ref_image_latents = [150                self.vae.encode(refimage[i : i + 1]).latent_dist.sample(generator=generator[i])151                for i in range(batch_size)152            ]153            ref_image_latents = torch.cat(ref_image_latents, dim=0)154        else:155            ref_image_latents = self.vae.encode(refimage).latent_dist.sample(generator=generator)156        ref_image_latents = self.vae.config.scaling_factor * ref_image_latents157 158        # duplicate mask and ref_image_latents for each generation per prompt, using mps friendly method159        if ref_image_latents.shape[0] < batch_size:160            if not batch_size % ref_image_latents.shape[0] == 0:161                raise ValueError(162                    "The passed images and the required batch size don't match. Images are supposed to be duplicated"163                    f" to a total batch size of {batch_size}, but {ref_image_latents.shape[0]} images were passed."164                    " Make sure the number of images that you pass is divisible by the total requested batch size."165                )166            ref_image_latents = ref_image_latents.repeat(batch_size // ref_image_latents.shape[0], 1, 1, 1)167 168        ref_image_latents = torch.cat([ref_image_latents] * 2) if do_classifier_free_guidance else ref_image_latents169 170        # aligning device to prevent device errors when concating it with the latent model input171        ref_image_latents = ref_image_latents.to(device=device, dtype=dtype)172        return ref_image_latents173 174    def prepare_ref_image(175        self,176        image,177        width,178        height,179        batch_size,180        num_images_per_prompt,181        device,182        dtype,183        do_classifier_free_guidance=False,184        guess_mode=False,185    ):186        if not isinstance(image, torch.Tensor):187            if isinstance(image, PIL.Image.Image):188                image = [image]189 190            if isinstance(image[0], PIL.Image.Image):191                images = []192 193                for image_ in image:194                    image_ = image_.convert("RGB")195                    image_ = image_.resize((width, height), resample=PIL_INTERPOLATION["lanczos"])196                    image_ = np.array(image_)197                    image_ = image_[None, :]198                    images.append(image_)199 200                image = images201 202                image = np.concatenate(image, axis=0)203                image = np.array(image).astype(np.float32) / 255.0204                image = (image - 0.5) / 0.5205                image = image.transpose(0, 3, 1, 2)206                image = torch.from_numpy(image)207 208            elif isinstance(image[0], torch.Tensor):209                image = torch.stack(image, dim=0)210 211        image_batch_size = image.shape[0]212 213        if image_batch_size == 1:214            repeat_by = batch_size215        else:216            repeat_by = num_images_per_prompt217 218        image = image.repeat_interleave(repeat_by, dim=0)219 220        image = image.to(device=device, dtype=dtype)221 222        if do_classifier_free_guidance and not guess_mode:223            image = torch.cat([image] * 2)224 225        return image226 227    def check_ref_inputs(228        self,229        ref_image,230        reference_guidance_start,231        reference_guidance_end,232        style_fidelity,233        reference_attn,234        reference_adain,235    ):236        ref_image_is_pil = isinstance(ref_image, PIL.Image.Image)237        ref_image_is_tensor = isinstance(ref_image, torch.Tensor)238 239        if not ref_image_is_pil and not ref_image_is_tensor:240            raise TypeError(241                f"ref image must be passed and be one of PIL image or torch tensor, but is {type(ref_image)}"242            )243 244        if not reference_attn and not reference_adain:245            raise ValueError("`reference_attn` or `reference_adain` must be True.")246 247        if style_fidelity < 0.0:248            raise ValueError(f"style fidelity: {style_fidelity} can't be smaller than 0.")249        if style_fidelity > 1.0:250            raise ValueError(f"style fidelity: {style_fidelity} can't be larger than 1.0.")251 252        if reference_guidance_start >= reference_guidance_end:253            raise ValueError(254                f"reference guidance start: {reference_guidance_start} cannot be larger or equal to reference guidance end: {reference_guidance_end}."255            )256        if reference_guidance_start < 0.0:257            raise ValueError(f"reference guidance start: {reference_guidance_start} can't be smaller than 0.")258        if reference_guidance_end > 1.0:259            raise ValueError(f"reference guidance end: {reference_guidance_end} can't be larger than 1.0.")260 261    @torch.no_grad()262    @replace_example_docstring(EXAMPLE_DOC_STRING)263    def __call__(264        self,265        prompt: Union[str, List[str]] = None,266        prompt_2: Optional[Union[str, List[str]]] = None,267        ref_image: Union[torch.Tensor, PIL.Image.Image] = None,268        height: Optional[int] = None,269        width: Optional[int] = None,270        num_inference_steps: int = 50,271        timesteps: List[int] = None,272        sigmas: List[float] = None,273        denoising_end: Optional[float] = None,274        guidance_scale: float = 5.0,275        negative_prompt: Optional[Union[str, List[str]]] = None,276        negative_prompt_2: Optional[Union[str, List[str]]] = None,277        num_images_per_prompt: Optional[int] = 1,278        eta: float = 0.0,279        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,280        latents: Optional[torch.Tensor] = None,281        prompt_embeds: Optional[torch.Tensor] = None,282        negative_prompt_embeds: Optional[torch.Tensor] = None,283        pooled_prompt_embeds: Optional[torch.Tensor] = None,284        negative_pooled_prompt_embeds: Optional[torch.Tensor] = None,285        ip_adapter_image: Optional[PipelineImageInput] = None,286        ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None,287        output_type: Optional[str] = "pil",288        return_dict: bool = True,289        cross_attention_kwargs: Optional[Dict[str, Any]] = None,290        guidance_rescale: float = 0.0,291        original_size: Optional[Tuple[int, int]] = None,292        crops_coords_top_left: Tuple[int, int] = (0, 0),293        target_size: Optional[Tuple[int, int]] = None,294        negative_original_size: Optional[Tuple[int, int]] = None,295        negative_crops_coords_top_left: Tuple[int, int] = (0, 0),296        negative_target_size: Optional[Tuple[int, int]] = None,297        clip_skip: Optional[int] = None,298        callback_on_step_end: Optional[299            Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks]300        ] = None,301        callback_on_step_end_tensor_inputs: List[str] = ["latents"],302        attention_auto_machine_weight: float = 1.0,303        gn_auto_machine_weight: float = 1.0,304        reference_guidance_start: float = 0.0,305        reference_guidance_end: float = 1.0,306        style_fidelity: float = 0.5,307        reference_attn: bool = True,308        reference_adain: bool = True,309        **kwargs,310    ):311        r"""312        Function invoked when calling the pipeline for generation.313 314        Args:315            prompt (`str` or `List[str]`, *optional*):316                The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.317                instead.318            prompt_2 (`str` or `List[str]`, *optional*):319                The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is320                used in both text-encoders321            ref_image (`torch.Tensor`, `PIL.Image.Image`):322                The Reference Control input condition. Reference Control uses this input condition to generate guidance to Unet. If323                the type is specified as `Torch.Tensor`, it is passed to Reference Control as is. `PIL.Image.Image` can324                also be accepted as an image.325            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):326                The height in pixels of the generated image. This is set to 1024 by default for the best results.327                Anything below 512 pixels won't work well for328                [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)329                and checkpoints that are not specifically fine-tuned on low resolutions.330            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):331                The width in pixels of the generated image. This is set to 1024 by default for the best results.332                Anything below 512 pixels won't work well for333                [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)334                and checkpoints that are not specifically fine-tuned on low resolutions.335            num_inference_steps (`int`, *optional*, defaults to 50):336                The number of denoising steps. More denoising steps usually lead to a higher quality image at the337                expense of slower inference.338            timesteps (`List[int]`, *optional*):339                Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument340                in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is341                passed will be used. Must be in descending order.342            sigmas (`List[float]`, *optional*):343                Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in344                their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed345                will be used.346            denoising_end (`float`, *optional*):347                When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be348                completed before it is intentionally prematurely terminated. As a result, the returned sample will349                still retain a substantial amount of noise as determined by the discrete timesteps selected by the350                scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a351                "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image352                Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output)353            guidance_scale (`float`, *optional*, defaults to 5.0):354                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).355                `guidance_scale` is defined as `w` of equation 2. of [Imagen356                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >357                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,358                usually at the expense of lower image quality.359            negative_prompt (`str` or `List[str]`, *optional*):360                The prompt or prompts not to guide the image generation. If not defined, one has to pass361                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is362                less than `1`).363            negative_prompt_2 (`str` or `List[str]`, *optional*):364                The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and365                `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders366            num_images_per_prompt (`int`, *optional*, defaults to 1):367                The number of images to generate per prompt.368            eta (`float`, *optional*, defaults to 0.0):369                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to370                [`schedulers.DDIMScheduler`], will be ignored for others.371            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):372                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)373                to make generation deterministic.374            latents (`torch.Tensor`, *optional*):375                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image376                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents377                tensor will ge generated by sampling using the supplied random `generator`.378            prompt_embeds (`torch.Tensor`, *optional*):379                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not380                provided, text embeddings will be generated from `prompt` input argument.381            negative_prompt_embeds (`torch.Tensor`, *optional*):382                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt383                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input384                argument.385            pooled_prompt_embeds (`torch.Tensor`, *optional*):386                Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.387                If not provided, pooled text embeddings will be generated from `prompt` input argument.388            negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):389                Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt390                weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`391                input argument.392            ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.393            ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*):394                Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of395                IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should396                contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not397                provided, embeddings are computed from the `ip_adapter_image` input argument.398            output_type (`str`, *optional*, defaults to `"pil"`):399                The output format of the generate image. Choose between400                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.401            return_dict (`bool`, *optional*, defaults to `True`):402                Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead403                of a plain tuple.404            cross_attention_kwargs (`dict`, *optional*):405                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under406                `self.processor` in407                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).408            guidance_rescale (`float`, *optional*, defaults to 0.0):409                Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are410                Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of411                [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf).412                Guidance rescale factor should fix overexposure when using zero terminal SNR.413            original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):414                If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.415                `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as416                explained in section 2.2 of417                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).418            crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):419                `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position420                `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting421                `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of422                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).423            target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):424                For most cases, `target_size` should be set to the desired height and width of the generated image. If425                not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in426                section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).427            negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):428                To negatively condition the generation process based on a specific image resolution. Part of SDXL's429                micro-conditioning as explained in section 2.2 of430                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more431                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.432            negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):433                To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's434                micro-conditioning as explained in section 2.2 of435                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more436                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.437            negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):438                To negatively condition the generation process based on a target image resolution. It should be as same439                as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of440                [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more441                information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.442            callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):443                A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of444                each denoising step during the inference. with the following arguments: `callback_on_step_end(self:445                DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a446                list of all tensors as specified by `callback_on_step_end_tensor_inputs`.447            callback_on_step_end_tensor_inputs (`List`, *optional*):448                The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list449                will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the450                `._callback_tensor_inputs` attribute of your pipeline class.451            attention_auto_machine_weight (`float`):452                Weight of using reference query for self attention's context.453                If attention_auto_machine_weight=1.0, use reference query for all self attention's context.454            gn_auto_machine_weight (`float`):455                Weight of using reference adain. If gn_auto_machine_weight=2.0, use all reference adain plugins.456            reference_guidance_start (`float`, *optional*, defaults to 0.0):457                The percentage of total steps at which the reference ControlNet starts applying.458            reference_guidance_end (`float`, *optional*, defaults to 1.0):459                The percentage of total steps at which the reference ControlNet stops applying.460            style_fidelity (`float`):461                style fidelity of ref_uncond_xt. If style_fidelity=1.0, control more important,462                elif style_fidelity=0.0, prompt more important, else balanced.463            reference_attn (`bool`):464                Whether to use reference query for self attention's context.465            reference_adain (`bool`):466                Whether to use reference adain.467 468        Examples:469 470        Returns:471            [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`:472            [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a473            `tuple`. When returning a tuple, the first element is a list with the generated images.474        """475 476        callback = kwargs.pop("callback", None)477        callback_steps = kwargs.pop("callback_steps", None)478 479        if callback is not None:480            deprecate(481                "callback",482                "1.0.0",483                "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",484            )485        if callback_steps is not None:486            deprecate(487                "callback_steps",488                "1.0.0",489                "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",490            )491 492        if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):493            callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs494 495        # 0. Default height and width to unet496        height = height or self.default_sample_size * self.vae_scale_factor497        width = width or self.default_sample_size * self.vae_scale_factor498 499        original_size = original_size or (height, width)500        target_size = target_size or (height, width)501 502        # 1. Check inputs. Raise error if not correct503        self.check_inputs(504            prompt,505            prompt_2,506            height,507            width,508            callback_steps,509            negative_prompt,510            negative_prompt_2,511            prompt_embeds,512            negative_prompt_embeds,513            pooled_prompt_embeds,514            negative_pooled_prompt_embeds,515            ip_adapter_image,516            ip_adapter_image_embeds,517            callback_on_step_end_tensor_inputs,518        )519 520        self.check_ref_inputs(521            ref_image,522            reference_guidance_start,523            reference_guidance_end,524            style_fidelity,525            reference_attn,526            reference_adain,527        )528 529        self._guidance_scale = guidance_scale530        self._guidance_rescale = guidance_rescale531        self._clip_skip = clip_skip532        self._cross_attention_kwargs = cross_attention_kwargs533        self._denoising_end = denoising_end534        self._interrupt = False535 536        # 2. Define call parameters537        if prompt is not None and isinstance(prompt, str):538            batch_size = 1539        elif prompt is not None and isinstance(prompt, list):540            batch_size = len(prompt)541        else:542            batch_size = prompt_embeds.shape[0]543 544        device = self._execution_device545 546        # 3. Encode input prompt547        lora_scale = (548            self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None549        )550 551        (552            prompt_embeds,553            negative_prompt_embeds,554            pooled_prompt_embeds,555            negative_pooled_prompt_embeds,556        ) = self.encode_prompt(557            prompt=prompt,558            prompt_2=prompt_2,559            device=device,560            num_images_per_prompt=num_images_per_prompt,561            do_classifier_free_guidance=self.do_classifier_free_guidance,562            negative_prompt=negative_prompt,563            negative_prompt_2=negative_prompt_2,564            prompt_embeds=prompt_embeds,565            negative_prompt_embeds=negative_prompt_embeds,566            pooled_prompt_embeds=pooled_prompt_embeds,567            negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,568            lora_scale=lora_scale,569            clip_skip=self.clip_skip,570        )571 572        # 4. Preprocess reference image573        ref_image = self.prepare_ref_image(574            image=ref_image,575            width=width,576            height=height,577            batch_size=batch_size * num_images_per_prompt,578            num_images_per_prompt=num_images_per_prompt,579            device=device,580            dtype=prompt_embeds.dtype,581        )582 583        # 5. Prepare timesteps584        timesteps, num_inference_steps = retrieve_timesteps(585            self.scheduler, num_inference_steps, device, timesteps, sigmas586        )587 588        # 6. Prepare latent variables589        num_channels_latents = self.unet.config.in_channels590        latents = self.prepare_latents(591            batch_size * num_images_per_prompt,592            num_channels_latents,593            height,594            width,595            prompt_embeds.dtype,596            device,597            generator,598            latents,599        )600 601        # 7. Prepare reference latent variables602        ref_image_latents = self.prepare_ref_latents(603            ref_image,604            batch_size * num_images_per_prompt,605            prompt_embeds.dtype,606            device,607            generator,608            self.do_classifier_free_guidance,609        )610 611        # 8. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline612        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)613 614        # 8.1 Create tensor stating which reference controlnets to keep615        reference_keeps = []616        for i in range(len(timesteps)):617            reference_keep = 1.0 - float(618                i / len(timesteps) < reference_guidance_start or (i + 1) / len(timesteps) > reference_guidance_end619            )620            reference_keeps.append(reference_keep)621 622        # 8.2 Modify self attention and group norm623        MODE = "write"624        uc_mask = (625            torch.Tensor([1] * batch_size * num_images_per_prompt + [0] * batch_size * num_images_per_prompt)626            .type_as(ref_image_latents)627            .bool()628        )629 630        do_classifier_free_guidance = self.do_classifier_free_guidance631 632        def hacked_basic_transformer_inner_forward(633            self,634            hidden_states: torch.Tensor,635            attention_mask: Optional[torch.Tensor] = None,636            encoder_hidden_states: Optional[torch.Tensor] = None,637            encoder_attention_mask: Optional[torch.Tensor] = None,638            timestep: Optional[torch.LongTensor] = None,639            cross_attention_kwargs: Dict[str, Any] = None,640            class_labels: Optional[torch.LongTensor] = None,641        ):642            if self.use_ada_layer_norm:643                norm_hidden_states = self.norm1(hidden_states, timestep)644            elif self.use_ada_layer_norm_zero:645                norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(646                    hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype647                )648            else:649                norm_hidden_states = self.norm1(hidden_states)650 651            # 1. Self-Attention652            cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {}653            if self.only_cross_attention:654                attn_output = self.attn1(655                    norm_hidden_states,656                    encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,657                    attention_mask=attention_mask,658                    **cross_attention_kwargs,659                )660            else:661                if MODE == "write":662                    self.bank.append(norm_hidden_states.detach().clone())663                    attn_output = self.attn1(664                        norm_hidden_states,665                        encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,666                        attention_mask=attention_mask,667                        **cross_attention_kwargs,668                    )669                if MODE == "read":670                    if attention_auto_machine_weight > self.attn_weight:671                        attn_output_uc = self.attn1(672                            norm_hidden_states,673                            encoder_hidden_states=torch.cat([norm_hidden_states] + self.bank, dim=1),674                            # attention_mask=attention_mask,675                            **cross_attention_kwargs,676                        )677                        attn_output_c = attn_output_uc.clone()678                        if do_classifier_free_guidance and style_fidelity > 0:679                            attn_output_c[uc_mask] = self.attn1(680                                norm_hidden_states[uc_mask],681                                encoder_hidden_states=norm_hidden_states[uc_mask],682                                **cross_attention_kwargs,683                            )684                        attn_output = style_fidelity * attn_output_c + (1.0 - style_fidelity) * attn_output_uc685                        self.bank.clear()686                    else:687                        attn_output = self.attn1(688                            norm_hidden_states,689                            encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,690                            attention_mask=attention_mask,691                            **cross_attention_kwargs,692                        )693            if self.use_ada_layer_norm_zero:694                attn_output = gate_msa.unsqueeze(1) * attn_output695            hidden_states = attn_output + hidden_states696 697            if self.attn2 is not None:698                norm_hidden_states = (699                    self.norm2(hidden_states, timestep) if self.use_ada_layer_norm else self.norm2(hidden_states)700                )701 702                # 2. Cross-Attention703                attn_output = self.attn2(704                    norm_hidden_states,705                    encoder_hidden_states=encoder_hidden_states,706                    attention_mask=encoder_attention_mask,707                    **cross_attention_kwargs,708                )709                hidden_states = attn_output + hidden_states710 711            # 3. Feed-forward712            norm_hidden_states = self.norm3(hidden_states)713 714            if self.use_ada_layer_norm_zero:715                norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]716 717            ff_output = self.ff(norm_hidden_states)718 719            if self.use_ada_layer_norm_zero:720                ff_output = gate_mlp.unsqueeze(1) * ff_output721 722            hidden_states = ff_output + hidden_states723 724            return hidden_states725 726        def hacked_mid_forward(self, *args, **kwargs):727            eps = 1e-6728            x = self.original_forward(*args, **kwargs)729            if MODE == "write":730                if gn_auto_machine_weight >= self.gn_weight:731                    var, mean = torch.var_mean(x, dim=(2, 3), keepdim=True, correction=0)732                    self.mean_bank.append(mean)733                    self.var_bank.append(var)734            if MODE == "read":735                if len(self.mean_bank) > 0 and len(self.var_bank) > 0:736                    var, mean = torch.var_mean(x, dim=(2, 3), keepdim=True, correction=0)737                    std = torch.maximum(var, torch.zeros_like(var) + eps) ** 0.5738                    mean_acc = sum(self.mean_bank) / float(len(self.mean_bank))739                    var_acc = sum(self.var_bank) / float(len(self.var_bank))740                    std_acc = torch.maximum(var_acc, torch.zeros_like(var_acc) + eps) ** 0.5741                    x_uc = (((x - mean) / std) * std_acc) + mean_acc742                    x_c = x_uc.clone()743                    if do_classifier_free_guidance and style_fidelity > 0:744                        x_c[uc_mask] = x[uc_mask]745                    x = style_fidelity * x_c + (1.0 - style_fidelity) * x_uc746                self.mean_bank = []747                self.var_bank = []748            return x749 750        def hack_CrossAttnDownBlock2D_forward(751            self,752            hidden_states: torch.Tensor,753            temb: Optional[torch.Tensor] = None,754            encoder_hidden_states: Optional[torch.Tensor] = None,755            attention_mask: Optional[torch.Tensor] = None,756            cross_attention_kwargs: Optional[Dict[str, Any]] = None,757            encoder_attention_mask: Optional[torch.Tensor] = None,758        ):759            eps = 1e-6760 761            # TODO(Patrick, William) - attention mask is not used762            output_states = ()763 764            for i, (resnet, attn) in enumerate(zip(self.resnets, self.attentions)):765                hidden_states = resnet(hidden_states, temb)766                hidden_states = attn(767                    hidden_states,768                    encoder_hidden_states=encoder_hidden_states,769                    cross_attention_kwargs=cross_attention_kwargs,770                    attention_mask=attention_mask,771                    encoder_attention_mask=encoder_attention_mask,772                    return_dict=False,773                )[0]774                if MODE == "write":775                    if gn_auto_machine_weight >= self.gn_weight:776                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)777                        self.mean_bank.append([mean])778                        self.var_bank.append([var])779                if MODE == "read":780                    if len(self.mean_bank) > 0 and len(self.var_bank) > 0:781                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)782                        std = torch.maximum(var, torch.zeros_like(var) + eps) ** 0.5783                        mean_acc = sum(self.mean_bank[i]) / float(len(self.mean_bank[i]))784                        var_acc = sum(self.var_bank[i]) / float(len(self.var_bank[i]))785                        std_acc = torch.maximum(var_acc, torch.zeros_like(var_acc) + eps) ** 0.5786                        hidden_states_uc = (((hidden_states - mean) / std) * std_acc) + mean_acc787                        hidden_states_c = hidden_states_uc.clone()788                        if do_classifier_free_guidance and style_fidelity > 0:789                            hidden_states_c[uc_mask] = hidden_states[uc_mask]790                        hidden_states = style_fidelity * hidden_states_c + (1.0 - style_fidelity) * hidden_states_uc791 792                output_states = output_states + (hidden_states,)793 794            if MODE == "read":795                self.mean_bank = []796                self.var_bank = []797 798            if self.downsamplers is not None:799                for downsampler in self.downsamplers:800                    hidden_states = downsampler(hidden_states)801 802                output_states = output_states + (hidden_states,)803 804            return hidden_states, output_states805 806        def hacked_DownBlock2D_forward(self, hidden_states, temb=None, *args, **kwargs):807            eps = 1e-6808 809            output_states = ()810 811            for i, resnet in enumerate(self.resnets):812                hidden_states = resnet(hidden_states, temb)813 814                if MODE == "write":815                    if gn_auto_machine_weight >= self.gn_weight:816                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)817                        self.mean_bank.append([mean])818                        self.var_bank.append([var])819                if MODE == "read":820                    if len(self.mean_bank) > 0 and len(self.var_bank) > 0:821                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)822                        std = torch.maximum(var, torch.zeros_like(var) + eps) ** 0.5823                        mean_acc = sum(self.mean_bank[i]) / float(len(self.mean_bank[i]))824                        var_acc = sum(self.var_bank[i]) / float(len(self.var_bank[i]))825                        std_acc = torch.maximum(var_acc, torch.zeros_like(var_acc) + eps) ** 0.5826                        hidden_states_uc = (((hidden_states - mean) / std) * std_acc) + mean_acc827                        hidden_states_c = hidden_states_uc.clone()828                        if do_classifier_free_guidance and style_fidelity > 0:829                            hidden_states_c[uc_mask] = hidden_states[uc_mask]830                        hidden_states = style_fidelity * hidden_states_c + (1.0 - style_fidelity) * hidden_states_uc831 832                output_states = output_states + (hidden_states,)833 834            if MODE == "read":835                self.mean_bank = []836                self.var_bank = []837 838            if self.downsamplers is not None:839                for downsampler in self.downsamplers:840                    hidden_states = downsampler(hidden_states)841 842                output_states = output_states + (hidden_states,)843 844            return hidden_states, output_states845 846        def hacked_CrossAttnUpBlock2D_forward(847            self,848            hidden_states: torch.Tensor,849            res_hidden_states_tuple: Tuple[torch.Tensor, ...],850            temb: Optional[torch.Tensor] = None,851            encoder_hidden_states: Optional[torch.Tensor] = None,852            cross_attention_kwargs: Optional[Dict[str, Any]] = None,853            upsample_size: Optional[int] = None,854            attention_mask: Optional[torch.Tensor] = None,855            encoder_attention_mask: Optional[torch.Tensor] = None,856        ):857            eps = 1e-6858            # TODO(Patrick, William) - attention mask is not used859            for i, (resnet, attn) in enumerate(zip(self.resnets, self.attentions)):860                # pop res hidden states861                res_hidden_states = res_hidden_states_tuple[-1]862                res_hidden_states_tuple = res_hidden_states_tuple[:-1]863                hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)864                hidden_states = resnet(hidden_states, temb)865                hidden_states = attn(866                    hidden_states,867                    encoder_hidden_states=encoder_hidden_states,868                    cross_attention_kwargs=cross_attention_kwargs,869                    attention_mask=attention_mask,870                    encoder_attention_mask=encoder_attention_mask,871                    return_dict=False,872                )[0]873 874                if MODE == "write":875                    if gn_auto_machine_weight >= self.gn_weight:876                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)877                        self.mean_bank.append([mean])878                        self.var_bank.append([var])879                if MODE == "read":880                    if len(self.mean_bank) > 0 and len(self.var_bank) > 0:881                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)882                        std = torch.maximum(var, torch.zeros_like(var) + eps) ** 0.5883                        mean_acc = sum(self.mean_bank[i]) / float(len(self.mean_bank[i]))884                        var_acc = sum(self.var_bank[i]) / float(len(self.var_bank[i]))885                        std_acc = torch.maximum(var_acc, torch.zeros_like(var_acc) + eps) ** 0.5886                        hidden_states_uc = (((hidden_states - mean) / std) * std_acc) + mean_acc887                        hidden_states_c = hidden_states_uc.clone()888                        if do_classifier_free_guidance and style_fidelity > 0:889                            hidden_states_c[uc_mask] = hidden_states[uc_mask]890                        hidden_states = style_fidelity * hidden_states_c + (1.0 - style_fidelity) * hidden_states_uc891 892            if MODE == "read":893                self.mean_bank = []894                self.var_bank = []895 896            if self.upsamplers is not None:897                for upsampler in self.upsamplers:898                    hidden_states = upsampler(hidden_states, upsample_size)899 900            return hidden_states901 902        def hacked_UpBlock2D_forward(903            self, hidden_states, res_hidden_states_tuple, temb=None, upsample_size=None, *args, **kwargs904        ):905            eps = 1e-6906            for i, resnet in enumerate(self.resnets):907                # pop res hidden states908                res_hidden_states = res_hidden_states_tuple[-1]909                res_hidden_states_tuple = res_hidden_states_tuple[:-1]910                hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)911                hidden_states = resnet(hidden_states, temb)912 913                if MODE == "write":914                    if gn_auto_machine_weight >= self.gn_weight:915                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)916                        self.mean_bank.append([mean])917                        self.var_bank.append([var])918                if MODE == "read":919                    if len(self.mean_bank) > 0 and len(self.var_bank) > 0:920                        var, mean = torch.var_mean(hidden_states, dim=(2, 3), keepdim=True, correction=0)921                        std = torch.maximum(var, torch.zeros_like(var) + eps) ** 0.5922                        mean_acc = sum(self.mean_bank[i]) / float(len(self.mean_bank[i]))923                        var_acc = sum(self.var_bank[i]) / float(len(self.var_bank[i]))924                        std_acc = torch.maximum(var_acc, torch.zeros_like(var_acc) + eps) ** 0.5925                        hidden_states_uc = (((hidden_states - mean) / std) * std_acc) + mean_acc926                        hidden_states_c = hidden_states_uc.clone()927                        if do_classifier_free_guidance and style_fidelity > 0:928                            hidden_states_c[uc_mask] = hidden_states[uc_mask]929                        hidden_states = style_fidelity * hidden_states_c + (1.0 - style_fidelity) * hidden_states_uc930 931            if MODE == "read":932                self.mean_bank = []933                self.var_bank = []934 935            if self.upsamplers is not None:936                for upsampler in self.upsamplers:937                    hidden_states = upsampler(hidden_states, upsample_size)938 939            return hidden_states940 941        if reference_attn:942            attn_modules = [module for module in torch_dfs(self.unet) if isinstance(module, BasicTransformerBlock)]943            attn_modules = sorted(attn_modules, key=lambda x: -x.norm1.normalized_shape[0])944 945            for i, module in enumerate(attn_modules):946                module._original_inner_forward = module.forward947                module.forward = hacked_basic_transformer_inner_forward.__get__(module, BasicTransformerBlock)948                module.bank = []949                module.attn_weight = float(i) / float(len(attn_modules))950 951        if reference_adain:952            gn_modules = [self.unet.mid_block]953            self.unet.mid_block.gn_weight = 0954 955            down_blocks = self.unet.down_blocks956            for w, module in enumerate(down_blocks):957                module.gn_weight = 1.0 - float(w) / float(len(down_blocks))958                gn_modules.append(module)959 960            up_blocks = self.unet.up_blocks961            for w, module in enumerate(up_blocks):962                module.gn_weight = float(w) / float(len(up_blocks))963                gn_modules.append(module)964 965            for i, module in enumerate(gn_modules):966                if getattr(module, "original_forward", None) is None:967                    module.original_forward = module.forward968                if i == 0:969                    # mid_block970                    module.forward = hacked_mid_forward.__get__(module, torch.nn.Module)971                elif isinstance(module, CrossAttnDownBlock2D):972                    module.forward = hack_CrossAttnDownBlock2D_forward.__get__(module, CrossAttnDownBlock2D)973                elif isinstance(module, DownBlock2D):974                    module.forward = hacked_DownBlock2D_forward.__get__(module, DownBlock2D)975                elif isinstance(module, CrossAttnUpBlock2D):976                    module.forward = hacked_CrossAttnUpBlock2D_forward.__get__(module, CrossAttnUpBlock2D)977                elif isinstance(module, UpBlock2D):978                    module.forward = hacked_UpBlock2D_forward.__get__(module, UpBlock2D)979                module.mean_bank = []980                module.var_bank = []981                module.gn_weight *= 2982 983        # 9. Prepare added time ids & embeddings984        add_text_embeds = pooled_prompt_embeds985        if self.text_encoder_2 is None:986            text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])987        else:988            text_encoder_projection_dim = self.text_encoder_2.config.projection_dim989 990        add_time_ids = self._get_add_time_ids(991            original_size,992            crops_coords_top_left,993            target_size,994            dtype=prompt_embeds.dtype,995            text_encoder_projection_dim=text_encoder_projection_dim,996        )997        if negative_original_size is not None and negative_target_size is not None:998            negative_add_time_ids = self._get_add_time_ids(999                negative_original_size,1000                negative_crops_coords_top_left,1001                negative_target_size,1002                dtype=prompt_embeds.dtype,1003                text_encoder_projection_dim=text_encoder_projection_dim,1004            )1005        else:1006            negative_add_time_ids = add_time_ids1007 1008        if self.do_classifier_free_guidance:1009            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)1010            add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)1011            add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0)1012 1013        prompt_embeds = prompt_embeds.to(device)1014        add_text_embeds = add_text_embeds.to(device)1015        add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)1016 1017        if ip_adapter_image is not None or ip_adapter_image_embeds is not None:1018            image_embeds = self.prepare_ip_adapter_image_embeds(1019                ip_adapter_image,1020                ip_adapter_image_embeds,1021                device,1022                batch_size * num_images_per_prompt,1023                self.do_classifier_free_guidance,1024            )1025 1026        # 10. Denoising loop1027        num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)1028 1029        # 10.1 Apply denoising_end1030        if (1031            self.denoising_end is not None1032            and isinstance(self.denoising_end, float)1033            and self.denoising_end > 01034            and self.denoising_end < 11035        ):1036            discrete_timestep_cutoff = int(1037                round(1038                    self.scheduler.config.num_train_timesteps1039                    - (self.denoising_end * self.scheduler.config.num_train_timesteps)1040                )1041            )1042            num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))1043            timesteps = timesteps[:num_inference_steps]1044 1045        # 11. Optionally get Guidance Scale Embedding1046        timestep_cond = None1047        if self.unet.config.time_cond_proj_dim is not None:1048            guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)1049            timestep_cond = self.get_guidance_scale_embedding(1050                guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim1051            ).to(device=device, dtype=latents.dtype)1052 1053        self._num_timesteps = len(timesteps)1054        with self.progress_bar(total=num_inference_steps) as progress_bar:1055            for i, t in enumerate(timesteps):1056                if self.interrupt:1057                    continue1058 1059                # expand the latents if we are doing classifier free guidance1060                latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents1061 1062                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)1063 1064                # predict the noise residual1065                added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}1066                if ip_adapter_image is not None or ip_adapter_image_embeds is not None:1067                    added_cond_kwargs["image_embeds"] = image_embeds1068 1069                # ref only part1070                if reference_keeps[i] > 0:1071                    noise = randn_tensor(1072                        ref_image_latents.shape, generator=generator, device=device, dtype=ref_image_latents.dtype1073                    )1074                    ref_xt = self.scheduler.add_noise(1075                        ref_image_latents,1076                        noise,1077                        t.reshape(1078                            1,1079                        ),1080                    )1081                    ref_xt = self.scheduler.scale_model_input(ref_xt, t)1082 1083                    MODE = "write"1084                    self.unet(1085                        ref_xt,1086                        t,1087                        encoder_hidden_states=prompt_embeds,1088                        cross_attention_kwargs=cross_attention_kwargs,1089                        added_cond_kwargs=added_cond_kwargs,1090                        return_dict=False,1091                    )1092 1093                # predict the noise residual1094                MODE = "read"1095                noise_pred = self.unet(1096                    latent_model_input,1097                    t,1098                    encoder_hidden_states=prompt_embeds,1099                    timestep_cond=timestep_cond,1100                    cross_attention_kwargs=self.cross_attention_kwargs,1101                    added_cond_kwargs=added_cond_kwargs,1102                    return_dict=False,1103                )[0]1104 1105                # perform guidance1106                if self.do_classifier_free_guidance:1107                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)1108                    noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)1109 1110                if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:1111                    # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf1112                    noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale)1113 1114                # compute the previous noisy sample x_t -> x_t-11115                latents_dtype = latents.dtype1116                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]1117                if latents.dtype != latents_dtype:1118                    if torch.backends.mps.is_available():1119                        # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/992721120                        latents = latents.to(latents_dtype)1121 1122                if callback_on_step_end is not None:1123                    callback_kwargs = {}1124                    for k in callback_on_step_end_tensor_inputs:1125                        callback_kwargs[k] = locals()[k]1126                    callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)1127 1128                    latents = callback_outputs.pop("latents", latents)1129                    prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)1130                    negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)1131                    add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)1132                    negative_pooled_prompt_embeds = callback_outputs.pop(1133                        "negative_pooled_prompt_embeds", negative_pooled_prompt_embeds1134                    )1135                    add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)1136                    negative_add_time_ids = callback_outputs.pop("negative_add_time_ids", negative_add_time_ids)1137 1138                # call the callback, if provided1139                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):1140                    progress_bar.update()1141                    if callback is not None and i % callback_steps == 0:1142                        step_idx = i // getattr(self.scheduler, "order", 1)1143                        callback(step_idx, t, latents)1144 1145                if XLA_AVAILABLE:1146                    xm.mark_step()1147 1148        if not output_type == "latent":1149            # make sure the VAE is in float32 mode, as it overflows in float161150            needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast1151 1152            if needs_upcasting:1153                self.upcast_vae()1154                latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)1155            elif latents.dtype != self.vae.dtype:1156                if torch.backends.mps.is_available():1157                    # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/992721158                    self.vae = self.vae.to(latents.dtype)1159 1160            # unscale/denormalize the latents1161            # denormalize with the mean and std if available and not None1162            has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None1163            has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None1164            if has_latents_mean and has_latents_std:1165                latents_mean = (1166                    torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype)1167                )1168                latents_std = (1169                    torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype)1170                )1171                latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean1172            else:1173                latents = latents / self.vae.config.scaling_factor1174 1175            image = self.vae.decode(latents, return_dict=False)[0]1176 1177            # cast back to fp16 if needed1178            if needs_upcasting:1179                self.vae.to(dtype=torch.float16)1180        else:1181            image = latents1182 1183        if not output_type == "latent":1184            # apply watermark if available1185            if self.watermark is not None:1186                image = self.watermark.apply_watermark(image)1187 1188            image = self.image_processor.postprocess(image, output_type=output_type)1189 1190        # Offload all models1191        self.maybe_free_model_hooks()1192 1193        if not return_dict:1194            return (image,)1195 1196        return StableDiffusionXLPipelineOutput(images=image)1197