diffusers/community-pipelines-mirror

1# Copyright 2024 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import inspect16from typing import Any, Callable, Dict, List, Optional, Union17 18import torch19import torch.nn as nn20import torch.nn.functional as F21from packaging import version22from safetensors import safe_open23from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection24 25from diffusers.configuration_utils import FrozenDict26from diffusers.image_processor import VaeImageProcessor27from diffusers.loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin28from diffusers.models import AutoencoderKL, UNet2DConditionModel29from diffusers.models.attention_processor import (30    AttnProcessor,31    AttnProcessor2_0,32    IPAdapterAttnProcessor,33    IPAdapterAttnProcessor2_0,34)35from diffusers.models.embeddings import MultiIPAdapterImageProjection36from diffusers.models.lora import adjust_lora_scale_text_encoder37from diffusers.pipelines.pipeline_utils import DiffusionPipeline, StableDiffusionMixin38from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput39from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker40from diffusers.schedulers import KarrasDiffusionSchedulers41from diffusers.utils import (42    USE_PEFT_BACKEND,43    _get_model_file,44    deprecate,45    logging,46    scale_lora_layers,47    unscale_lora_layers,48)49from diffusers.utils.torch_utils import randn_tensor50 51 52logger = logging.get_logger(__name__)  # pylint: disable=invalid-name53 54 55class IPAdapterFullImageProjection(nn.Module):56    def __init__(self, image_embed_dim=1024, cross_attention_dim=1024, mult=1, num_tokens=1):57        super().__init__()58        from diffusers.models.attention import FeedForward59 60        self.num_tokens = num_tokens61        self.cross_attention_dim = cross_attention_dim62        self.ff = FeedForward(image_embed_dim, cross_attention_dim * num_tokens, mult=mult, activation_fn="gelu")63        self.norm = nn.LayerNorm(cross_attention_dim)64 65    def forward(self, image_embeds: torch.Tensor):66        x = self.ff(image_embeds)67        x = x.reshape(-1, self.num_tokens, self.cross_attention_dim)68        return self.norm(x)69 70 71def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):72    """73    Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and74    Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.475    """76    std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)77    std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)78    # rescale the results from guidance (fixes overexposure)79    noise_pred_rescaled = noise_cfg * (std_text / std_cfg)80    # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images81    noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg82    return noise_cfg83 84 85def retrieve_timesteps(86    scheduler,87    num_inference_steps: Optional[int] = None,88    device: Optional[Union[str, torch.device]] = None,89    timesteps: Optional[List[int]] = None,90    **kwargs,91):92    """93    Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles94    custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.95 96    Args:97        scheduler (`SchedulerMixin`):98            The scheduler to get timesteps from.99        num_inference_steps (`int`):100            The number of diffusion steps used when generating samples with a pre-trained model. If used,101            `timesteps` must be `None`.102        device (`str` or `torch.device`, *optional*):103            The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.104        timesteps (`List[int]`, *optional*):105                Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default106                timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps`107                must be `None`.108 109    Returns:110        `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the111        second element is the number of inference steps.112    """113    if timesteps is not None:114        accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())115        if not accepts_timesteps:116            raise ValueError(117                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"118                f" timestep schedules. Please check whether you are using the correct scheduler."119            )120        scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)121        timesteps = scheduler.timesteps122        num_inference_steps = len(timesteps)123    else:124        scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)125        timesteps = scheduler.timesteps126    return timesteps, num_inference_steps127 128 129class IPAdapterFaceIDStableDiffusionPipeline(130    DiffusionPipeline,131    StableDiffusionMixin,132    TextualInversionLoaderMixin,133    LoraLoaderMixin,134    IPAdapterMixin,135    FromSingleFileMixin,136):137    r"""138    Pipeline for text-to-image generation using Stable Diffusion.139 140    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods141    implemented for all pipelines (downloading, saving, running on a particular device, etc.).142 143    The pipeline also inherits the following loading methods:144        - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings145        - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights146        - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights147        - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files148        - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters149 150    Args:151        vae ([`AutoencoderKL`]):152            Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.153        text_encoder ([`~transformers.CLIPTextModel`]):154            Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).155        tokenizer ([`~transformers.CLIPTokenizer`]):156            A `CLIPTokenizer` to tokenize text.157        unet ([`UNet2DConditionModel`]):158            A `UNet2DConditionModel` to denoise the encoded image latents.159        scheduler ([`SchedulerMixin`]):160            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of161            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].162        safety_checker ([`StableDiffusionSafetyChecker`]):163            Classification module that estimates whether generated images could be considered offensive or harmful.164            Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details165            about a model's potential harms.166        feature_extractor ([`~transformers.CLIPImageProcessor`]):167            A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`.168    """169 170    model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae"171    _optional_components = ["safety_checker", "feature_extractor", "image_encoder"]172    _exclude_from_cpu_offload = ["safety_checker"]173    _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]174 175    def __init__(176        self,177        vae: AutoencoderKL,178        text_encoder: CLIPTextModel,179        tokenizer: CLIPTokenizer,180        unet: UNet2DConditionModel,181        scheduler: KarrasDiffusionSchedulers,182        safety_checker: StableDiffusionSafetyChecker,183        feature_extractor: CLIPImageProcessor,184        image_encoder: CLIPVisionModelWithProjection = None,185        requires_safety_checker: bool = True,186    ):187        super().__init__()188 189        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:190            deprecation_message = (191                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"192                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "193                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"194                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"195                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"196                " file"197            )198            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)199            new_config = dict(scheduler.config)200            new_config["steps_offset"] = 1201            scheduler._internal_dict = FrozenDict(new_config)202 203        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:204            deprecation_message = (205                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."206                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"207                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"208                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"209                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"210            )211            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)212            new_config = dict(scheduler.config)213            new_config["clip_sample"] = False214            scheduler._internal_dict = FrozenDict(new_config)215 216        if safety_checker is None and requires_safety_checker:217            logger.warning(218                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"219                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"220                " results in services or applications open to the public. Both the diffusers team and Hugging Face"221                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"222                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"223                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."224            )225 226        if safety_checker is not None and feature_extractor is None:227            raise ValueError(228                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"229                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."230            )231 232        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(233            version.parse(unet.config._diffusers_version).base_version234        ) < version.parse("0.9.0.dev0")235        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64236        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:237            deprecation_message = (238                "The configuration file of the unet has set the default `sample_size` to smaller than"239                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"240                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"241                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"242                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"243                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"244                " in the config might lead to incorrect results in future versions. If you have downloaded this"245                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"246                " the `unet/config.json` file"247            )248            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)249            new_config = dict(unet.config)250            new_config["sample_size"] = 64251            unet._internal_dict = FrozenDict(new_config)252 253        self.register_modules(254            vae=vae,255            text_encoder=text_encoder,256            tokenizer=tokenizer,257            unet=unet,258            scheduler=scheduler,259            safety_checker=safety_checker,260            feature_extractor=feature_extractor,261            image_encoder=image_encoder,262        )263        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)264        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)265        self.register_to_config(requires_safety_checker=requires_safety_checker)266 267    def load_ip_adapter_face_id(self, pretrained_model_name_or_path_or_dict, weight_name, **kwargs):268        cache_dir = kwargs.pop("cache_dir", None)269        force_download = kwargs.pop("force_download", False)270        resume_download = kwargs.pop("resume_download", False)271        proxies = kwargs.pop("proxies", None)272        local_files_only = kwargs.pop("local_files_only", None)273        token = kwargs.pop("token", None)274        revision = kwargs.pop("revision", None)275        subfolder = kwargs.pop("subfolder", None)276 277        user_agent = {278            "file_type": "attn_procs_weights",279            "framework": "pytorch",280        }281        model_file = _get_model_file(282            pretrained_model_name_or_path_or_dict,283            weights_name=weight_name,284            cache_dir=cache_dir,285            force_download=force_download,286            resume_download=resume_download,287            proxies=proxies,288            local_files_only=local_files_only,289            token=token,290            revision=revision,291            subfolder=subfolder,292            user_agent=user_agent,293        )294        if weight_name.endswith(".safetensors"):295            state_dict = {"image_proj": {}, "ip_adapter": {}}296            with safe_open(model_file, framework="pt", device="cpu") as f:297                for key in f.keys():298                    if key.startswith("image_proj."):299                        state_dict["image_proj"][key.replace("image_proj.", "")] = f.get_tensor(key)300                    elif key.startswith("ip_adapter."):301                        state_dict["ip_adapter"][key.replace("ip_adapter.", "")] = f.get_tensor(key)302        else:303            state_dict = torch.load(model_file, map_location="cpu")304        self._load_ip_adapter_weights(state_dict)305 306    def convert_ip_adapter_image_proj_to_diffusers(self, state_dict):307        updated_state_dict = {}308        clip_embeddings_dim_in = state_dict["proj.0.weight"].shape[1]309        clip_embeddings_dim_out = state_dict["proj.0.weight"].shape[0]310        multiplier = clip_embeddings_dim_out // clip_embeddings_dim_in311        norm_layer = "norm.weight"312        cross_attention_dim = state_dict[norm_layer].shape[0]313        num_tokens = state_dict["proj.2.weight"].shape[0] // cross_attention_dim314 315        image_projection = IPAdapterFullImageProjection(316            cross_attention_dim=cross_attention_dim,317            image_embed_dim=clip_embeddings_dim_in,318            mult=multiplier,319            num_tokens=num_tokens,320        )321 322        for key, value in state_dict.items():323            diffusers_name = key.replace("proj.0", "ff.net.0.proj")324            diffusers_name = diffusers_name.replace("proj.2", "ff.net.2")325            updated_state_dict[diffusers_name] = value326 327        image_projection.load_state_dict(updated_state_dict)328        return image_projection329 330    def _load_ip_adapter_weights(self, state_dict):331        num_image_text_embeds = 4332 333        self.unet.encoder_hid_proj = None334 335        # set ip-adapter cross-attention processors & load state_dict336        attn_procs = {}337        lora_dict = {}338        key_id = 0339        for name in self.unet.attn_processors.keys():340            cross_attention_dim = None if name.endswith("attn1.processor") else self.unet.config.cross_attention_dim341            if name.startswith("mid_block"):342                hidden_size = self.unet.config.block_out_channels[-1]343            elif name.startswith("up_blocks"):344                block_id = int(name[len("up_blocks.")])345                hidden_size = list(reversed(self.unet.config.block_out_channels))[block_id]346            elif name.startswith("down_blocks"):347                block_id = int(name[len("down_blocks.")])348                hidden_size = self.unet.config.block_out_channels[block_id]349            if cross_attention_dim is None or "motion_modules" in name:350                attn_processor_class = (351                    AttnProcessor2_0 if hasattr(F, "scaled_dot_product_attention") else AttnProcessor352                )353                attn_procs[name] = attn_processor_class()354 355                lora_dict.update(356                    {f"unet.{name}.to_k_lora.down.weight": state_dict["ip_adapter"][f"{key_id}.to_k_lora.down.weight"]}357                )358                lora_dict.update(359                    {f"unet.{name}.to_q_lora.down.weight": state_dict["ip_adapter"][f"{key_id}.to_q_lora.down.weight"]}360                )361                lora_dict.update(362                    {f"unet.{name}.to_v_lora.down.weight": state_dict["ip_adapter"][f"{key_id}.to_v_lora.down.weight"]}363                )364                lora_dict.update(365                    {366                        f"unet.{name}.to_out_lora.down.weight": state_dict["ip_adapter"][367                            f"{key_id}.to_out_lora.down.weight"368                        ]369                    }370                )371                lora_dict.update(372                    {f"unet.{name}.to_k_lora.up.weight": state_dict["ip_adapter"][f"{key_id}.to_k_lora.up.weight"]}373                )374                lora_dict.update(375                    {f"unet.{name}.to_q_lora.up.weight": state_dict["ip_adapter"][f"{key_id}.to_q_lora.up.weight"]}376                )377                lora_dict.update(378                    {f"unet.{name}.to_v_lora.up.weight": state_dict["ip_adapter"][f"{key_id}.to_v_lora.up.weight"]}379                )380                lora_dict.update(381                    {f"unet.{name}.to_out_lora.up.weight": state_dict["ip_adapter"][f"{key_id}.to_out_lora.up.weight"]}382                )383                key_id += 1384            else:385                attn_processor_class = (386                    IPAdapterAttnProcessor2_0 if hasattr(F, "scaled_dot_product_attention") else IPAdapterAttnProcessor387                )388                attn_procs[name] = attn_processor_class(389                    hidden_size=hidden_size,390                    cross_attention_dim=cross_attention_dim,391                    scale=1.0,392                    num_tokens=num_image_text_embeds,393                ).to(dtype=self.dtype, device=self.device)394 395                lora_dict.update(396                    {f"unet.{name}.to_k_lora.down.weight": state_dict["ip_adapter"][f"{key_id}.to_k_lora.down.weight"]}397                )398                lora_dict.update(399                    {f"unet.{name}.to_q_lora.down.weight": state_dict["ip_adapter"][f"{key_id}.to_q_lora.down.weight"]}400                )401                lora_dict.update(402                    {f"unet.{name}.to_v_lora.down.weight": state_dict["ip_adapter"][f"{key_id}.to_v_lora.down.weight"]}403                )404                lora_dict.update(405                    {406                        f"unet.{name}.to_out_lora.down.weight": state_dict["ip_adapter"][407                            f"{key_id}.to_out_lora.down.weight"408                        ]409                    }410                )411                lora_dict.update(412                    {f"unet.{name}.to_k_lora.up.weight": state_dict["ip_adapter"][f"{key_id}.to_k_lora.up.weight"]}413                )414                lora_dict.update(415                    {f"unet.{name}.to_q_lora.up.weight": state_dict["ip_adapter"][f"{key_id}.to_q_lora.up.weight"]}416                )417                lora_dict.update(418                    {f"unet.{name}.to_v_lora.up.weight": state_dict["ip_adapter"][f"{key_id}.to_v_lora.up.weight"]}419                )420                lora_dict.update(421                    {f"unet.{name}.to_out_lora.up.weight": state_dict["ip_adapter"][f"{key_id}.to_out_lora.up.weight"]}422                )423 424                value_dict = {}425                value_dict.update({"to_k_ip.0.weight": state_dict["ip_adapter"][f"{key_id}.to_k_ip.weight"]})426                value_dict.update({"to_v_ip.0.weight": state_dict["ip_adapter"][f"{key_id}.to_v_ip.weight"]})427                attn_procs[name].load_state_dict(value_dict)428                key_id += 1429 430        self.unet.set_attn_processor(attn_procs)431 432        self.load_lora_weights(lora_dict, adapter_name="faceid")433        self.set_adapters(["faceid"], adapter_weights=[1.0])434 435        # convert IP-Adapter Image Projection layers to diffusers436        image_projection = self.convert_ip_adapter_image_proj_to_diffusers(state_dict["image_proj"])437        image_projection_layers = [image_projection.to(device=self.device, dtype=self.dtype)]438 439        self.unet.encoder_hid_proj = MultiIPAdapterImageProjection(image_projection_layers)440        self.unet.config.encoder_hid_dim_type = "ip_image_proj"441 442    def set_ip_adapter_scale(self, scale):443        unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet444        for attn_processor in unet.attn_processors.values():445            if isinstance(attn_processor, (IPAdapterAttnProcessor, IPAdapterAttnProcessor2_0)):446                attn_processor.scale = [scale]447 448    def _encode_prompt(449        self,450        prompt,451        device,452        num_images_per_prompt,453        do_classifier_free_guidance,454        negative_prompt=None,455        prompt_embeds: Optional[torch.Tensor] = None,456        negative_prompt_embeds: Optional[torch.Tensor] = None,457        lora_scale: Optional[float] = None,458        **kwargs,459    ):460        deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple."461        deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False)462 463        prompt_embeds_tuple = self.encode_prompt(464            prompt=prompt,465            device=device,466            num_images_per_prompt=num_images_per_prompt,467            do_classifier_free_guidance=do_classifier_free_guidance,468            negative_prompt=negative_prompt,469            prompt_embeds=prompt_embeds,470            negative_prompt_embeds=negative_prompt_embeds,471            lora_scale=lora_scale,472            **kwargs,473        )474 475        # concatenate for backwards comp476        prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]])477 478        return prompt_embeds479 480    def encode_prompt(481        self,482        prompt,483        device,484        num_images_per_prompt,485        do_classifier_free_guidance,486        negative_prompt=None,487        prompt_embeds: Optional[torch.Tensor] = None,488        negative_prompt_embeds: Optional[torch.Tensor] = None,489        lora_scale: Optional[float] = None,490        clip_skip: Optional[int] = None,491    ):492        r"""493        Encodes the prompt into text encoder hidden states.494 495        Args:496            prompt (`str` or `List[str]`, *optional*):497                prompt to be encoded498            device: (`torch.device`):499                torch device500            num_images_per_prompt (`int`):501                number of images that should be generated per prompt502            do_classifier_free_guidance (`bool`):503                whether to use classifier free guidance or not504            negative_prompt (`str` or `List[str]`, *optional*):505                The prompt or prompts not to guide the image generation. If not defined, one has to pass506                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is507                less than `1`).508            prompt_embeds (`torch.Tensor`, *optional*):509                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not510                provided, text embeddings will be generated from `prompt` input argument.511            negative_prompt_embeds (`torch.Tensor`, *optional*):512                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt513                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input514                argument.515            lora_scale (`float`, *optional*):516                A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.517            clip_skip (`int`, *optional*):518                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that519                the output of the pre-final layer will be used for computing the prompt embeddings.520        """521        # set lora scale so that monkey patched LoRA522        # function of text encoder can correctly access it523        if lora_scale is not None and isinstance(self, LoraLoaderMixin):524            self._lora_scale = lora_scale525 526            # dynamically adjust the LoRA scale527            if not USE_PEFT_BACKEND:528                adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)529            else:530                scale_lora_layers(self.text_encoder, lora_scale)531 532        if prompt is not None and isinstance(prompt, str):533            batch_size = 1534        elif prompt is not None and isinstance(prompt, list):535            batch_size = len(prompt)536        else:537            batch_size = prompt_embeds.shape[0]538 539        if prompt_embeds is None:540            # textual inversion: process multi-vector tokens if necessary541            if isinstance(self, TextualInversionLoaderMixin):542                prompt = self.maybe_convert_prompt(prompt, self.tokenizer)543 544            text_inputs = self.tokenizer(545                prompt,546                padding="max_length",547                max_length=self.tokenizer.model_max_length,548                truncation=True,549                return_tensors="pt",550            )551            text_input_ids = text_inputs.input_ids552            untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids553 554            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(555                text_input_ids, untruncated_ids556            ):557                removed_text = self.tokenizer.batch_decode(558                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]559                )560                logger.warning(561                    "The following part of your input was truncated because CLIP can only handle sequences up to"562                    f" {self.tokenizer.model_max_length} tokens: {removed_text}"563                )564 565            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:566                attention_mask = text_inputs.attention_mask.to(device)567            else:568                attention_mask = None569 570            if clip_skip is None:571                prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask)572                prompt_embeds = prompt_embeds[0]573            else:574                prompt_embeds = self.text_encoder(575                    text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True576                )577                # Access the `hidden_states` first, that contains a tuple of578                # all the hidden states from the encoder layers. Then index into579                # the tuple to access the hidden states from the desired layer.580                prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)]581                # We also need to apply the final LayerNorm here to not mess with the582                # representations. The `last_hidden_states` that we typically use for583                # obtaining the final prompt representations passes through the LayerNorm584                # layer.585                prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds)586 587        if self.text_encoder is not None:588            prompt_embeds_dtype = self.text_encoder.dtype589        elif self.unet is not None:590            prompt_embeds_dtype = self.unet.dtype591        else:592            prompt_embeds_dtype = prompt_embeds.dtype593 594        prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)595 596        bs_embed, seq_len, _ = prompt_embeds.shape597        # duplicate text embeddings for each generation per prompt, using mps friendly method598        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)599        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)600 601        # get unconditional embeddings for classifier free guidance602        if do_classifier_free_guidance and negative_prompt_embeds is None:603            uncond_tokens: List[str]604            if negative_prompt is None:605                uncond_tokens = [""] * batch_size606            elif prompt is not None and type(prompt) is not type(negative_prompt):607                raise TypeError(608                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="609                    f" {type(prompt)}."610                )611            elif isinstance(negative_prompt, str):612                uncond_tokens = [negative_prompt]613            elif batch_size != len(negative_prompt):614                raise ValueError(615                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"616                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"617                    " the batch size of `prompt`."618                )619            else:620                uncond_tokens = negative_prompt621 622            # textual inversion: process multi-vector tokens if necessary623            if isinstance(self, TextualInversionLoaderMixin):624                uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer)625 626            max_length = prompt_embeds.shape[1]627            uncond_input = self.tokenizer(628                uncond_tokens,629                padding="max_length",630                max_length=max_length,631                truncation=True,632                return_tensors="pt",633            )634 635            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:636                attention_mask = uncond_input.attention_mask.to(device)637            else:638                attention_mask = None639 640            negative_prompt_embeds = self.text_encoder(641                uncond_input.input_ids.to(device),642                attention_mask=attention_mask,643            )644            negative_prompt_embeds = negative_prompt_embeds[0]645 646        if do_classifier_free_guidance:647            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method648            seq_len = negative_prompt_embeds.shape[1]649 650            negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)651 652            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)653            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)654 655        if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND:656            # Retrieve the original scale by scaling back the LoRA layers657            unscale_lora_layers(self.text_encoder, lora_scale)658 659        return prompt_embeds, negative_prompt_embeds660 661    def run_safety_checker(self, image, device, dtype):662        if self.safety_checker is None:663            has_nsfw_concept = None664        else:665            if torch.is_tensor(image):666                feature_extractor_input = self.image_processor.postprocess(image, output_type="pil")667            else:668                feature_extractor_input = self.image_processor.numpy_to_pil(image)669            safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device)670            image, has_nsfw_concept = self.safety_checker(671                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)672            )673        return image, has_nsfw_concept674 675    def decode_latents(self, latents):676        deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead"677        deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False)678 679        latents = 1 / self.vae.config.scaling_factor * latents680        image = self.vae.decode(latents, return_dict=False)[0]681        image = (image / 2 + 0.5).clamp(0, 1)682        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16683        image = image.cpu().permute(0, 2, 3, 1).float().numpy()684        return image685 686    def prepare_extra_step_kwargs(self, generator, eta):687        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature688        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.689        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502690        # and should be between [0, 1]691 692        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())693        extra_step_kwargs = {}694        if accepts_eta:695            extra_step_kwargs["eta"] = eta696 697        # check if the scheduler accepts generator698        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())699        if accepts_generator:700            extra_step_kwargs["generator"] = generator701        return extra_step_kwargs702 703    def check_inputs(704        self,705        prompt,706        height,707        width,708        callback_steps,709        negative_prompt=None,710        prompt_embeds=None,711        negative_prompt_embeds=None,712        callback_on_step_end_tensor_inputs=None,713    ):714        if height % 8 != 0 or width % 8 != 0:715            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")716 717        if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):718            raise ValueError(719                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"720                f" {type(callback_steps)}."721            )722        if callback_on_step_end_tensor_inputs is not None and not all(723            k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs724        ):725            raise ValueError(726                f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"727            )728 729        if prompt is not None and prompt_embeds is not None:730            raise ValueError(731                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"732                " only forward one of the two."733            )734        elif prompt is None and prompt_embeds is None:735            raise ValueError(736                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."737            )738        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):739            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")740 741        if negative_prompt is not None and negative_prompt_embeds is not None:742            raise ValueError(743                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"744                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."745            )746 747        if prompt_embeds is not None and negative_prompt_embeds is not None:748            if prompt_embeds.shape != negative_prompt_embeds.shape:749                raise ValueError(750                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"751                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"752                    f" {negative_prompt_embeds.shape}."753                )754 755    def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):756        shape = (757            batch_size,758            num_channels_latents,759            int(height) // self.vae_scale_factor,760            int(width) // self.vae_scale_factor,761        )762        if isinstance(generator, list) and len(generator) != batch_size:763            raise ValueError(764                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"765                f" size of {batch_size}. Make sure the batch size matches the length of the generators."766            )767 768        if latents is None:769            latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)770        else:771            latents = latents.to(device)772 773        # scale the initial noise by the standard deviation required by the scheduler774        latents = latents * self.scheduler.init_noise_sigma775        return latents776 777    # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding778    def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32):779        """780        See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298781 782        Args:783            timesteps (`torch.Tensor`):784                generate embedding vectors at these timesteps785            embedding_dim (`int`, *optional*, defaults to 512):786                dimension of the embeddings to generate787            dtype:788                data type of the generated embeddings789 790        Returns:791            `torch.Tensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)`792        """793        assert len(w.shape) == 1794        w = w * 1000.0795 796        half_dim = embedding_dim // 2797        emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)798        emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)799        emb = w.to(dtype)[:, None] * emb[None, :]800        emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)801        if embedding_dim % 2 == 1:  # zero pad802            emb = torch.nn.functional.pad(emb, (0, 1))803        assert emb.shape == (w.shape[0], embedding_dim)804        return emb805 806    @property807    def guidance_scale(self):808        return self._guidance_scale809 810    @property811    def guidance_rescale(self):812        return self._guidance_rescale813 814    @property815    def clip_skip(self):816        return self._clip_skip817 818    # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)819    # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`820    # corresponds to doing no classifier free guidance.821    @property822    def do_classifier_free_guidance(self):823        return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None824 825    @property826    def cross_attention_kwargs(self):827        return self._cross_attention_kwargs828 829    @property830    def num_timesteps(self):831        return self._num_timesteps832 833    @property834    def interrupt(self):835        return self._interrupt836 837    @torch.no_grad()838    def __call__(839        self,840        prompt: Union[str, List[str]] = None,841        height: Optional[int] = None,842        width: Optional[int] = None,843        num_inference_steps: int = 50,844        timesteps: List[int] = None,845        guidance_scale: float = 7.5,846        negative_prompt: Optional[Union[str, List[str]]] = None,847        num_images_per_prompt: Optional[int] = 1,848        eta: float = 0.0,849        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,850        latents: Optional[torch.Tensor] = None,851        prompt_embeds: Optional[torch.Tensor] = None,852        negative_prompt_embeds: Optional[torch.Tensor] = None,853        image_embeds: Optional[torch.Tensor] = None,854        output_type: Optional[str] = "pil",855        return_dict: bool = True,856        cross_attention_kwargs: Optional[Dict[str, Any]] = None,857        guidance_rescale: float = 0.0,858        clip_skip: Optional[int] = None,859        callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,860        callback_on_step_end_tensor_inputs: List[str] = ["latents"],861        **kwargs,862    ):863        r"""864        The call function to the pipeline for generation.865 866        Args:867            prompt (`str` or `List[str]`, *optional*):868                The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.869            height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):870                The height in pixels of the generated image.871            width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):872                The width in pixels of the generated image.873            num_inference_steps (`int`, *optional*, defaults to 50):874                The number of denoising steps. More denoising steps usually lead to a higher quality image at the875                expense of slower inference.876            timesteps (`List[int]`, *optional*):877                Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument878                in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is879                passed will be used. Must be in descending order.880            guidance_scale (`float`, *optional*, defaults to 7.5):881                A higher guidance scale value encourages the model to generate images closely linked to the text882                `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.883            negative_prompt (`str` or `List[str]`, *optional*):884                The prompt or prompts to guide what to not include in image generation. If not defined, you need to885                pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).886            num_images_per_prompt (`int`, *optional*, defaults to 1):887                The number of images to generate per prompt.888            eta (`float`, *optional*, defaults to 0.0):889                Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies890                to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.891            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):892                A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make893                generation deterministic.894            latents (`torch.Tensor`, *optional*):895                Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image896                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents897                tensor is generated by sampling using the supplied random `generator`.898            prompt_embeds (`torch.Tensor`, *optional*):899                Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not900                provided, text embeddings are generated from the `prompt` input argument.901            negative_prompt_embeds (`torch.Tensor`, *optional*):902                Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If903                not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.904            image_embeds (`torch.Tensor`, *optional*):905                Pre-generated image embeddings.906            output_type (`str`, *optional*, defaults to `"pil"`):907                The output format of the generated image. Choose between `PIL.Image` or `np.array`.908            return_dict (`bool`, *optional*, defaults to `True`):909                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a910                plain tuple.911            cross_attention_kwargs (`dict`, *optional*):912                A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in913                [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).914            guidance_rescale (`float`, *optional*, defaults to 0.0):915                Guidance rescale factor from [Common Diffusion Noise Schedules and Sample Steps are916                Flawed](https://arxiv.org/pdf/2305.08891.pdf). Guidance rescale factor should fix overexposure when917                using zero terminal SNR.918            clip_skip (`int`, *optional*):919                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that920                the output of the pre-final layer will be used for computing the prompt embeddings.921            callback_on_step_end (`Callable`, *optional*):922                A function that calls at the end of each denoising steps during the inference. The function is called923                with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,924                callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by925                `callback_on_step_end_tensor_inputs`.926            callback_on_step_end_tensor_inputs (`List`, *optional*):927                The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list928                will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the929                `._callback_tensor_inputs` attribute of your pipeline class.930 931        Examples:932 933        Returns:934            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:935                If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned,936                otherwise a `tuple` is returned where the first element is a list with the generated images and the937                second element is a list of `bool`s indicating whether the corresponding generated image contains938                "not-safe-for-work" (nsfw) content.939        """940 941        callback = kwargs.pop("callback", None)942        callback_steps = kwargs.pop("callback_steps", None)943 944        if callback is not None:945            deprecate(946                "callback",947                "1.0.0",948                "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",949            )950        if callback_steps is not None:951            deprecate(952                "callback_steps",953                "1.0.0",954                "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",955            )956 957        # 0. Default height and width to unet958        height = height or self.unet.config.sample_size * self.vae_scale_factor959        width = width or self.unet.config.sample_size * self.vae_scale_factor960        # to deal with lora scaling and other possible forward hooks961 962        # 1. Check inputs. Raise error if not correct963        self.check_inputs(964            prompt,965            height,966            width,967            callback_steps,968            negative_prompt,969            prompt_embeds,970            negative_prompt_embeds,971            callback_on_step_end_tensor_inputs,972        )973 974        self._guidance_scale = guidance_scale975        self._guidance_rescale = guidance_rescale976        self._clip_skip = clip_skip977        self._cross_attention_kwargs = cross_attention_kwargs978        self._interrupt = False979 980        # 2. Define call parameters981        if prompt is not None and isinstance(prompt, str):982            batch_size = 1983        elif prompt is not None and isinstance(prompt, list):984            batch_size = len(prompt)985        else:986            batch_size = prompt_embeds.shape[0]987 988        device = self._execution_device989 990        # 3. Encode input prompt991        lora_scale = (992            self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None993        )994 995        prompt_embeds, negative_prompt_embeds = self.encode_prompt(996            prompt,997            device,998            num_images_per_prompt,999            self.do_classifier_free_guidance,1000            negative_prompt,1001            prompt_embeds=prompt_embeds,1002            negative_prompt_embeds=negative_prompt_embeds,1003            lora_scale=lora_scale,1004            clip_skip=self.clip_skip,1005        )1006 1007        # For classifier free guidance, we need to do two forward passes.1008        # Here we concatenate the unconditional and text embeddings into a single batch1009        # to avoid doing two forward passes1010        if self.do_classifier_free_guidance:1011            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])1012 1013        if image_embeds is not None:1014            image_embeds = torch.stack([image_embeds] * num_images_per_prompt, dim=0).to(1015                device=device, dtype=prompt_embeds.dtype1016            )1017            negative_image_embeds = torch.zeros_like(image_embeds)1018            if self.do_classifier_free_guidance:1019                image_embeds = torch.cat([negative_image_embeds, image_embeds])1020        image_embeds = [image_embeds]1021        # 4. Prepare timesteps1022        timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps)1023 1024        # 5. Prepare latent variables1025        num_channels_latents = self.unet.config.in_channels1026        latents = self.prepare_latents(1027            batch_size * num_images_per_prompt,1028            num_channels_latents,1029            height,1030            width,1031            prompt_embeds.dtype,1032            device,1033            generator,1034            latents,1035        )1036 1037        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline1038        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)1039 1040        # 6.1 Add image embeds for IP-Adapter1041        added_cond_kwargs = {"image_embeds": image_embeds} if image_embeds is not None else {}1042 1043        # 6.2 Optionally get Guidance Scale Embedding1044        timestep_cond = None1045        if self.unet.config.time_cond_proj_dim is not None:1046            guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)1047            timestep_cond = self.get_guidance_scale_embedding(1048                guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim1049            ).to(device=device, dtype=latents.dtype)1050 1051        # 7. Denoising loop1052        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order1053        self._num_timesteps = len(timesteps)1054        with self.progress_bar(total=num_inference_steps) as progress_bar:1055            for i, t in enumerate(timesteps):1056                if self.interrupt:1057                    continue1058 1059                # expand the latents if we are doing classifier free guidance1060                latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents1061                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)1062 1063                # predict the noise residual1064                noise_pred = self.unet(1065                    latent_model_input,1066                    t,1067                    encoder_hidden_states=prompt_embeds,1068                    timestep_cond=timestep_cond,1069                    cross_attention_kwargs=self.cross_attention_kwargs,1070                    added_cond_kwargs=added_cond_kwargs,1071                    return_dict=False,1072                )[0]1073 1074                # perform guidance1075                if self.do_classifier_free_guidance:1076                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)1077                    noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)1078 1079                if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:1080                    # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf1081                    noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale)1082 1083                # compute the previous noisy sample x_t -> x_t-11084                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]1085 1086                if callback_on_step_end is not None:1087                    callback_kwargs = {}1088                    for k in callback_on_step_end_tensor_inputs:1089                        callback_kwargs[k] = locals()[k]1090                    callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)1091 1092                    latents = callback_outputs.pop("latents", latents)1093                    prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)1094                    negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)1095 1096                # call the callback, if provided1097                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):1098                    progress_bar.update()1099                    if callback is not None and i % callback_steps == 0:1100                        step_idx = i // getattr(self.scheduler, "order", 1)1101                        callback(step_idx, t, latents)1102 1103        if not output_type == "latent":1104            image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[1105                01106            ]1107            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)1108        else:1109            image = latents1110            has_nsfw_concept = None1111 1112        if has_nsfw_concept is None:1113            do_denormalize = [True] * image.shape[0]1114        else:1115            do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept]1116 1117        image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)1118 1119        # Offload all models1120        self.maybe_free_model_hooks()1121 1122        if not return_dict:1123            return (image, has_nsfw_concept)1124 1125        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)1126