diffusers/community-pipelines-mirror

1# Implementation of StableDiffusionPipeline with PAG2# https://ku-cvlab.github.io/Perturbed-Attention-Guidance3 4import inspect5from typing import Any, Callable, Dict, List, Optional, Union6 7import torch8import torch.nn.functional as F9from packaging import version10from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection11 12from diffusers.configuration_utils import FrozenDict13from diffusers.image_processor import PipelineImageInput, VaeImageProcessor14from diffusers.loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin15from diffusers.models import AutoencoderKL, ImageProjection, UNet2DConditionModel16from diffusers.models.attention_processor import Attention, AttnProcessor2_0, FusedAttnProcessor2_017from diffusers.models.lora import adjust_lora_scale_text_encoder18from diffusers.pipelines.pipeline_utils import DiffusionPipeline19from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput20from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker21from diffusers.schedulers import KarrasDiffusionSchedulers22from diffusers.utils import (23    USE_PEFT_BACKEND,24    deprecate,25    logging,26    replace_example_docstring,27    scale_lora_layers,28    unscale_lora_layers,29)30from diffusers.utils.torch_utils import randn_tensor31 32 33logger = logging.get_logger(__name__)  # pylint: disable=invalid-name34 35EXAMPLE_DOC_STRING = """36    Examples:37        ```py38        >>> import torch39        >>> from diffusers import StableDiffusionPipeline40        >>> pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16)41        >>> pipe = pipe.to("cuda")42        >>> prompt = "a photo of an astronaut riding a horse on mars"43        >>> image = pipe(prompt).images[0]44        ```45"""46 47 48class PAGIdentitySelfAttnProcessor:49    r"""50    Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).51    """52 53    def __init__(self):54        if not hasattr(F, "scaled_dot_product_attention"):55            raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")56 57    def __call__(58        self,59        attn: Attention,60        hidden_states: torch.Tensor,61        encoder_hidden_states: Optional[torch.Tensor] = None,62        attention_mask: Optional[torch.Tensor] = None,63        temb: Optional[torch.Tensor] = None,64        *args,65        **kwargs,66    ) -> torch.Tensor:67        if len(args) > 0 or kwargs.get("scale", None) is not None:68            deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`."69            deprecate("scale", "1.0.0", deprecation_message)70 71        residual = hidden_states72        if attn.spatial_norm is not None:73            hidden_states = attn.spatial_norm(hidden_states, temb)74 75        input_ndim = hidden_states.ndim76        if input_ndim == 4:77            batch_size, channel, height, width = hidden_states.shape78            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)79 80        # chunk81        hidden_states_org, hidden_states_ptb = hidden_states.chunk(2)82 83        # original path84        batch_size, sequence_length, _ = hidden_states_org.shape85 86        if attention_mask is not None:87            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)88            # scaled_dot_product_attention expects attention_mask shape to be89            # (batch, heads, source_length, target_length)90            attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])91 92        if attn.group_norm is not None:93            hidden_states_org = attn.group_norm(hidden_states_org.transpose(1, 2)).transpose(1, 2)94 95        query = attn.to_q(hidden_states_org)96        key = attn.to_k(hidden_states_org)97        value = attn.to_v(hidden_states_org)98 99        inner_dim = key.shape[-1]100        head_dim = inner_dim // attn.heads101 102        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)103 104        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)105        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)106 107        # the output of sdp = (batch, num_heads, seq_len, head_dim)108        # TODO: add support for attn.scale when we move to Torch 2.1109        hidden_states_org = F.scaled_dot_product_attention(110            query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False111        )112 113        hidden_states_org = hidden_states_org.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)114        hidden_states_org = hidden_states_org.to(query.dtype)115 116        # linear proj117        hidden_states_org = attn.to_out[0](hidden_states_org)118        # dropout119        hidden_states_org = attn.to_out[1](hidden_states_org)120 121        if input_ndim == 4:122            hidden_states_org = hidden_states_org.transpose(-1, -2).reshape(batch_size, channel, height, width)123 124        # perturbed path (identity attention)125        batch_size, sequence_length, _ = hidden_states_ptb.shape126 127        if attention_mask is not None:128            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)129            # scaled_dot_product_attention expects attention_mask shape to be130            # (batch, heads, source_length, target_length)131            attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])132 133        if attn.group_norm is not None:134            hidden_states_ptb = attn.group_norm(hidden_states_ptb.transpose(1, 2)).transpose(1, 2)135 136        value = attn.to_v(hidden_states_ptb)137 138        # hidden_states_ptb = torch.zeros(value.shape).to(value.get_device())139        hidden_states_ptb = value140 141        hidden_states_ptb = hidden_states_ptb.to(query.dtype)142 143        # linear proj144        hidden_states_ptb = attn.to_out[0](hidden_states_ptb)145        # dropout146        hidden_states_ptb = attn.to_out[1](hidden_states_ptb)147 148        if input_ndim == 4:149            hidden_states_ptb = hidden_states_ptb.transpose(-1, -2).reshape(batch_size, channel, height, width)150 151        # cat152        hidden_states = torch.cat([hidden_states_org, hidden_states_ptb])153 154        if attn.residual_connection:155            hidden_states = hidden_states + residual156 157        hidden_states = hidden_states / attn.rescale_output_factor158 159        return hidden_states160 161 162class PAGCFGIdentitySelfAttnProcessor:163    r"""164    Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).165    """166 167    def __init__(self):168        if not hasattr(F, "scaled_dot_product_attention"):169            raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")170 171    def __call__(172        self,173        attn: Attention,174        hidden_states: torch.Tensor,175        encoder_hidden_states: Optional[torch.Tensor] = None,176        attention_mask: Optional[torch.Tensor] = None,177        temb: Optional[torch.Tensor] = None,178        *args,179        **kwargs,180    ) -> torch.Tensor:181        if len(args) > 0 or kwargs.get("scale", None) is not None:182            deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`."183            deprecate("scale", "1.0.0", deprecation_message)184 185        residual = hidden_states186        if attn.spatial_norm is not None:187            hidden_states = attn.spatial_norm(hidden_states, temb)188 189        input_ndim = hidden_states.ndim190        if input_ndim == 4:191            batch_size, channel, height, width = hidden_states.shape192            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)193 194        # chunk195        hidden_states_uncond, hidden_states_org, hidden_states_ptb = hidden_states.chunk(3)196        hidden_states_org = torch.cat([hidden_states_uncond, hidden_states_org])197 198        # original path199        batch_size, sequence_length, _ = hidden_states_org.shape200 201        if attention_mask is not None:202            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)203            # scaled_dot_product_attention expects attention_mask shape to be204            # (batch, heads, source_length, target_length)205            attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])206 207        if attn.group_norm is not None:208            hidden_states_org = attn.group_norm(hidden_states_org.transpose(1, 2)).transpose(1, 2)209 210        query = attn.to_q(hidden_states_org)211        key = attn.to_k(hidden_states_org)212        value = attn.to_v(hidden_states_org)213 214        inner_dim = key.shape[-1]215        head_dim = inner_dim // attn.heads216 217        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)218 219        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)220        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)221 222        # the output of sdp = (batch, num_heads, seq_len, head_dim)223        # TODO: add support for attn.scale when we move to Torch 2.1224        hidden_states_org = F.scaled_dot_product_attention(225            query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False226        )227 228        hidden_states_org = hidden_states_org.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)229        hidden_states_org = hidden_states_org.to(query.dtype)230 231        # linear proj232        hidden_states_org = attn.to_out[0](hidden_states_org)233        # dropout234        hidden_states_org = attn.to_out[1](hidden_states_org)235 236        if input_ndim == 4:237            hidden_states_org = hidden_states_org.transpose(-1, -2).reshape(batch_size, channel, height, width)238 239        # perturbed path (identity attention)240        batch_size, sequence_length, _ = hidden_states_ptb.shape241 242        if attention_mask is not None:243            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)244            # scaled_dot_product_attention expects attention_mask shape to be245            # (batch, heads, source_length, target_length)246            attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])247 248        if attn.group_norm is not None:249            hidden_states_ptb = attn.group_norm(hidden_states_ptb.transpose(1, 2)).transpose(1, 2)250 251        value = attn.to_v(hidden_states_ptb)252        hidden_states_ptb = value253        hidden_states_ptb = hidden_states_ptb.to(query.dtype)254 255        # linear proj256        hidden_states_ptb = attn.to_out[0](hidden_states_ptb)257        # dropout258        hidden_states_ptb = attn.to_out[1](hidden_states_ptb)259 260        if input_ndim == 4:261            hidden_states_ptb = hidden_states_ptb.transpose(-1, -2).reshape(batch_size, channel, height, width)262 263        # cat264        hidden_states = torch.cat([hidden_states_org, hidden_states_ptb])265 266        if attn.residual_connection:267            hidden_states = hidden_states + residual268 269        hidden_states = hidden_states / attn.rescale_output_factor270 271        return hidden_states272 273 274def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):275    """276    Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and277    Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4278    """279    std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)280    std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)281    # rescale the results from guidance (fixes overexposure)282    noise_pred_rescaled = noise_cfg * (std_text / std_cfg)283    # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images284    noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg285    return noise_cfg286 287 288def retrieve_timesteps(289    scheduler,290    num_inference_steps: Optional[int] = None,291    device: Optional[Union[str, torch.device]] = None,292    timesteps: Optional[List[int]] = None,293    **kwargs,294):295    """296    Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles297    custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.298    Args:299        scheduler (`SchedulerMixin`):300            The scheduler to get timesteps from.301        num_inference_steps (`int`):302            The number of diffusion steps used when generating samples with a pre-trained model. If used,303            `timesteps` must be `None`.304        device (`str` or `torch.device`, *optional*):305            The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.306        timesteps (`List[int]`, *optional*):307                Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default308                timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps`309                must be `None`.310    Returns:311        `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the312        second element is the number of inference steps.313    """314    if timesteps is not None:315        accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())316        if not accepts_timesteps:317            raise ValueError(318                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"319                f" timestep schedules. Please check whether you are using the correct scheduler."320            )321        scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)322        timesteps = scheduler.timesteps323        num_inference_steps = len(timesteps)324    else:325        scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)326        timesteps = scheduler.timesteps327    return timesteps, num_inference_steps328 329 330class StableDiffusionPAGPipeline(331    DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin, IPAdapterMixin, FromSingleFileMixin332):333    r"""334    Pipeline for text-to-image generation using Stable Diffusion.335    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods336    implemented for all pipelines (downloading, saving, running on a particular device, etc.).337    The pipeline also inherits the following loading methods:338        - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings339        - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights340        - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights341        - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files342        - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters343    Args:344        vae ([`AutoencoderKL`]):345            Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.346        text_encoder ([`~transformers.CLIPTextModel`]):347            Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).348        tokenizer ([`~transformers.CLIPTokenizer`]):349            A `CLIPTokenizer` to tokenize text.350        unet ([`UNet2DConditionModel`]):351            A `UNet2DConditionModel` to denoise the encoded image latents.352        scheduler ([`SchedulerMixin`]):353            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of354            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].355        safety_checker ([`StableDiffusionSafetyChecker`]):356            Classification module that estimates whether generated images could be considered offensive or harmful.357            Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details358            about a model's potential harms.359        feature_extractor ([`~transformers.CLIPImageProcessor`]):360            A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`.361    """362 363    model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae"364    _optional_components = ["safety_checker", "feature_extractor", "image_encoder"]365    _exclude_from_cpu_offload = ["safety_checker"]366    _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]367 368    def __init__(369        self,370        vae: AutoencoderKL,371        text_encoder: CLIPTextModel,372        tokenizer: CLIPTokenizer,373        unet: UNet2DConditionModel,374        scheduler: KarrasDiffusionSchedulers,375        safety_checker: StableDiffusionSafetyChecker,376        feature_extractor: CLIPImageProcessor,377        image_encoder: CLIPVisionModelWithProjection = None,378        requires_safety_checker: bool = True,379    ):380        super().__init__()381 382        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:383            deprecation_message = (384                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"385                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "386                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"387                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"388                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"389                " file"390            )391            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)392            new_config = dict(scheduler.config)393            new_config["steps_offset"] = 1394            scheduler._internal_dict = FrozenDict(new_config)395 396        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:397            deprecation_message = (398                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."399                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"400                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"401                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"402                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"403            )404            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)405            new_config = dict(scheduler.config)406            new_config["clip_sample"] = False407            scheduler._internal_dict = FrozenDict(new_config)408 409        if safety_checker is None and requires_safety_checker:410            logger.warning(411                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"412                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"413                " results in services or applications open to the public. Both the diffusers team and Hugging Face"414                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"415                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"416                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."417            )418 419        if safety_checker is not None and feature_extractor is None:420            raise ValueError(421                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"422                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."423            )424 425        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(426            version.parse(unet.config._diffusers_version).base_version427        ) < version.parse("0.9.0.dev0")428        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64429        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:430            deprecation_message = (431                "The configuration file of the unet has set the default `sample_size` to smaller than"432                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"433                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"434                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"435                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"436                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"437                " in the config might lead to incorrect results in future versions. If you have downloaded this"438                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"439                " the `unet/config.json` file"440            )441            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)442            new_config = dict(unet.config)443            new_config["sample_size"] = 64444            unet._internal_dict = FrozenDict(new_config)445 446        self.register_modules(447            vae=vae,448            text_encoder=text_encoder,449            tokenizer=tokenizer,450            unet=unet,451            scheduler=scheduler,452            safety_checker=safety_checker,453            feature_extractor=feature_extractor,454            image_encoder=image_encoder,455        )456        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)457        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)458        self.register_to_config(requires_safety_checker=requires_safety_checker)459 460    def enable_vae_slicing(self):461        r"""462        Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to463        compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.464        """465        self.vae.enable_slicing()466 467    def disable_vae_slicing(self):468        r"""469        Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to470        computing decoding in one step.471        """472        self.vae.disable_slicing()473 474    def enable_vae_tiling(self):475        r"""476        Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to477        compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow478        processing larger images.479        """480        self.vae.enable_tiling()481 482    def disable_vae_tiling(self):483        r"""484        Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to485        computing decoding in one step.486        """487        self.vae.disable_tiling()488 489    def _encode_prompt(490        self,491        prompt,492        device,493        num_images_per_prompt,494        do_classifier_free_guidance,495        negative_prompt=None,496        prompt_embeds: Optional[torch.Tensor] = None,497        negative_prompt_embeds: Optional[torch.Tensor] = None,498        lora_scale: Optional[float] = None,499        **kwargs,500    ):501        deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple."502        deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False)503 504        prompt_embeds_tuple = self.encode_prompt(505            prompt=prompt,506            device=device,507            num_images_per_prompt=num_images_per_prompt,508            do_classifier_free_guidance=do_classifier_free_guidance,509            negative_prompt=negative_prompt,510            prompt_embeds=prompt_embeds,511            negative_prompt_embeds=negative_prompt_embeds,512            lora_scale=lora_scale,513            **kwargs,514        )515 516        # concatenate for backwards comp517        prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]])518 519        return prompt_embeds520 521    def encode_prompt(522        self,523        prompt,524        device,525        num_images_per_prompt,526        do_classifier_free_guidance,527        negative_prompt=None,528        prompt_embeds: Optional[torch.Tensor] = None,529        negative_prompt_embeds: Optional[torch.Tensor] = None,530        lora_scale: Optional[float] = None,531        clip_skip: Optional[int] = None,532    ):533        r"""534        Encodes the prompt into text encoder hidden states.535        Args:536            prompt (`str` or `List[str]`, *optional*):537                prompt to be encoded538            device: (`torch.device`):539                torch device540            num_images_per_prompt (`int`):541                number of images that should be generated per prompt542            do_classifier_free_guidance (`bool`):543                whether to use classifier free guidance or not544            negative_prompt (`str` or `List[str]`, *optional*):545                The prompt or prompts not to guide the image generation. If not defined, one has to pass546                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is547                less than `1`).548            prompt_embeds (`torch.Tensor`, *optional*):549                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not550                provided, text embeddings will be generated from `prompt` input argument.551            negative_prompt_embeds (`torch.Tensor`, *optional*):552                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt553                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input554                argument.555            lora_scale (`float`, *optional*):556                A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.557            clip_skip (`int`, *optional*):558                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that559                the output of the pre-final layer will be used for computing the prompt embeddings.560        """561        # set lora scale so that monkey patched LoRA562        # function of text encoder can correctly access it563        if lora_scale is not None and isinstance(self, LoraLoaderMixin):564            self._lora_scale = lora_scale565 566            # dynamically adjust the LoRA scale567            if not USE_PEFT_BACKEND:568                adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)569            else:570                scale_lora_layers(self.text_encoder, lora_scale)571 572        if prompt is not None and isinstance(prompt, str):573            batch_size = 1574        elif prompt is not None and isinstance(prompt, list):575            batch_size = len(prompt)576        else:577            batch_size = prompt_embeds.shape[0]578 579        if prompt_embeds is None:580            # textual inversion: process multi-vector tokens if necessary581            if isinstance(self, TextualInversionLoaderMixin):582                prompt = self.maybe_convert_prompt(prompt, self.tokenizer)583 584            text_inputs = self.tokenizer(585                prompt,586                padding="max_length",587                max_length=self.tokenizer.model_max_length,588                truncation=True,589                return_tensors="pt",590            )591            text_input_ids = text_inputs.input_ids592            untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids593 594            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(595                text_input_ids, untruncated_ids596            ):597                removed_text = self.tokenizer.batch_decode(598                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]599                )600                logger.warning(601                    "The following part of your input was truncated because CLIP can only handle sequences up to"602                    f" {self.tokenizer.model_max_length} tokens: {removed_text}"603                )604 605            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:606                attention_mask = text_inputs.attention_mask.to(device)607            else:608                attention_mask = None609 610            if clip_skip is None:611                prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask)612                prompt_embeds = prompt_embeds[0]613            else:614                prompt_embeds = self.text_encoder(615                    text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True616                )617                # Access the `hidden_states` first, that contains a tuple of618                # all the hidden states from the encoder layers. Then index into619                # the tuple to access the hidden states from the desired layer.620                prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)]621                # We also need to apply the final LayerNorm here to not mess with the622                # representations. The `last_hidden_states` that we typically use for623                # obtaining the final prompt representations passes through the LayerNorm624                # layer.625                prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds)626 627        if self.text_encoder is not None:628            prompt_embeds_dtype = self.text_encoder.dtype629        elif self.unet is not None:630            prompt_embeds_dtype = self.unet.dtype631        else:632            prompt_embeds_dtype = prompt_embeds.dtype633 634        prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)635 636        bs_embed, seq_len, _ = prompt_embeds.shape637        # duplicate text embeddings for each generation per prompt, using mps friendly method638        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)639        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)640 641        # get unconditional embeddings for classifier free guidance642        if do_classifier_free_guidance and negative_prompt_embeds is None:643            uncond_tokens: List[str]644            if negative_prompt is None:645                uncond_tokens = [""] * batch_size646            elif prompt is not None and type(prompt) is not type(negative_prompt):647                raise TypeError(648                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="649                    f" {type(prompt)}."650                )651            elif isinstance(negative_prompt, str):652                uncond_tokens = [negative_prompt]653            elif batch_size != len(negative_prompt):654                raise ValueError(655                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"656                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"657                    " the batch size of `prompt`."658                )659            else:660                uncond_tokens = negative_prompt661 662            # textual inversion: process multi-vector tokens if necessary663            if isinstance(self, TextualInversionLoaderMixin):664                uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer)665 666            max_length = prompt_embeds.shape[1]667            uncond_input = self.tokenizer(668                uncond_tokens,669                padding="max_length",670                max_length=max_length,671                truncation=True,672                return_tensors="pt",673            )674 675            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:676                attention_mask = uncond_input.attention_mask.to(device)677            else:678                attention_mask = None679 680            negative_prompt_embeds = self.text_encoder(681                uncond_input.input_ids.to(device),682                attention_mask=attention_mask,683            )684            negative_prompt_embeds = negative_prompt_embeds[0]685 686        if do_classifier_free_guidance:687            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method688            seq_len = negative_prompt_embeds.shape[1]689 690            negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)691 692            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)693            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)694 695        if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND:696            # Retrieve the original scale by scaling back the LoRA layers697            unscale_lora_layers(self.text_encoder, lora_scale)698 699        return prompt_embeds, negative_prompt_embeds700 701    def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):702        dtype = next(self.image_encoder.parameters()).dtype703 704        if not isinstance(image, torch.Tensor):705            image = self.feature_extractor(image, return_tensors="pt").pixel_values706 707        image = image.to(device=device, dtype=dtype)708        if output_hidden_states:709            image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]710            image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)711            uncond_image_enc_hidden_states = self.image_encoder(712                torch.zeros_like(image), output_hidden_states=True713            ).hidden_states[-2]714            uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(715                num_images_per_prompt, dim=0716            )717            return image_enc_hidden_states, uncond_image_enc_hidden_states718        else:719            image_embeds = self.image_encoder(image).image_embeds720            image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)721            uncond_image_embeds = torch.zeros_like(image_embeds)722 723            return image_embeds, uncond_image_embeds724 725    def prepare_ip_adapter_image_embeds(726        self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt727    ):728        if ip_adapter_image_embeds is None:729            if not isinstance(ip_adapter_image, list):730                ip_adapter_image = [ip_adapter_image]731 732            if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers):733                raise ValueError(734                    f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters."735                )736 737            image_embeds = []738            for single_ip_adapter_image, image_proj_layer in zip(739                ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers740            ):741                output_hidden_state = not isinstance(image_proj_layer, ImageProjection)742                single_image_embeds, single_negative_image_embeds = self.encode_image(743                    single_ip_adapter_image, device, 1, output_hidden_state744                )745                single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0)746                single_negative_image_embeds = torch.stack(747                    [single_negative_image_embeds] * num_images_per_prompt, dim=0748                )749 750                if self.do_classifier_free_guidance:751                    single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds])752                    single_image_embeds = single_image_embeds.to(device)753 754                image_embeds.append(single_image_embeds)755        else:756            image_embeds = ip_adapter_image_embeds757        return image_embeds758 759    def run_safety_checker(self, image, device, dtype):760        if self.safety_checker is None:761            has_nsfw_concept = None762        else:763            if torch.is_tensor(image):764                feature_extractor_input = self.image_processor.postprocess(image, output_type="pil")765            else:766                feature_extractor_input = self.image_processor.numpy_to_pil(image)767            safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device)768            image, has_nsfw_concept = self.safety_checker(769                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)770            )771        return image, has_nsfw_concept772 773    def decode_latents(self, latents):774        deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead"775        deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False)776 777        latents = 1 / self.vae.config.scaling_factor * latents778        image = self.vae.decode(latents, return_dict=False)[0]779        image = (image / 2 + 0.5).clamp(0, 1)780        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16781        image = image.cpu().permute(0, 2, 3, 1).float().numpy()782        return image783 784    def prepare_extra_step_kwargs(self, generator, eta):785        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature786        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.787        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502788        # and should be between [0, 1]789 790        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())791        extra_step_kwargs = {}792        if accepts_eta:793            extra_step_kwargs["eta"] = eta794 795        # check if the scheduler accepts generator796        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())797        if accepts_generator:798            extra_step_kwargs["generator"] = generator799        return extra_step_kwargs800 801    def check_inputs(802        self,803        prompt,804        height,805        width,806        callback_steps,807        negative_prompt=None,808        prompt_embeds=None,809        negative_prompt_embeds=None,810        ip_adapter_image=None,811        ip_adapter_image_embeds=None,812        callback_on_step_end_tensor_inputs=None,813    ):814        if height % 8 != 0 or width % 8 != 0:815            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")816 817        if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):818            raise ValueError(819                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"820                f" {type(callback_steps)}."821            )822        if callback_on_step_end_tensor_inputs is not None and not all(823            k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs824        ):825            raise ValueError(826                f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"827            )828 829        if prompt is not None and prompt_embeds is not None:830            raise ValueError(831                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"832                " only forward one of the two."833            )834        elif prompt is None and prompt_embeds is None:835            raise ValueError(836                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."837            )838        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):839            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")840 841        if negative_prompt is not None and negative_prompt_embeds is not None:842            raise ValueError(843                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"844                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."845            )846 847        if prompt_embeds is not None and negative_prompt_embeds is not None:848            if prompt_embeds.shape != negative_prompt_embeds.shape:849                raise ValueError(850                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"851                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"852                    f" {negative_prompt_embeds.shape}."853                )854 855        if ip_adapter_image is not None and ip_adapter_image_embeds is not None:856            raise ValueError(857                "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined."858            )859 860    def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):861        shape = (862            batch_size,863            num_channels_latents,864            int(height) // self.vae_scale_factor,865            int(width) // self.vae_scale_factor,866        )867        if isinstance(generator, list) and len(generator) != batch_size:868            raise ValueError(869                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"870                f" size of {batch_size}. Make sure the batch size matches the length of the generators."871            )872 873        if latents is None:874            latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)875        else:876            latents = latents.to(device)877 878        # scale the initial noise by the standard deviation required by the scheduler879        latents = latents * self.scheduler.init_noise_sigma880        return latents881 882    def enable_freeu(self, s1: float, s2: float, b1: float, b2: float):883        r"""Enables the FreeU mechanism as in https://arxiv.org/abs/2309.11497.884        The suffixes after the scaling factors represent the stages where they are being applied.885        Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of the values886        that are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL.887        Args:888            s1 (`float`):889                Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to890                mitigate "oversmoothing effect" in the enhanced denoising process.891            s2 (`float`):892                Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to893                mitigate "oversmoothing effect" in the enhanced denoising process.894            b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features.895            b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features.896        """897        if not hasattr(self, "unet"):898            raise ValueError("The pipeline must have `unet` for using FreeU.")899        self.unet.enable_freeu(s1=s1, s2=s2, b1=b1, b2=b2)900 901    def disable_freeu(self):902        """Disables the FreeU mechanism if enabled."""903        self.unet.disable_freeu()904 905    # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.fuse_qkv_projections906    def fuse_qkv_projections(self, unet: bool = True, vae: bool = True):907        """908        Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query,909        key, value) are fused. For cross-attention modules, key and value projection matrices are fused.910        <Tip warning={true}>911        This API is 🧪 experimental.912        </Tip>913        Args:914            unet (`bool`, defaults to `True`): To apply fusion on the UNet.915            vae (`bool`, defaults to `True`): To apply fusion on the VAE.916        """917        self.fusing_unet = False918        self.fusing_vae = False919 920        if unet:921            self.fusing_unet = True922            self.unet.fuse_qkv_projections()923            self.unet.set_attn_processor(FusedAttnProcessor2_0())924 925        if vae:926            if not isinstance(self.vae, AutoencoderKL):927                raise ValueError("`fuse_qkv_projections()` is only supported for the VAE of type `AutoencoderKL`.")928 929            self.fusing_vae = True930            self.vae.fuse_qkv_projections()931            self.vae.set_attn_processor(FusedAttnProcessor2_0())932 933    # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.unfuse_qkv_projections934    def unfuse_qkv_projections(self, unet: bool = True, vae: bool = True):935        """Disable QKV projection fusion if enabled.936        <Tip warning={true}>937        This API is 🧪 experimental.938        </Tip>939        Args:940            unet (`bool`, defaults to `True`): To apply fusion on the UNet.941            vae (`bool`, defaults to `True`): To apply fusion on the VAE.942        """943        if unet:944            if not self.fusing_unet:945                logger.warning("The UNet was not initially fused for QKV projections. Doing nothing.")946            else:947                self.unet.unfuse_qkv_projections()948                self.fusing_unet = False949 950        if vae:951            if not self.fusing_vae:952                logger.warning("The VAE was not initially fused for QKV projections. Doing nothing.")953            else:954                self.vae.unfuse_qkv_projections()955                self.fusing_vae = False956 957    # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding958    def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32):959        """960        See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298961        Args:962            timesteps (`torch.Tensor`):963                generate embedding vectors at these timesteps964            embedding_dim (`int`, *optional*, defaults to 512):965                dimension of the embeddings to generate966            dtype:967                data type of the generated embeddings968        Returns:969            `torch.Tensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)`970        """971        assert len(w.shape) == 1972        w = w * 1000.0973 974        half_dim = embedding_dim // 2975        emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)976        emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)977        emb = w.to(dtype)[:, None] * emb[None, :]978        emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)979        if embedding_dim % 2 == 1:  # zero pad980            emb = torch.nn.functional.pad(emb, (0, 1))981        assert emb.shape == (w.shape[0], embedding_dim)982        return emb983 984    def pred_z0(self, sample, model_output, timestep):985        alpha_prod_t = self.scheduler.alphas_cumprod[timestep].to(sample.device)986 987        beta_prod_t = 1 - alpha_prod_t988        if self.scheduler.config.prediction_type == "epsilon":989            pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5)990        elif self.scheduler.config.prediction_type == "sample":991            pred_original_sample = model_output992        elif self.scheduler.config.prediction_type == "v_prediction":993            pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output994            # predict V995            model_output = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample996        else:997            raise ValueError(998                f"prediction_type given as {self.scheduler.config.prediction_type} must be one of `epsilon`, `sample`,"999                " or `v_prediction`"1000            )1001 1002        return pred_original_sample1003 1004    def pred_x0(self, latents, noise_pred, t, generator, device, prompt_embeds, output_type):1005        pred_z0 = self.pred_z0(latents, noise_pred, t)1006        pred_x0 = self.vae.decode(pred_z0 / self.vae.config.scaling_factor, return_dict=False, generator=generator)[0]1007        pred_x0, ____ = self.run_safety_checker(pred_x0, device, prompt_embeds.dtype)1008        do_denormalize = [True] * pred_x0.shape[0]1009        pred_x0 = self.image_processor.postprocess(pred_x0, output_type=output_type, do_denormalize=do_denormalize)1010 1011        return pred_x01012 1013    @property1014    def guidance_scale(self):1015        return self._guidance_scale1016 1017    @property1018    def guidance_rescale(self):1019        return self._guidance_rescale1020 1021    @property1022    def clip_skip(self):1023        return self._clip_skip1024 1025    # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)1026    # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`1027    # corresponds to doing no classifier free guidance.1028    @property1029    def do_classifier_free_guidance(self):1030        return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None1031 1032    @property1033    def cross_attention_kwargs(self):1034        return self._cross_attention_kwargs1035 1036    @property1037    def num_timesteps(self):1038        return self._num_timesteps1039 1040    @property1041    def interrupt(self):1042        return self._interrupt1043 1044    @property1045    def pag_scale(self):1046        return self._pag_scale1047 1048    @property1049    def do_perturbed_attention_guidance(self):1050        return self._pag_scale > 01051 1052    @property1053    def pag_adaptive_scaling(self):1054        return self._pag_adaptive_scaling1055 1056    @property1057    def do_pag_adaptive_scaling(self):1058        return self._pag_adaptive_scaling > 01059 1060    @property1061    def pag_applied_layers_index(self):1062        return self._pag_applied_layers_index1063 1064    @torch.no_grad()1065    @replace_example_docstring(EXAMPLE_DOC_STRING)1066    def __call__(1067        self,1068        prompt: Union[str, List[str]] = None,1069        height: Optional[int] = None,1070        width: Optional[int] = None,1071        num_inference_steps: int = 50,1072        timesteps: List[int] = None,1073        guidance_scale: float = 7.5,1074        pag_scale: float = 0.0,1075        pag_adaptive_scaling: float = 0.0,1076        pag_applied_layers_index: List[str] = ["d4"],  # ['d4', 'd5', 'm0']1077        negative_prompt: Optional[Union[str, List[str]]] = None,1078        num_images_per_prompt: Optional[int] = 1,1079        eta: float = 0.0,1080        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,1081        latents: Optional[torch.Tensor] = None,1082        prompt_embeds: Optional[torch.Tensor] = None,1083        negative_prompt_embeds: Optional[torch.Tensor] = None,1084        ip_adapter_image: Optional[PipelineImageInput] = None,1085        ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None,1086        output_type: Optional[str] = "pil",1087        return_dict: bool = True,1088        cross_attention_kwargs: Optional[Dict[str, Any]] = None,1089        guidance_rescale: float = 0.0,1090        clip_skip: Optional[int] = None,1091        callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,1092        callback_on_step_end_tensor_inputs: List[str] = ["latents"],1093        **kwargs,1094    ):1095        r"""1096        The call function to the pipeline for generation.1097        Args:1098            prompt (`str` or `List[str]`, *optional*):1099                The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.1100            height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):1101                The height in pixels of the generated image.1102            width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):1103                The width in pixels of the generated image.1104            num_inference_steps (`int`, *optional*, defaults to 50):1105                The number of denoising steps. More denoising steps usually lead to a higher quality image at the1106                expense of slower inference.1107            timesteps (`List[int]`, *optional*):1108                Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument1109                in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is1110                passed will be used. Must be in descending order.1111            guidance_scale (`float`, *optional*, defaults to 7.5):1112                A higher guidance scale value encourages the model to generate images closely linked to the text1113                `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.1114            negative_prompt (`str` or `List[str]`, *optional*):1115                The prompt or prompts to guide what to not include in image generation. If not defined, you need to1116                pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).1117            num_images_per_prompt (`int`, *optional*, defaults to 1):1118                The number of images to generate per prompt.1119            eta (`float`, *optional*, defaults to 0.0):1120                Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies1121                to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.1122            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):1123                A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make1124                generation deterministic.1125            latents (`torch.Tensor`, *optional*):1126                Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image1127                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents1128                tensor is generated by sampling using the supplied random `generator`.1129            prompt_embeds (`torch.Tensor`, *optional*):1130                Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not1131                provided, text embeddings are generated from the `prompt` input argument.1132            negative_prompt_embeds (`torch.Tensor`, *optional*):1133                Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If1134                not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.1135            ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.1136            ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*):1137                Pre-generated image embeddings for IP-Adapter. If not1138                provided, embeddings are computed from the `ip_adapter_image` input argument.1139            output_type (`str`, *optional*, defaults to `"pil"`):1140                The output format of the generated image. Choose between `PIL.Image` or `np.array`.1141            return_dict (`bool`, *optional*, defaults to `True`):1142                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a1143                plain tuple.1144            cross_attention_kwargs (`dict`, *optional*):1145                A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in1146                [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).1147            guidance_rescale (`float`, *optional*, defaults to 0.0):1148                Guidance rescale factor from [Common Diffusion Noise Schedules and Sample Steps are1149                Flawed](https://arxiv.org/pdf/2305.08891.pdf). Guidance rescale factor should fix overexposure when1150                using zero terminal SNR.1151            clip_skip (`int`, *optional*):1152                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that1153                the output of the pre-final layer will be used for computing the prompt embeddings.1154            callback_on_step_end (`Callable`, *optional*):1155                A function that calls at the end of each denoising steps during the inference. The function is called1156                with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,1157                callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by1158                `callback_on_step_end_tensor_inputs`.1159            callback_on_step_end_tensor_inputs (`List`, *optional*):1160                The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list1161                will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the1162                `._callback_tensor_inputs` attribute of your pipeline class.1163        Examples:1164        Returns:1165            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:1166                If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned,1167                otherwise a `tuple` is returned where the first element is a list with the generated images and the1168                second element is a list of `bool`s indicating whether the corresponding generated image contains1169                "not-safe-for-work" (nsfw) content.1170        """1171 1172        callback = kwargs.pop("callback", None)1173        callback_steps = kwargs.pop("callback_steps", None)1174 1175        if callback is not None:1176            deprecate(1177                "callback",1178                "1.0.0",1179                "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",1180            )1181        if callback_steps is not None:1182            deprecate(1183                "callback_steps",1184                "1.0.0",1185                "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",1186            )1187 1188        # 0. Default height and width to unet1189        height = height or self.unet.config.sample_size * self.vae_scale_factor1190        width = width or self.unet.config.sample_size * self.vae_scale_factor1191        # to deal with lora scaling and other possible forward hooks1192 1193        # 1. Check inputs. Raise error if not correct1194        self.check_inputs(1195            prompt,1196            height,1197            width,1198            callback_steps,1199            negative_prompt,1200            prompt_embeds,

Showing the first 1,200 of 1472 lines. Download the file for the rest.