diffusers/community-pipelines-mirror

1import inspect2from typing import Any, Dict, List, Optional, Union3 4import torch5import torch.nn as nn6from transformers import AutoModel, AutoTokenizer, CLIPImageProcessor7 8from diffusers import DiffusionPipeline9from diffusers.image_processor import VaeImageProcessor10from diffusers.loaders import LoraLoaderMixin11from diffusers.models import AutoencoderKL, UNet2DConditionModel12from diffusers.models.lora import adjust_lora_scale_text_encoder13from diffusers.pipelines.pipeline_utils import StableDiffusionMixin14from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput15from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker16from diffusers.schedulers import KarrasDiffusionSchedulers17from diffusers.utils import (18    USE_PEFT_BACKEND,19    logging,20    scale_lora_layers,21    unscale_lora_layers,22)23from diffusers.utils.torch_utils import randn_tensor24 25 26logger = logging.get_logger(__name__)  # pylint: disable=invalid-name27 28 29class TranslatorBase(nn.Module):30    def __init__(self, num_tok, dim, dim_out, mult=2):31        super().__init__()32 33        self.dim_in = dim34        self.dim_out = dim_out35 36        self.net_tok = nn.Sequential(37            nn.Linear(num_tok, int(num_tok * mult)),38            nn.LayerNorm(int(num_tok * mult)),39            nn.GELU(),40            nn.Linear(int(num_tok * mult), int(num_tok * mult)),41            nn.LayerNorm(int(num_tok * mult)),42            nn.GELU(),43            nn.Linear(int(num_tok * mult), num_tok),44            nn.LayerNorm(num_tok),45        )46 47        self.net_sen = nn.Sequential(48            nn.Linear(dim, int(dim * mult)),49            nn.LayerNorm(int(dim * mult)),50            nn.GELU(),51            nn.Linear(int(dim * mult), int(dim * mult)),52            nn.LayerNorm(int(dim * mult)),53            nn.GELU(),54            nn.Linear(int(dim * mult), dim_out),55            nn.LayerNorm(dim_out),56        )57 58    def forward(self, x):59        if self.dim_in == self.dim_out:60            indentity_0 = x61            x = self.net_sen(x)62            x += indentity_063            x = x.transpose(1, 2)64 65            indentity_1 = x66            x = self.net_tok(x)67            x += indentity_168            x = x.transpose(1, 2)69        else:70            x = self.net_sen(x)71            x = x.transpose(1, 2)72 73            x = self.net_tok(x)74            x = x.transpose(1, 2)75        return x76 77 78class TranslatorBaseNoLN(nn.Module):79    def __init__(self, num_tok, dim, dim_out, mult=2):80        super().__init__()81 82        self.dim_in = dim83        self.dim_out = dim_out84 85        self.net_tok = nn.Sequential(86            nn.Linear(num_tok, int(num_tok * mult)),87            nn.GELU(),88            nn.Linear(int(num_tok * mult), int(num_tok * mult)),89            nn.GELU(),90            nn.Linear(int(num_tok * mult), num_tok),91        )92 93        self.net_sen = nn.Sequential(94            nn.Linear(dim, int(dim * mult)),95            nn.GELU(),96            nn.Linear(int(dim * mult), int(dim * mult)),97            nn.GELU(),98            nn.Linear(int(dim * mult), dim_out),99        )100 101    def forward(self, x):102        if self.dim_in == self.dim_out:103            indentity_0 = x104            x = self.net_sen(x)105            x += indentity_0106            x = x.transpose(1, 2)107 108            indentity_1 = x109            x = self.net_tok(x)110            x += indentity_1111            x = x.transpose(1, 2)112        else:113            x = self.net_sen(x)114            x = x.transpose(1, 2)115 116            x = self.net_tok(x)117            x = x.transpose(1, 2)118        return x119 120 121class TranslatorNoLN(nn.Module):122    def __init__(self, num_tok, dim, dim_out, mult=2, depth=5):123        super().__init__()124 125        self.blocks = nn.ModuleList([TranslatorBase(num_tok, dim, dim, mult=2) for d in range(depth)])126        self.gelu = nn.GELU()127 128        self.tail = TranslatorBaseNoLN(num_tok, dim, dim_out, mult=2)129 130    def forward(self, x):131        for block in self.blocks:132            x = block(x) + x133            x = self.gelu(x)134 135        x = self.tail(x)136        return x137 138 139def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):140    """141    Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and142    Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4143    """144    std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)145    std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)146    # rescale the results from guidance (fixes overexposure)147    noise_pred_rescaled = noise_cfg * (std_text / std_cfg)148    # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images149    noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg150    return noise_cfg151 152 153def retrieve_timesteps(154    scheduler,155    num_inference_steps: Optional[int] = None,156    device: Optional[Union[str, torch.device]] = None,157    timesteps: Optional[List[int]] = None,158    **kwargs,159):160    """161    Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles162    custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.163 164    Args:165        scheduler (`SchedulerMixin`):166            The scheduler to get timesteps from.167        num_inference_steps (`int`):168            The number of diffusion steps used when generating samples with a pre-trained model. If used,169            `timesteps` must be `None`.170        device (`str` or `torch.device`, *optional*):171            The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.172        timesteps (`List[int]`, *optional*):173                Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default174                timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps`175                must be `None`.176 177    Returns:178        `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the179        second element is the number of inference steps.180    """181    if timesteps is not None:182        accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())183        if not accepts_timesteps:184            raise ValueError(185                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"186                f" timestep schedules. Please check whether you are using the correct scheduler."187            )188        scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)189        timesteps = scheduler.timesteps190        num_inference_steps = len(timesteps)191    else:192        scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)193        timesteps = scheduler.timesteps194    return timesteps, num_inference_steps195 196 197class GlueGenStableDiffusionPipeline(DiffusionPipeline, StableDiffusionMixin, LoraLoaderMixin):198    def __init__(199        self,200        vae: AutoencoderKL,201        text_encoder: AutoModel,202        tokenizer: AutoTokenizer,203        unet: UNet2DConditionModel,204        scheduler: KarrasDiffusionSchedulers,205        safety_checker: StableDiffusionSafetyChecker,206        feature_extractor: CLIPImageProcessor,207        language_adapter: TranslatorNoLN = None,208        tensor_norm: torch.Tensor = None,209        requires_safety_checker: bool = True,210    ):211        super().__init__()212 213        self.register_modules(214            vae=vae,215            text_encoder=text_encoder,216            tokenizer=tokenizer,217            unet=unet,218            scheduler=scheduler,219            safety_checker=safety_checker,220            feature_extractor=feature_extractor,221            language_adapter=language_adapter,222            tensor_norm=tensor_norm,223        )224        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)225        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)226        self.register_to_config(requires_safety_checker=requires_safety_checker)227 228    def load_language_adapter(229        self,230        model_path: str,231        num_token: int,232        dim: int,233        dim_out: int,234        tensor_norm: torch.Tensor,235        mult: int = 2,236        depth: int = 5,237    ):238        device = self._execution_device239        self.tensor_norm = tensor_norm.to(device)240        self.language_adapter = TranslatorNoLN(num_tok=num_token, dim=dim, dim_out=dim_out, mult=mult, depth=depth).to(241            device242        )243        self.language_adapter.load_state_dict(torch.load(model_path))244 245    def _adapt_language(self, prompt_embeds: torch.Tensor):246        prompt_embeds = prompt_embeds / 3247        prompt_embeds = self.language_adapter(prompt_embeds) * (self.tensor_norm / 2)248        return prompt_embeds249 250    def encode_prompt(251        self,252        prompt,253        device,254        num_images_per_prompt,255        do_classifier_free_guidance,256        negative_prompt=None,257        prompt_embeds: Optional[torch.Tensor] = None,258        negative_prompt_embeds: Optional[torch.Tensor] = None,259        lora_scale: Optional[float] = None,260        clip_skip: Optional[int] = None,261    ):262        r"""263        Encodes the prompt into text encoder hidden states.264 265        Args:266            prompt (`str` or `List[str]`, *optional*):267                prompt to be encoded268            device: (`torch.device`):269                torch device270            num_images_per_prompt (`int`):271                number of images that should be generated per prompt272            do_classifier_free_guidance (`bool`):273                whether to use classifier free guidance or not274            negative_prompt (`str` or `List[str]`, *optional*):275                The prompt or prompts not to guide the image generation. If not defined, one has to pass276                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is277                less than `1`).278            prompt_embeds (`torch.Tensor`, *optional*):279                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not280                provided, text embeddings will be generated from `prompt` input argument.281            negative_prompt_embeds (`torch.Tensor`, *optional*):282                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt283                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input284                argument.285            lora_scale (`float`, *optional*):286                A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.287            clip_skip (`int`, *optional*):288                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that289                the output of the pre-final layer will be used for computing the prompt embeddings.290        """291        # set lora scale so that monkey patched LoRA292        # function of text encoder can correctly access it293        if lora_scale is not None and isinstance(self, LoraLoaderMixin):294            self._lora_scale = lora_scale295 296            # dynamically adjust the LoRA scale297            if not USE_PEFT_BACKEND:298                adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)299            else:300                scale_lora_layers(self.text_encoder, lora_scale)301 302        if prompt is not None and isinstance(prompt, str):303            batch_size = 1304        elif prompt is not None and isinstance(prompt, list):305            batch_size = len(prompt)306        else:307            batch_size = prompt_embeds.shape[0]308 309        if prompt_embeds is None:310            text_inputs = self.tokenizer(311                prompt,312                padding="max_length",313                max_length=self.tokenizer.model_max_length,314                truncation=True,315                return_tensors="pt",316            )317            text_input_ids = text_inputs.input_ids318            untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids319 320            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(321                text_input_ids, untruncated_ids322            ):323                removed_text = self.tokenizer.batch_decode(324                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]325                )326                logger.warning(327                    "The following part of your input was truncated because CLIP can only handle sequences up to"328                    f" {self.tokenizer.model_max_length} tokens: {removed_text}"329                )330 331            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:332                attention_mask = text_inputs.attention_mask.to(device)333            elif self.language_adapter is not None:334                attention_mask = text_inputs.attention_mask.to(device)335            else:336                attention_mask = None337 338            if clip_skip is None:339                prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask)340                prompt_embeds = prompt_embeds[0]341 342            else:343                prompt_embeds = self.text_encoder(344                    text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True345                )346                # Access the `hidden_states` first, that contains a tuple of347                # all the hidden states from the encoder layers. Then index into348                # the tuple to access the hidden states from the desired layer.349                prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)]350                # We also need to apply the final LayerNorm here to not mess with the351                # representations. The `last_hidden_states` that we typically use for352                # obtaining the final prompt representations passes through the LayerNorm353                # layer.354                prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds)355 356            # Run prompt language adapter357            if self.language_adapter is not None:358                prompt_embeds = self._adapt_language(prompt_embeds)359 360        if self.text_encoder is not None:361            prompt_embeds_dtype = self.text_encoder.dtype362        elif self.unet is not None:363            prompt_embeds_dtype = self.unet.dtype364        else:365            prompt_embeds_dtype = prompt_embeds.dtype366 367        prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)368 369        bs_embed, seq_len, _ = prompt_embeds.shape370        # duplicate text embeddings for each generation per prompt, using mps friendly method371        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)372        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)373 374        # get unconditional embeddings for classifier free guidance375        if do_classifier_free_guidance and negative_prompt_embeds is None:376            uncond_tokens: List[str]377            if negative_prompt is None:378                uncond_tokens = [""] * batch_size379            elif prompt is not None and type(prompt) is not type(negative_prompt):380                raise TypeError(381                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="382                    f" {type(prompt)}."383                )384            elif isinstance(negative_prompt, str):385                uncond_tokens = [negative_prompt]386            elif batch_size != len(negative_prompt):387                raise ValueError(388                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"389                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"390                    " the batch size of `prompt`."391                )392            else:393                uncond_tokens = negative_prompt394 395            max_length = prompt_embeds.shape[1]396            uncond_input = self.tokenizer(397                uncond_tokens,398                padding="max_length",399                max_length=max_length,400                truncation=True,401                return_tensors="pt",402            )403 404            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:405                attention_mask = uncond_input.attention_mask.to(device)406            else:407                attention_mask = None408 409            negative_prompt_embeds = self.text_encoder(410                uncond_input.input_ids.to(device),411                attention_mask=attention_mask,412            )413            negative_prompt_embeds = negative_prompt_embeds[0]414            # Run negative prompt language adapter415            if self.language_adapter is not None:416                negative_prompt_embeds = self._adapt_language(negative_prompt_embeds)417 418        if do_classifier_free_guidance:419            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method420            seq_len = negative_prompt_embeds.shape[1]421 422            negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)423 424            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)425            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)426 427        if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND:428            # Retrieve the original scale by scaling back the LoRA layers429            unscale_lora_layers(self.text_encoder, lora_scale)430 431        return prompt_embeds, negative_prompt_embeds432 433    def run_safety_checker(self, image, device, dtype):434        if self.safety_checker is None:435            has_nsfw_concept = None436        else:437            if torch.is_tensor(image):438                feature_extractor_input = self.image_processor.postprocess(image, output_type="pil")439            else:440                feature_extractor_input = self.image_processor.numpy_to_pil(image)441            safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device)442            image, has_nsfw_concept = self.safety_checker(443                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)444            )445        return image, has_nsfw_concept446 447    def prepare_extra_step_kwargs(self, generator, eta):448        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature449        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.450        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502451        # and should be between [0, 1]452 453        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())454        extra_step_kwargs = {}455        if accepts_eta:456            extra_step_kwargs["eta"] = eta457 458        # check if the scheduler accepts generator459        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())460        if accepts_generator:461            extra_step_kwargs["generator"] = generator462        return extra_step_kwargs463 464    def check_inputs(465        self,466        prompt,467        height,468        width,469        negative_prompt=None,470        prompt_embeds=None,471        negative_prompt_embeds=None,472    ):473        if height % 8 != 0 or width % 8 != 0:474            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")475 476        if prompt is not None and prompt_embeds is not None:477            raise ValueError(478                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"479                " only forward one of the two."480            )481        elif prompt is None and prompt_embeds is None:482            raise ValueError(483                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."484            )485        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):486            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")487 488        if negative_prompt is not None and negative_prompt_embeds is not None:489            raise ValueError(490                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"491                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."492            )493 494        if prompt_embeds is not None and negative_prompt_embeds is not None:495            if prompt_embeds.shape != negative_prompt_embeds.shape:496                raise ValueError(497                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"498                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"499                    f" {negative_prompt_embeds.shape}."500                )501 502    def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):503        shape = (504            batch_size,505            num_channels_latents,506            int(height) // self.vae_scale_factor,507            int(width) // self.vae_scale_factor,508        )509        if isinstance(generator, list) and len(generator) != batch_size:510            raise ValueError(511                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"512                f" size of {batch_size}. Make sure the batch size matches the length of the generators."513            )514 515        if latents is None:516            latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)517        else:518            latents = latents.to(device)519 520        # scale the initial noise by the standard deviation required by the scheduler521        latents = latents * self.scheduler.init_noise_sigma522        return latents523 524    # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding525    def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32):526        """527        See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298528 529        Args:530            timesteps (`torch.Tensor`):531                generate embedding vectors at these timesteps532            embedding_dim (`int`, *optional*, defaults to 512):533                dimension of the embeddings to generate534            dtype:535                data type of the generated embeddings536 537        Returns:538            `torch.Tensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)`539        """540        assert len(w.shape) == 1541        w = w * 1000.0542 543        half_dim = embedding_dim // 2544        emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)545        emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)546        emb = w.to(dtype)[:, None] * emb[None, :]547        emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)548        if embedding_dim % 2 == 1:  # zero pad549            emb = torch.nn.functional.pad(emb, (0, 1))550        assert emb.shape == (w.shape[0], embedding_dim)551        return emb552 553    @property554    def guidance_scale(self):555        return self._guidance_scale556 557    @property558    def guidance_rescale(self):559        return self._guidance_rescale560 561    @property562    def clip_skip(self):563        return self._clip_skip564 565    # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)566    # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`567    # corresponds to doing no classifier free guidance.568    @property569    def do_classifier_free_guidance(self):570        return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None571 572    @property573    def cross_attention_kwargs(self):574        return self._cross_attention_kwargs575 576    @property577    def num_timesteps(self):578        return self._num_timesteps579 580    @property581    def interrupt(self):582        return self._interrupt583 584    @torch.no_grad()585    def __call__(586        self,587        prompt: Union[str, List[str]] = None,588        height: Optional[int] = None,589        width: Optional[int] = None,590        num_inference_steps: int = 50,591        timesteps: List[int] = None,592        guidance_scale: float = 7.5,593        negative_prompt: Optional[Union[str, List[str]]] = None,594        num_images_per_prompt: Optional[int] = 1,595        eta: float = 0.0,596        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,597        latents: Optional[torch.Tensor] = None,598        prompt_embeds: Optional[torch.Tensor] = None,599        negative_prompt_embeds: Optional[torch.Tensor] = None,600        output_type: Optional[str] = "pil",601        return_dict: bool = True,602        cross_attention_kwargs: Optional[Dict[str, Any]] = None,603        guidance_rescale: float = 0.0,604        clip_skip: Optional[int] = None,605        **kwargs,606    ):607        r"""608        The call function to the pipeline for generation.609 610        Args:611            prompt (`str` or `List[str]`, *optional*):612                The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.613            height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):614                The height in pixels of the generated image.615            width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):616                The width in pixels of the generated image.617            num_inference_steps (`int`, *optional*, defaults to 50):618                The number of denoising steps. More denoising steps usually lead to a higher quality image at the619                expense of slower inference.620            timesteps (`List[int]`, *optional*):621                Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument622                in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is623                passed will be used. Must be in descending order.624            guidance_scale (`float`, *optional*, defaults to 7.5):625                A higher guidance scale value encourages the model to generate images closely linked to the text626                `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.627            negative_prompt (`str` or `List[str]`, *optional*):628                The prompt or prompts to guide what to not include in image generation. If not defined, you need to629                pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).630            num_images_per_prompt (`int`, *optional*, defaults to 1):631                The number of images to generate per prompt.632            eta (`float`, *optional*, defaults to 0.0):633                Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies634                to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.635            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):636                A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make637                generation deterministic.638            latents (`torch.Tensor`, *optional*):639                Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image640                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents641                tensor is generated by sampling using the supplied random `generator`.642            prompt_embeds (`torch.Tensor`, *optional*):643                Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not644                provided, text embeddings are generated from the `prompt` input argument.645            negative_prompt_embeds (`torch.Tensor`, *optional*):646                Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If647                not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.648            ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.649            output_type (`str`, *optional*, defaults to `"pil"`):650                The output format of the generated image. Choose between `PIL.Image` or `np.array`.651            return_dict (`bool`, *optional*, defaults to `True`):652                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a653                plain tuple.654            cross_attention_kwargs (`dict`, *optional*):655                A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in656                [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).657            guidance_rescale (`float`, *optional*, defaults to 0.0):658                Guidance rescale factor from [Common Diffusion Noise Schedules and Sample Steps are659                Flawed](https://arxiv.org/pdf/2305.08891.pdf). Guidance rescale factor should fix overexposure when660                using zero terminal SNR.661            clip_skip (`int`, *optional*):662                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that663                the output of the pre-final layer will be used for computing the prompt embeddings.664 665        Examples:666 667        Returns:668            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:669                If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned,670                otherwise a `tuple` is returned where the first element is a list with the generated images and the671                second element is a list of `bool`s indicating whether the corresponding generated image contains672                "not-safe-for-work" (nsfw) content.673        """674 675        # 0. Default height and width to unet676        height = height or self.unet.config.sample_size * self.vae_scale_factor677        width = width or self.unet.config.sample_size * self.vae_scale_factor678        # to deal with lora scaling and other possible forward hooks679 680        # 1. Check inputs. Raise error if not correct681        self.check_inputs(682            prompt,683            height,684            width,685            negative_prompt,686            prompt_embeds,687            negative_prompt_embeds,688        )689 690        self._guidance_scale = guidance_scale691        self._guidance_rescale = guidance_rescale692        self._clip_skip = clip_skip693        self._cross_attention_kwargs = cross_attention_kwargs694        self._interrupt = False695 696        # 2. Define call parameters697        if prompt is not None and isinstance(prompt, str):698            batch_size = 1699        elif prompt is not None and isinstance(prompt, list):700            batch_size = len(prompt)701        else:702            batch_size = prompt_embeds.shape[0]703 704        device = self._execution_device705 706        # 3. Encode input prompt707        lora_scale = (708            self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None709        )710 711        prompt_embeds, negative_prompt_embeds = self.encode_prompt(712            prompt,713            device,714            num_images_per_prompt,715            self.do_classifier_free_guidance,716            negative_prompt,717            prompt_embeds=prompt_embeds,718            negative_prompt_embeds=negative_prompt_embeds,719            lora_scale=lora_scale,720            clip_skip=self.clip_skip,721        )722 723        # For classifier free guidance, we need to do two forward passes.724        # Here we concatenate the unconditional and text embeddings into a single batch725        # to avoid doing two forward passes726        if self.do_classifier_free_guidance:727            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])728 729        # 4. Prepare timesteps730        timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps)731 732        # 5. Prepare latent variables733        num_channels_latents = self.unet.config.in_channels734        latents = self.prepare_latents(735            batch_size * num_images_per_prompt,736            num_channels_latents,737            height,738            width,739            prompt_embeds.dtype,740            device,741            generator,742            latents,743        )744 745        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline746        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)747 748        # 6.2 Optionally get Guidance Scale Embedding749        timestep_cond = None750        if self.unet.config.time_cond_proj_dim is not None:751            guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)752            timestep_cond = self.get_guidance_scale_embedding(753                guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim754            ).to(device=device, dtype=latents.dtype)755 756        # 7. Denoising loop757        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order758        self._num_timesteps = len(timesteps)759        with self.progress_bar(total=num_inference_steps) as progress_bar:760            for i, t in enumerate(timesteps):761                if self.interrupt:762                    continue763 764                # expand the latents if we are doing classifier free guidance765                latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents766                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)767 768                # predict the noise residual769                noise_pred = self.unet(770                    latent_model_input,771                    t,772                    encoder_hidden_states=prompt_embeds,773                    timestep_cond=timestep_cond,774                    cross_attention_kwargs=self.cross_attention_kwargs,775                    return_dict=False,776                )[0]777 778                # perform guidance779                if self.do_classifier_free_guidance:780                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)781                    noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)782 783                if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:784                    # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf785                    noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale)786 787                # compute the previous noisy sample x_t -> x_t-1788                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]789 790                # call the callback, if provided791                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):792                    progress_bar.update()793 794        if not output_type == "latent":795            image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[796                0797            ]798            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)799        else:800            image = latents801            has_nsfw_concept = None802 803        if has_nsfw_concept is None:804            do_denormalize = [True] * image.shape[0]805        else:806            do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept]807 808        image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)809 810        # Offload all models811        self.maybe_free_model_hooks()812 813        if not return_dict:814            return (image, has_nsfw_concept)815 816        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)817