diffusers/community-pipelines-mirror

1# Copyright 2024 The HuggingFace Team. All rights reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import inspect
16from typing import Any, Callable, Dict, List, Optional, Union
17
18import torch
19from packaging import version
20from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer
21
22from diffusers.configuration_utils import FrozenDict
23from diffusers.image_processor import VaeImageProcessor
24from diffusers.loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin
25from diffusers.models import AutoencoderKL, UNet2DConditionModel
26from diffusers.models.lora import adjust_lora_scale_text_encoder
27from diffusers.pipelines.pipeline_utils import DiffusionPipeline, StableDiffusionMixin
28from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput
29from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker
30from diffusers.schedulers import KarrasDiffusionSchedulers
31from diffusers.utils import (
32    deprecate,
33    logging,
34)
35from diffusers.utils.torch_utils import randn_tensor
36
37
38logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
39
40
41def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):
42    """
43    Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and
44    Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4
45    """
46    std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)
47    std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)
48    # rescale the results from guidance (fixes overexposure)
49    noise_pred_rescaled = noise_cfg * (std_text / std_cfg)
50    # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images
51    noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg
52    return noise_cfg
53
54
55class InstaFlowPipeline(
56    DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin, FromSingleFileMixin
57):
58    r"""
59    Pipeline for text-to-image generation using Rectified Flow and Euler discretization.
60    This customized pipeline is based on StableDiffusionPipeline from the official Diffusers library (0.21.4)
61
62    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods
63    implemented for all pipelines (downloading, saving, running on a particular device, etc.).
64
65    The pipeline also inherits the following loading methods:
66        - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
67        - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights
68        - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights
69        - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
70
71    Args:
72        vae ([`AutoencoderKL`]):
73            Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.
74        text_encoder ([`~transformers.CLIPTextModel`]):
75            Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).
76        tokenizer ([`~transformers.CLIPTokenizer`]):
77            A `CLIPTokenizer` to tokenize text.
78        unet ([`UNet2DConditionModel`]):
79            A `UNet2DConditionModel` to denoise the encoded image latents.
80        scheduler ([`SchedulerMixin`]):
81            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
82            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
83        safety_checker ([`StableDiffusionSafetyChecker`]):
84            Classification module that estimates whether generated images could be considered offensive or harmful.
85            Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details
86            about a model's potential harms.
87        feature_extractor ([`~transformers.CLIPImageProcessor`]):
88            A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`.
89    """
90
91    model_cpu_offload_seq = "text_encoder->unet->vae"
92    _optional_components = ["safety_checker", "feature_extractor"]
93    _exclude_from_cpu_offload = ["safety_checker"]
94
95    def __init__(
96        self,
97        vae: AutoencoderKL,
98        text_encoder: CLIPTextModel,
99        tokenizer: CLIPTokenizer,
100        unet: UNet2DConditionModel,
101        scheduler: KarrasDiffusionSchedulers,
102        safety_checker: StableDiffusionSafetyChecker,
103        feature_extractor: CLIPImageProcessor,
104        requires_safety_checker: bool = True,
105    ):
106        super().__init__()
107
108        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:
109            deprecation_message = (
110                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"
111                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "
112                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"
113                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"
114                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"
115                " file"
116            )
117            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)
118            new_config = dict(scheduler.config)
119            new_config["steps_offset"] = 1
120            scheduler._internal_dict = FrozenDict(new_config)
121
122        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:
123            deprecation_message = (
124                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."
125                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"
126                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"
127                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"
128                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"
129            )
130            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)
131            new_config = dict(scheduler.config)
132            new_config["clip_sample"] = False
133            scheduler._internal_dict = FrozenDict(new_config)
134
135        if safety_checker is None and requires_safety_checker:
136            logger.warning(
137                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"
138                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"
139                " results in services or applications open to the public. Both the diffusers team and Hugging Face"
140                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"
141                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"
142                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."
143            )
144
145        if safety_checker is not None and feature_extractor is None:
146            raise ValueError(
147                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"
148                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
149            )
150
151        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(
152            version.parse(unet.config._diffusers_version).base_version
153        ) < version.parse("0.9.0.dev0")
154        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64
155        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:
156            deprecation_message = (
157                "The configuration file of the unet has set the default `sample_size` to smaller than"
158                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"
159                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"
160                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"
161                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"
162                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"
163                " in the config might lead to incorrect results in future versions. If you have downloaded this"
164                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"
165                " the `unet/config.json` file"
166            )
167            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)
168            new_config = dict(unet.config)
169            new_config["sample_size"] = 64
170            unet._internal_dict = FrozenDict(new_config)
171
172        self.register_modules(
173            vae=vae,
174            text_encoder=text_encoder,
175            tokenizer=tokenizer,
176            unet=unet,
177            scheduler=scheduler,
178            safety_checker=safety_checker,
179            feature_extractor=feature_extractor,
180        )
181        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
182        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
183        self.register_to_config(requires_safety_checker=requires_safety_checker)
184
185    def _encode_prompt(
186        self,
187        prompt,
188        device,
189        num_images_per_prompt,
190        do_classifier_free_guidance,
191        negative_prompt=None,
192        prompt_embeds: Optional[torch.Tensor] = None,
193        negative_prompt_embeds: Optional[torch.Tensor] = None,
194        lora_scale: Optional[float] = None,
195    ):
196        deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple."
197        deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False)
198
199        prompt_embeds_tuple = self.encode_prompt(
200            prompt=prompt,
201            device=device,
202            num_images_per_prompt=num_images_per_prompt,
203            do_classifier_free_guidance=do_classifier_free_guidance,
204            negative_prompt=negative_prompt,
205            prompt_embeds=prompt_embeds,
206            negative_prompt_embeds=negative_prompt_embeds,
207            lora_scale=lora_scale,
208        )
209
210        # concatenate for backwards comp
211        prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]])
212
213        return prompt_embeds
214
215    def encode_prompt(
216        self,
217        prompt,
218        device,
219        num_images_per_prompt,
220        do_classifier_free_guidance,
221        negative_prompt=None,
222        prompt_embeds: Optional[torch.Tensor] = None,
223        negative_prompt_embeds: Optional[torch.Tensor] = None,
224        lora_scale: Optional[float] = None,
225    ):
226        r"""
227        Encodes the prompt into text encoder hidden states.
228
229        Args:
230            prompt (`str` or `List[str]`, *optional*):
231                prompt to be encoded
232            device: (`torch.device`):
233                torch device
234            num_images_per_prompt (`int`):
235                number of images that should be generated per prompt
236            do_classifier_free_guidance (`bool`):
237                whether to use classifier free guidance or not
238            negative_prompt (`str` or `List[str]`, *optional*):
239                The prompt or prompts not to guide the image generation. If not defined, one has to pass
240                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
241                less than `1`).
242            prompt_embeds (`torch.Tensor`, *optional*):
243                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
244                provided, text embeddings will be generated from `prompt` input argument.
245            negative_prompt_embeds (`torch.Tensor`, *optional*):
246                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
247                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
248                argument.
249            lora_scale (`float`, *optional*):
250                A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
251        """
252        # set lora scale so that monkey patched LoRA
253        # function of text encoder can correctly access it
254        if lora_scale is not None and isinstance(self, LoraLoaderMixin):
255            self._lora_scale = lora_scale
256
257            # dynamically adjust the LoRA scale
258            adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)
259
260        if prompt is not None and isinstance(prompt, str):
261            batch_size = 1
262        elif prompt is not None and isinstance(prompt, list):
263            batch_size = len(prompt)
264        else:
265            batch_size = prompt_embeds.shape[0]
266
267        if prompt_embeds is None:
268            # textual inversion: procecss multi-vector tokens if necessary
269            if isinstance(self, TextualInversionLoaderMixin):
270                prompt = self.maybe_convert_prompt(prompt, self.tokenizer)
271
272            text_inputs = self.tokenizer(
273                prompt,
274                padding="max_length",
275                max_length=self.tokenizer.model_max_length,
276                truncation=True,
277                return_tensors="pt",
278            )
279            text_input_ids = text_inputs.input_ids
280            untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
281
282            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
283                text_input_ids, untruncated_ids
284            ):
285                removed_text = self.tokenizer.batch_decode(
286                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]
287                )
288                logger.warning(
289                    "The following part of your input was truncated because CLIP can only handle sequences up to"
290                    f" {self.tokenizer.model_max_length} tokens: {removed_text}"
291                )
292
293            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:
294                attention_mask = text_inputs.attention_mask.to(device)
295            else:
296                attention_mask = None
297
298            prompt_embeds = self.text_encoder(
299                text_input_ids.to(device),
300                attention_mask=attention_mask,
301            )
302            prompt_embeds = prompt_embeds[0]
303
304        if self.text_encoder is not None:
305            prompt_embeds_dtype = self.text_encoder.dtype
306        elif self.unet is not None:
307            prompt_embeds_dtype = self.unet.dtype
308        else:
309            prompt_embeds_dtype = prompt_embeds.dtype
310
311        prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
312
313        bs_embed, seq_len, _ = prompt_embeds.shape
314        # duplicate text embeddings for each generation per prompt, using mps friendly method
315        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
316        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
317
318        # get unconditional embeddings for classifier free guidance
319        if do_classifier_free_guidance and negative_prompt_embeds is None:
320            uncond_tokens: List[str]
321            if negative_prompt is None:
322                uncond_tokens = [""] * batch_size
323            elif prompt is not None and type(prompt) is not type(negative_prompt):
324                raise TypeError(
325                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
326                    f" {type(prompt)}."
327                )
328            elif isinstance(negative_prompt, str):
329                uncond_tokens = [negative_prompt]
330            elif batch_size != len(negative_prompt):
331                raise ValueError(
332                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
333                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
334                    " the batch size of `prompt`."
335                )
336            else:
337                uncond_tokens = negative_prompt
338
339            # textual inversion: procecss multi-vector tokens if necessary
340            if isinstance(self, TextualInversionLoaderMixin):
341                uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer)
342
343            max_length = prompt_embeds.shape[1]
344            uncond_input = self.tokenizer(
345                uncond_tokens,
346                padding="max_length",
347                max_length=max_length,
348                truncation=True,
349                return_tensors="pt",
350            )
351
352            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:
353                attention_mask = uncond_input.attention_mask.to(device)
354            else:
355                attention_mask = None
356
357            negative_prompt_embeds = self.text_encoder(
358                uncond_input.input_ids.to(device),
359                attention_mask=attention_mask,
360            )
361            negative_prompt_embeds = negative_prompt_embeds[0]
362
363        if do_classifier_free_guidance:
364            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
365            seq_len = negative_prompt_embeds.shape[1]
366
367            negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
368
369            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
370            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
371
372        return prompt_embeds, negative_prompt_embeds
373
374    def run_safety_checker(self, image, device, dtype):
375        if self.safety_checker is None:
376            has_nsfw_concept = None
377        else:
378            if torch.is_tensor(image):
379                feature_extractor_input = self.image_processor.postprocess(image, output_type="pil")
380            else:
381                feature_extractor_input = self.image_processor.numpy_to_pil(image)
382            safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device)
383            image, has_nsfw_concept = self.safety_checker(
384                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)
385            )
386        return image, has_nsfw_concept
387
388    def decode_latents(self, latents):
389        deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead"
390        deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False)
391
392        latents = 1 / self.vae.config.scaling_factor * latents
393        image = self.vae.decode(latents, return_dict=False)[0]
394        image = (image / 2 + 0.5).clamp(0, 1)
395        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
396        image = image.cpu().permute(0, 2, 3, 1).float().numpy()
397        return image
398
399    def merge_dW_to_unet(pipe, dW_dict, alpha=1.0):
400        _tmp_sd = pipe.unet.state_dict()
401        for key in dW_dict.keys():
402            _tmp_sd[key] += dW_dict[key] * alpha
403        pipe.unet.load_state_dict(_tmp_sd, strict=False)
404        return pipe
405
406    def prepare_extra_step_kwargs(self, generator, eta):
407        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
408        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
409        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
410        # and should be between [0, 1]
411
412        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
413        extra_step_kwargs = {}
414        if accepts_eta:
415            extra_step_kwargs["eta"] = eta
416
417        # check if the scheduler accepts generator
418        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
419        if accepts_generator:
420            extra_step_kwargs["generator"] = generator
421        return extra_step_kwargs
422
423    def check_inputs(
424        self,
425        prompt,
426        height,
427        width,
428        callback_steps,
429        negative_prompt=None,
430        prompt_embeds=None,
431        negative_prompt_embeds=None,
432    ):
433        if height % 8 != 0 or width % 8 != 0:
434            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
435
436        if (callback_steps is None) or (
437            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)
438        ):
439            raise ValueError(
440                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
441                f" {type(callback_steps)}."
442            )
443
444        if prompt is not None and prompt_embeds is not None:
445            raise ValueError(
446                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
447                " only forward one of the two."
448            )
449        elif prompt is None and prompt_embeds is None:
450            raise ValueError(
451                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
452            )
453        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
454            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
455
456        if negative_prompt is not None and negative_prompt_embeds is not None:
457            raise ValueError(
458                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
459                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
460            )
461
462        if prompt_embeds is not None and negative_prompt_embeds is not None:
463            if prompt_embeds.shape != negative_prompt_embeds.shape:
464                raise ValueError(
465                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
466                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
467                    f" {negative_prompt_embeds.shape}."
468                )
469
470    def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):
471        shape = (
472            batch_size,
473            num_channels_latents,
474            int(height) // self.vae_scale_factor,
475            int(width) // self.vae_scale_factor,
476        )
477        if isinstance(generator, list) and len(generator) != batch_size:
478            raise ValueError(
479                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
480                f" size of {batch_size}. Make sure the batch size matches the length of the generators."
481            )
482
483        if latents is None:
484            latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
485        else:
486            latents = latents.to(device)
487
488        # scale the initial noise by the standard deviation required by the scheduler
489        latents = latents * self.scheduler.init_noise_sigma
490        return latents
491
492    @torch.no_grad()
493    def __call__(
494        self,
495        prompt: Union[str, List[str]] = None,
496        height: Optional[int] = None,
497        width: Optional[int] = None,
498        num_inference_steps: int = 50,
499        guidance_scale: float = 7.5,
500        negative_prompt: Optional[Union[str, List[str]]] = None,
501        num_images_per_prompt: Optional[int] = 1,
502        eta: float = 0.0,
503        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
504        latents: Optional[torch.Tensor] = None,
505        prompt_embeds: Optional[torch.Tensor] = None,
506        negative_prompt_embeds: Optional[torch.Tensor] = None,
507        output_type: Optional[str] = "pil",
508        return_dict: bool = True,
509        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,
510        callback_steps: int = 1,
511        cross_attention_kwargs: Optional[Dict[str, Any]] = None,
512        guidance_rescale: float = 0.0,
513    ):
514        r"""
515        The call function to the pipeline for generation.
516
517        Args:
518            prompt (`str` or `List[str]`, *optional*):
519                The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.
520            height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):
521                The height in pixels of the generated image.
522            width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):
523                The width in pixels of the generated image.
524            num_inference_steps (`int`, *optional*, defaults to 50):
525                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
526                expense of slower inference.
527            guidance_scale (`float`, *optional*, defaults to 7.5):
528                A higher guidance scale value encourages the model to generate images closely linked to the text
529                `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.
530            negative_prompt (`str` or `List[str]`, *optional*):
531                The prompt or prompts to guide what to not include in image generation. If not defined, you need to
532                pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).
533            num_images_per_prompt (`int`, *optional*, defaults to 1):
534                The number of images to generate per prompt.
535            eta (`float`, *optional*, defaults to 0.0):
536                Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies
537                to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.
538            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
539                A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
540                generation deterministic.
541            latents (`torch.Tensor`, *optional*):
542                Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image
543                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
544                tensor is generated by sampling using the supplied random `generator`.
545            prompt_embeds (`torch.Tensor`, *optional*):
546                Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
547                provided, text embeddings are generated from the `prompt` input argument.
548            negative_prompt_embeds (`torch.Tensor`, *optional*):
549                Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
550                not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
551            output_type (`str`, *optional*, defaults to `"pil"`):
552                The output format of the generated image. Choose between `PIL.Image` or `np.array`.
553            return_dict (`bool`, *optional*, defaults to `True`):
554                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
555                plain tuple.
556            callback (`Callable`, *optional*):
557                A function that calls every `callback_steps` steps during inference. The function is called with the
558                following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.
559            callback_steps (`int`, *optional*, defaults to 1):
560                The frequency at which the `callback` function is called. If not specified, the callback is called at
561                every step.
562            cross_attention_kwargs (`dict`, *optional*):
563                A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in
564                [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
565            guidance_rescale (`float`, *optional*, defaults to 0.7):
566                Guidance rescale factor from [Common Diffusion Noise Schedules and Sample Steps are
567                Flawed](https://arxiv.org/pdf/2305.08891.pdf). Guidance rescale factor should fix overexposure when
568                using zero terminal SNR.
569
570        Examples:
571
572        Returns:
573            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
574                If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned,
575                otherwise a `tuple` is returned where the first element is a list with the generated images and the
576                second element is a list of `bool`s indicating whether the corresponding generated image contains
577                "not-safe-for-work" (nsfw) content.
578        """
579        # 0. Default height and width to unet
580        height = height or self.unet.config.sample_size * self.vae_scale_factor
581        width = width or self.unet.config.sample_size * self.vae_scale_factor
582
583        # 1. Check inputs. Raise error if not correct
584        self.check_inputs(
585            prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds
586        )
587
588        # 2. Define call parameters
589        if prompt is not None and isinstance(prompt, str):
590            batch_size = 1
591        elif prompt is not None and isinstance(prompt, list):
592            batch_size = len(prompt)
593        else:
594            batch_size = prompt_embeds.shape[0]
595
596        device = self._execution_device
597        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
598        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
599        # corresponds to doing no classifier free guidance.
600        do_classifier_free_guidance = guidance_scale > 1.0
601
602        # 3. Encode input prompt
603        text_encoder_lora_scale = (
604            cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None
605        )
606        prompt_embeds, negative_prompt_embeds = self.encode_prompt(
607            prompt,
608            device,
609            num_images_per_prompt,
610            do_classifier_free_guidance,
611            negative_prompt,
612            prompt_embeds=prompt_embeds,
613            negative_prompt_embeds=negative_prompt_embeds,
614            lora_scale=text_encoder_lora_scale,
615        )
616        # For classifier free guidance, we need to do two forward passes.
617        # Here we concatenate the unconditional and text embeddings into a single batch
618        # to avoid doing two forward passes
619        if do_classifier_free_guidance:
620            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])
621
622        # 4. Prepare timesteps
623        timesteps = [(1.0 - i / num_inference_steps) * 1000.0 for i in range(num_inference_steps)]
624
625        # 5. Prepare latent variables
626        num_channels_latents = self.unet.config.in_channels
627        latents = self.prepare_latents(
628            batch_size * num_images_per_prompt,
629            num_channels_latents,
630            height,
631            width,
632            prompt_embeds.dtype,
633            device,
634            generator,
635            latents,
636        )
637
638        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
639        dt = 1.0 / num_inference_steps
640
641        # 7. Denoising loop of Euler discretization from t = 0 to t = 1
642        with self.progress_bar(total=num_inference_steps) as progress_bar:
643            for i, t in enumerate(timesteps):
644                # expand the latents if we are doing classifier free guidance
645                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
646
647                vec_t = torch.ones((latent_model_input.shape[0],), device=latents.device) * t
648
649                v_pred = self.unet(latent_model_input, vec_t, encoder_hidden_states=prompt_embeds).sample
650
651                # perform guidance
652                if do_classifier_free_guidance:
653                    v_pred_neg, v_pred_text = v_pred.chunk(2)
654                    v_pred = v_pred_neg + guidance_scale * (v_pred_text - v_pred_neg)
655
656                latents = latents + dt * v_pred
657
658                # call the callback, if provided
659                if i == len(timesteps) - 1 or ((i + 1) % self.scheduler.order == 0):
660                    progress_bar.update()
661                    if callback is not None and i % callback_steps == 0:
662                        step_idx = i // getattr(self.scheduler, "order", 1)
663                        callback(step_idx, t, latents)
664
665        if not output_type == "latent":
666            image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]
667            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)
668        else:
669            image = latents
670            has_nsfw_concept = None
671
672        if has_nsfw_concept is None:
673            do_denormalize = [True] * image.shape[0]
674        else:
675            do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept]
676
677        image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)
678
679        # Offload all models
680        self.maybe_free_model_hooks()
681
682        if not return_dict:
683            return (image, has_nsfw_concept)
684
685        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)
686