diffusers/community-pipelines-mirror

1# Copyright 2023 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import inspect16from typing import Any, Callable, Dict, List, Optional, Union17 18import intel_extension_for_pytorch as ipex19import torch20from packaging import version21from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer22 23from diffusers.configuration_utils import FrozenDict24from diffusers.loaders import LoraLoaderMixin, TextualInversionLoaderMixin25from diffusers.models import AutoencoderKL, UNet2DConditionModel26from diffusers.pipelines.pipeline_utils import DiffusionPipeline27from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput28from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker29from diffusers.schedulers import KarrasDiffusionSchedulers30from diffusers.utils import (31    deprecate,32    is_accelerate_available,33    is_accelerate_version,34    logging,35    replace_example_docstring,36)37from diffusers.utils.torch_utils import randn_tensor38 39 40logger = logging.get_logger(__name__)  # pylint: disable=invalid-name41 42EXAMPLE_DOC_STRING = """43    Examples:44        ```py45        >>> import torch46        >>> from diffusers import StableDiffusionPipeline47 48        >>> pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", custom_pipeline="stable_diffusion_ipex")49 50        >>> # For Float3251        >>> pipe.prepare_for_ipex(prompt, dtype=torch.float32, height=512, width=512) #value of image height/width should be consistent with the pipeline inference52        >>> # For BFloat1653        >>> pipe.prepare_for_ipex(prompt, dtype=torch.bfloat16, height=512, width=512) #value of image height/width should be consistent with the pipeline inference54 55        >>> prompt = "a photo of an astronaut riding a horse on mars"56        >>> # For Float3257        >>> image = pipe(prompt, num_inference_steps=num_inference_steps, height=512, width=512).images[0] #value of image height/width should be consistent with 'prepare_for_ipex()'58        >>> # For BFloat1659        >>> with torch.cpu.amp.autocast(enabled=True, dtype=torch.bfloat16):60        >>>     image = pipe(prompt, num_inference_steps=num_inference_steps, height=512, width=512).images[0] #value of image height/width should be consistent with 'prepare_for_ipex()'61        ```62"""63 64 65class StableDiffusionIPEXPipeline(DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin):66    r"""67    Pipeline for text-to-image generation using Stable Diffusion on IPEX.68 69    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the70    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)71 72    Args:73        vae ([`AutoencoderKL`]):74            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.75        text_encoder ([`CLIPTextModel`]):76            Frozen text-encoder. Stable Diffusion uses the text portion of77            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically78            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.79        tokenizer (`CLIPTokenizer`):80            Tokenizer of class81            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).82        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.83        scheduler ([`SchedulerMixin`]):84            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of85            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].86        safety_checker ([`StableDiffusionSafetyChecker`]):87            Classification module that estimates whether generated images could be considered offensive or harmful.88            Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details.89        feature_extractor ([`CLIPFeatureExtractor`]):90            Model that extracts features from generated images to be used as inputs for the `safety_checker`.91    """92 93    _optional_components = ["safety_checker", "feature_extractor"]94 95    def __init__(96        self,97        vae: AutoencoderKL,98        text_encoder: CLIPTextModel,99        tokenizer: CLIPTokenizer,100        unet: UNet2DConditionModel,101        scheduler: KarrasDiffusionSchedulers,102        safety_checker: StableDiffusionSafetyChecker,103        feature_extractor: CLIPFeatureExtractor,104        requires_safety_checker: bool = True,105    ):106        super().__init__()107 108        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:109            deprecation_message = (110                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"111                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "112                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"113                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"114                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"115                " file"116            )117            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)118            new_config = dict(scheduler.config)119            new_config["steps_offset"] = 1120            scheduler._internal_dict = FrozenDict(new_config)121 122        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:123            deprecation_message = (124                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."125                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"126                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"127                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"128                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"129            )130            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)131            new_config = dict(scheduler.config)132            new_config["clip_sample"] = False133            scheduler._internal_dict = FrozenDict(new_config)134 135        if safety_checker is None and requires_safety_checker:136            logger.warning(137                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"138                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"139                " results in services or applications open to the public. Both the diffusers team and Hugging Face"140                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"141                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"142                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."143            )144 145        if safety_checker is not None and feature_extractor is None:146            raise ValueError(147                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"148                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."149            )150 151        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(152            version.parse(unet.config._diffusers_version).base_version153        ) < version.parse("0.9.0.dev0")154        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64155        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:156            deprecation_message = (157                "The configuration file of the unet has set the default `sample_size` to smaller than"158                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"159                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"160                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"161                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"162                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"163                " in the config might lead to incorrect results in future versions. If you have downloaded this"164                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"165                " the `unet/config.json` file"166            )167            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)168            new_config = dict(unet.config)169            new_config["sample_size"] = 64170            unet._internal_dict = FrozenDict(new_config)171 172        self.register_modules(173            vae=vae,174            text_encoder=text_encoder,175            tokenizer=tokenizer,176            unet=unet,177            scheduler=scheduler,178            safety_checker=safety_checker,179            feature_extractor=feature_extractor,180        )181        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)182        self.register_to_config(requires_safety_checker=requires_safety_checker)183 184    def get_input_example(self, prompt, height=None, width=None, guidance_scale=7.5, num_images_per_prompt=1):185        prompt_embeds = None186        negative_prompt_embeds = None187        negative_prompt = None188        callback_steps = 1189        generator = None190        latents = None191 192        # 0. Default height and width to unet193        height = height or self.unet.config.sample_size * self.vae_scale_factor194        width = width or self.unet.config.sample_size * self.vae_scale_factor195 196        # 1. Check inputs. Raise error if not correct197        self.check_inputs(198            prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds199        )200 201        # 2. Define call parameters202        if prompt is not None and isinstance(prompt, str):203            batch_size = 1204        elif prompt is not None and isinstance(prompt, list):205            batch_size = len(prompt)206 207        device = "cpu"208        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)209        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`210        # corresponds to doing no classifier free guidance.211        do_classifier_free_guidance = guidance_scale > 1.0212 213        # 3. Encode input prompt214        prompt_embeds = self._encode_prompt(215            prompt,216            device,217            num_images_per_prompt,218            do_classifier_free_guidance,219            negative_prompt,220            prompt_embeds=prompt_embeds,221            negative_prompt_embeds=negative_prompt_embeds,222        )223 224        # 5. Prepare latent variables225        latents = self.prepare_latents(226            batch_size * num_images_per_prompt,227            self.unet.in_channels,228            height,229            width,230            prompt_embeds.dtype,231            device,232            generator,233            latents,234        )235        dummy = torch.ones(1, dtype=torch.int32)236        latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents237        latent_model_input = self.scheduler.scale_model_input(latent_model_input, dummy)238 239        unet_input_example = (latent_model_input, dummy, prompt_embeds)240        vae_decoder_input_example = latents241 242        return unet_input_example, vae_decoder_input_example243 244    def prepare_for_ipex(self, promt, dtype=torch.float32, height=None, width=None, guidance_scale=7.5):245        self.unet = self.unet.to(memory_format=torch.channels_last)246        self.vae.decoder = self.vae.decoder.to(memory_format=torch.channels_last)247        self.text_encoder = self.text_encoder.to(memory_format=torch.channels_last)248        if self.safety_checker is not None:249            self.safety_checker = self.safety_checker.to(memory_format=torch.channels_last)250 251        unet_input_example, vae_decoder_input_example = self.get_input_example(promt, height, width, guidance_scale)252 253        # optimize with ipex254        if dtype == torch.bfloat16:255            self.unet = ipex.optimize(self.unet.eval(), dtype=torch.bfloat16, inplace=True)256            self.vae.decoder = ipex.optimize(self.vae.decoder.eval(), dtype=torch.bfloat16, inplace=True)257            self.text_encoder = ipex.optimize(self.text_encoder.eval(), dtype=torch.bfloat16, inplace=True)258            if self.safety_checker is not None:259                self.safety_checker = ipex.optimize(self.safety_checker.eval(), dtype=torch.bfloat16, inplace=True)260        elif dtype == torch.float32:261            self.unet = ipex.optimize(262                self.unet.eval(),263                dtype=torch.float32,264                inplace=True,265                weights_prepack=True,266                auto_kernel_selection=False,267            )268            self.vae.decoder = ipex.optimize(269                self.vae.decoder.eval(),270                dtype=torch.float32,271                inplace=True,272                weights_prepack=True,273                auto_kernel_selection=False,274            )275            self.text_encoder = ipex.optimize(276                self.text_encoder.eval(),277                dtype=torch.float32,278                inplace=True,279                weights_prepack=True,280                auto_kernel_selection=False,281            )282            if self.safety_checker is not None:283                self.safety_checker = ipex.optimize(284                    self.safety_checker.eval(),285                    dtype=torch.float32,286                    inplace=True,287                    weights_prepack=True,288                    auto_kernel_selection=False,289                )290        else:291            raise ValueError(" The value of 'dtype' should be 'torch.bfloat16' or 'torch.float32' !")292 293        # trace unet model to get better performance on IPEX294        with torch.cpu.amp.autocast(enabled=dtype == torch.bfloat16), torch.no_grad():295            unet_trace_model = torch.jit.trace(self.unet, unet_input_example, check_trace=False, strict=False)296            unet_trace_model = torch.jit.freeze(unet_trace_model)297        self.unet.forward = unet_trace_model.forward298 299        # trace vae.decoder model to get better performance on IPEX300        with torch.cpu.amp.autocast(enabled=dtype == torch.bfloat16), torch.no_grad():301            ave_decoder_trace_model = torch.jit.trace(302                self.vae.decoder, vae_decoder_input_example, check_trace=False, strict=False303            )304            ave_decoder_trace_model = torch.jit.freeze(ave_decoder_trace_model)305        self.vae.decoder.forward = ave_decoder_trace_model.forward306 307    def enable_vae_slicing(self):308        r"""309        Enable sliced VAE decoding.310 311        When this option is enabled, the VAE will split the input tensor in slices to compute decoding in several312        steps. This is useful to save some memory and allow larger batch sizes.313        """314        self.vae.enable_slicing()315 316    def disable_vae_slicing(self):317        r"""318        Disable sliced VAE decoding. If `enable_vae_slicing` was previously invoked, this method will go back to319        computing decoding in one step.320        """321        self.vae.disable_slicing()322 323    def enable_vae_tiling(self):324        r"""325        Enable tiled VAE decoding.326 327        When this option is enabled, the VAE will split the input tensor into tiles to compute decoding and encoding in328        several steps. This is useful to save a large amount of memory and to allow the processing of larger images.329        """330        self.vae.enable_tiling()331 332    def disable_vae_tiling(self):333        r"""334        Disable tiled VAE decoding. If `enable_vae_tiling` was previously invoked, this method will go back to335        computing decoding in one step.336        """337        self.vae.disable_tiling()338 339    def enable_sequential_cpu_offload(self, gpu_id=0):340        r"""341        Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet,342        text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a343        `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called.344        Note that offloading happens on a submodule basis. Memory savings are higher than with345        `enable_model_cpu_offload`, but performance is lower.346        """347        if is_accelerate_available() and is_accelerate_version(">=", "0.14.0"):348            from accelerate import cpu_offload349        else:350            raise ImportError("`enable_sequential_cpu_offload` requires `accelerate v0.14.0` or higher")351 352        device = torch.device(f"cuda:{gpu_id}")353 354        if self.device.type != "cpu":355            self.to("cpu", silence_dtype_warnings=True)356            torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)357 358        for cpu_offloaded_model in [self.unet, self.text_encoder, self.vae]:359            cpu_offload(cpu_offloaded_model, device)360 361        if self.safety_checker is not None:362            cpu_offload(self.safety_checker, execution_device=device, offload_buffers=True)363 364    def enable_model_cpu_offload(self, gpu_id=0):365        r"""366        Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared367        to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward`368        method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with369        `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`.370        """371        if is_accelerate_available() and is_accelerate_version(">=", "0.17.0.dev0"):372            from accelerate import cpu_offload_with_hook373        else:374            raise ImportError("`enable_model_offload` requires `accelerate v0.17.0` or higher.")375 376        device = torch.device(f"cuda:{gpu_id}")377 378        if self.device.type != "cpu":379            self.to("cpu", silence_dtype_warnings=True)380            torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)381 382        hook = None383        for cpu_offloaded_model in [self.text_encoder, self.unet, self.vae]:384            _, hook = cpu_offload_with_hook(cpu_offloaded_model, device, prev_module_hook=hook)385 386        if self.safety_checker is not None:387            _, hook = cpu_offload_with_hook(self.safety_checker, device, prev_module_hook=hook)388 389        # We'll offload the last model manually.390        self.final_offload_hook = hook391 392    @property393    def _execution_device(self):394        r"""395        Returns the device on which the pipeline's models will be executed. After calling396        `pipeline.enable_sequential_cpu_offload()` the execution device can only be inferred from Accelerate's module397        hooks.398        """399        if not hasattr(self.unet, "_hf_hook"):400            return self.device401        for module in self.unet.modules():402            if (403                hasattr(module, "_hf_hook")404                and hasattr(module._hf_hook, "execution_device")405                and module._hf_hook.execution_device is not None406            ):407                return torch.device(module._hf_hook.execution_device)408        return self.device409 410    def _encode_prompt(411        self,412        prompt,413        device,414        num_images_per_prompt,415        do_classifier_free_guidance,416        negative_prompt=None,417        prompt_embeds: Optional[torch.FloatTensor] = None,418        negative_prompt_embeds: Optional[torch.FloatTensor] = None,419    ):420        r"""421        Encodes the prompt into text encoder hidden states.422 423        Args:424             prompt (`str` or `List[str]`, *optional*):425                prompt to be encoded426            device: (`torch.device`):427                torch device428            num_images_per_prompt (`int`):429                number of images that should be generated per prompt430            do_classifier_free_guidance (`bool`):431                whether to use classifier free guidance or not432            negative_prompt (`str` or `List[str]`, *optional*):433                The prompt or prompts not to guide the image generation. If not defined, one has to pass434                `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.435                Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).436            prompt_embeds (`torch.FloatTensor`, *optional*):437                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not438                provided, text embeddings will be generated from `prompt` input argument.439            negative_prompt_embeds (`torch.FloatTensor`, *optional*):440                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt441                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input442                argument.443        """444        if prompt is not None and isinstance(prompt, str):445            batch_size = 1446        elif prompt is not None and isinstance(prompt, list):447            batch_size = len(prompt)448        else:449            batch_size = prompt_embeds.shape[0]450 451        if prompt_embeds is None:452            # textual inversion: procecss multi-vector tokens if necessary453            if isinstance(self, TextualInversionLoaderMixin):454                prompt = self.maybe_convert_prompt(prompt, self.tokenizer)455 456            text_inputs = self.tokenizer(457                prompt,458                padding="max_length",459                max_length=self.tokenizer.model_max_length,460                truncation=True,461                return_tensors="pt",462            )463            text_input_ids = text_inputs.input_ids464            untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids465 466            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(467                text_input_ids, untruncated_ids468            ):469                removed_text = self.tokenizer.batch_decode(470                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]471                )472                logger.warning(473                    "The following part of your input was truncated because CLIP can only handle sequences up to"474                    f" {self.tokenizer.model_max_length} tokens: {removed_text}"475                )476 477            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:478                attention_mask = text_inputs.attention_mask.to(device)479            else:480                attention_mask = None481 482            prompt_embeds = self.text_encoder(483                text_input_ids.to(device),484                attention_mask=attention_mask,485            )486            prompt_embeds = prompt_embeds[0]487 488        prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)489 490        bs_embed, seq_len, _ = prompt_embeds.shape491        # duplicate text embeddings for each generation per prompt, using mps friendly method492        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)493        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)494 495        # get unconditional embeddings for classifier free guidance496        if do_classifier_free_guidance and negative_prompt_embeds is None:497            uncond_tokens: List[str]498            if negative_prompt is None:499                uncond_tokens = [""] * batch_size500            elif type(prompt) is not type(negative_prompt):501                raise TypeError(502                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="503                    f" {type(prompt)}."504                )505            elif isinstance(negative_prompt, str):506                uncond_tokens = [negative_prompt]507            elif batch_size != len(negative_prompt):508                raise ValueError(509                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"510                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"511                    " the batch size of `prompt`."512                )513            else:514                uncond_tokens = negative_prompt515 516            # textual inversion: procecss multi-vector tokens if necessary517            if isinstance(self, TextualInversionLoaderMixin):518                uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer)519 520            max_length = prompt_embeds.shape[1]521            uncond_input = self.tokenizer(522                uncond_tokens,523                padding="max_length",524                max_length=max_length,525                truncation=True,526                return_tensors="pt",527            )528 529            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:530                attention_mask = uncond_input.attention_mask.to(device)531            else:532                attention_mask = None533 534            negative_prompt_embeds = self.text_encoder(535                uncond_input.input_ids.to(device),536                attention_mask=attention_mask,537            )538            negative_prompt_embeds = negative_prompt_embeds[0]539 540        if do_classifier_free_guidance:541            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method542            seq_len = negative_prompt_embeds.shape[1]543 544            negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)545 546            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)547            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)548 549            # For classifier free guidance, we need to do two forward passes.550            # Here we concatenate the unconditional and text embeddings into a single batch551            # to avoid doing two forward passes552            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])553 554        return prompt_embeds555 556    def run_safety_checker(self, image, device, dtype):557        if self.safety_checker is not None:558            safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)559            image, has_nsfw_concept = self.safety_checker(560                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)561            )562        else:563            has_nsfw_concept = None564        return image, has_nsfw_concept565 566    def decode_latents(self, latents):567        latents = 1 / self.vae.config.scaling_factor * latents568        image = self.vae.decode(latents).sample569        image = (image / 2 + 0.5).clamp(0, 1)570        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16571        image = image.cpu().permute(0, 2, 3, 1).float().numpy()572        return image573 574    def prepare_extra_step_kwargs(self, generator, eta):575        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature576        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.577        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502578        # and should be between [0, 1]579 580        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())581        extra_step_kwargs = {}582        if accepts_eta:583            extra_step_kwargs["eta"] = eta584 585        # check if the scheduler accepts generator586        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())587        if accepts_generator:588            extra_step_kwargs["generator"] = generator589        return extra_step_kwargs590 591    def check_inputs(592        self,593        prompt,594        height,595        width,596        callback_steps,597        negative_prompt=None,598        prompt_embeds=None,599        negative_prompt_embeds=None,600    ):601        if height % 8 != 0 or width % 8 != 0:602            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")603 604        if (callback_steps is None) or (605            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)606        ):607            raise ValueError(608                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"609                f" {type(callback_steps)}."610            )611 612        if prompt is not None and prompt_embeds is not None:613            raise ValueError(614                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"615                " only forward one of the two."616            )617        elif prompt is None and prompt_embeds is None:618            raise ValueError(619                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."620            )621        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):622            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")623 624        if negative_prompt is not None and negative_prompt_embeds is not None:625            raise ValueError(626                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"627                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."628            )629 630        if prompt_embeds is not None and negative_prompt_embeds is not None:631            if prompt_embeds.shape != negative_prompt_embeds.shape:632                raise ValueError(633                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"634                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"635                    f" {negative_prompt_embeds.shape}."636                )637 638    def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):639        shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor)640        if isinstance(generator, list) and len(generator) != batch_size:641            raise ValueError(642                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"643                f" size of {batch_size}. Make sure the batch size matches the length of the generators."644            )645 646        if latents is None:647            latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)648        else:649            latents = latents.to(device)650 651        # scale the initial noise by the standard deviation required by the scheduler652        latents = latents * self.scheduler.init_noise_sigma653        return latents654 655    @torch.no_grad()656    @replace_example_docstring(EXAMPLE_DOC_STRING)657    def __call__(658        self,659        prompt: Union[str, List[str]] = None,660        height: Optional[int] = None,661        width: Optional[int] = None,662        num_inference_steps: int = 50,663        guidance_scale: float = 7.5,664        negative_prompt: Optional[Union[str, List[str]]] = None,665        num_images_per_prompt: Optional[int] = 1,666        eta: float = 0.0,667        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,668        latents: Optional[torch.FloatTensor] = None,669        prompt_embeds: Optional[torch.FloatTensor] = None,670        negative_prompt_embeds: Optional[torch.FloatTensor] = None,671        output_type: Optional[str] = "pil",672        return_dict: bool = True,673        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,674        callback_steps: int = 1,675        cross_attention_kwargs: Optional[Dict[str, Any]] = None,676    ):677        r"""678        Function invoked when calling the pipeline for generation.679 680        Args:681            prompt (`str` or `List[str]`, *optional*):682                The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.683                instead.684            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):685                The height in pixels of the generated image.686            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):687                The width in pixels of the generated image.688            num_inference_steps (`int`, *optional*, defaults to 50):689                The number of denoising steps. More denoising steps usually lead to a higher quality image at the690                expense of slower inference.691            guidance_scale (`float`, *optional*, defaults to 7.5):692                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).693                `guidance_scale` is defined as `w` of equation 2. of [Imagen694                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >695                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,696                usually at the expense of lower image quality.697            negative_prompt (`str` or `List[str]`, *optional*):698                The prompt or prompts not to guide the image generation. If not defined, one has to pass699                `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.700                Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).701            num_images_per_prompt (`int`, *optional*, defaults to 1):702                The number of images to generate per prompt.703            eta (`float`, *optional*, defaults to 0.0):704                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to705                [`schedulers.DDIMScheduler`], will be ignored for others.706            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):707                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)708                to make generation deterministic.709            latents (`torch.FloatTensor`, *optional*):710                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image711                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents712                tensor will ge generated by sampling using the supplied random `generator`.713            prompt_embeds (`torch.FloatTensor`, *optional*):714                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not715                provided, text embeddings will be generated from `prompt` input argument.716            negative_prompt_embeds (`torch.FloatTensor`, *optional*):717                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt718                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input719                argument.720            output_type (`str`, *optional*, defaults to `"pil"`):721                The output format of the generate image. Choose between722                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.723            return_dict (`bool`, *optional*, defaults to `True`):724                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a725                plain tuple.726            callback (`Callable`, *optional*):727                A function that will be called every `callback_steps` steps during inference. The function will be728                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.729            callback_steps (`int`, *optional*, defaults to 1):730                The frequency at which the `callback` function will be called. If not specified, the callback will be731                called at every step.732            cross_attention_kwargs (`dict`, *optional*):733                A kwargs dictionary that if specified is passed along to the `AttnProcessor` as defined under734                `self.processor` in735                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).736 737        Examples:738 739        Returns:740            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:741            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.742            When returning a tuple, the first element is a list with the generated images, and the second element is a743            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"744            (nsfw) content, according to the `safety_checker`.745        """746        # 0. Default height and width to unet747        height = height or self.unet.config.sample_size * self.vae_scale_factor748        width = width or self.unet.config.sample_size * self.vae_scale_factor749 750        # 1. Check inputs. Raise error if not correct751        self.check_inputs(752            prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds753        )754 755        # 2. Define call parameters756        if prompt is not None and isinstance(prompt, str):757            batch_size = 1758        elif prompt is not None and isinstance(prompt, list):759            batch_size = len(prompt)760        else:761            batch_size = prompt_embeds.shape[0]762 763        device = self._execution_device764        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)765        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`766        # corresponds to doing no classifier free guidance.767        do_classifier_free_guidance = guidance_scale > 1.0768 769        # 3. Encode input prompt770        prompt_embeds = self._encode_prompt(771            prompt,772            device,773            num_images_per_prompt,774            do_classifier_free_guidance,775            negative_prompt,776            prompt_embeds=prompt_embeds,777            negative_prompt_embeds=negative_prompt_embeds,778        )779 780        # 4. Prepare timesteps781        self.scheduler.set_timesteps(num_inference_steps, device=device)782        timesteps = self.scheduler.timesteps783 784        # 5. Prepare latent variables785        num_channels_latents = self.unet.in_channels786        latents = self.prepare_latents(787            batch_size * num_images_per_prompt,788            num_channels_latents,789            height,790            width,791            prompt_embeds.dtype,792            device,793            generator,794            latents,795        )796 797        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline798        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)799 800        # 7. Denoising loop801        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order802        with self.progress_bar(total=num_inference_steps) as progress_bar:803            for i, t in enumerate(timesteps):804                # expand the latents if we are doing classifier free guidance805                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents806                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)807 808                # predict the noise residual809                noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=prompt_embeds)["sample"]810 811                # perform guidance812                if do_classifier_free_guidance:813                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)814                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)815 816                # compute the previous noisy sample x_t -> x_t-1817                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample818 819                # call the callback, if provided820                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):821                    progress_bar.update()822                    if callback is not None and i % callback_steps == 0:823                        step_idx = i // getattr(self.scheduler, "order", 1)824                        callback(step_idx, t, latents)825 826        if output_type == "latent":827            image = latents828            has_nsfw_concept = None829        elif output_type == "pil":830            # 8. Post-processing831            image = self.decode_latents(latents)832 833            # 9. Run safety checker834            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)835 836            # 10. Convert to PIL837            image = self.numpy_to_pil(image)838        else:839            # 8. Post-processing840            image = self.decode_latents(latents)841 842            # 9. Run safety checker843            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)844 845        # Offload last model to CPU846        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:847            self.final_offload_hook.offload()848 849        if not return_dict:850            return (image, has_nsfw_concept)851 852        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)853