diffusers/community-pipelines-mirror

1# Copyright 2023 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import inspect16from typing import Any, Callable, Dict, List, Optional, Union17 18import intel_extension_for_pytorch as ipex19import torch20from packaging import version21from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer22 23from diffusers.configuration_utils import FrozenDict24from diffusers.loaders import TextualInversionLoaderMixin25from diffusers.models import AutoencoderKL, UNet2DConditionModel26from diffusers.pipeline_utils import DiffusionPipeline27from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput28from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker29from diffusers.schedulers import KarrasDiffusionSchedulers30from diffusers.utils import (31    deprecate,32    is_accelerate_available,33    is_accelerate_version,34    logging,35    replace_example_docstring,36)37from diffusers.utils.torch_utils import randn_tensor38 39 40logger = logging.get_logger(__name__)  # pylint: disable=invalid-name41 42EXAMPLE_DOC_STRING = """43    Examples:44        ```py45        >>> import torch46        >>> from diffusers import StableDiffusionPipeline47 48        >>> pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", custom_pipeline="stable_diffusion_ipex")49 50        >>> # For Float3251        >>> pipe.prepare_for_ipex(prompt, dtype=torch.float32, height=512, width=512) #value of image height/width should be consistent with the pipeline inference52        >>> # For BFloat1653        >>> pipe.prepare_for_ipex(prompt, dtype=torch.bfloat16, height=512, width=512) #value of image height/width should be consistent with the pipeline inference54 55        >>> prompt = "a photo of an astronaut riding a horse on mars"56        >>> # For Float3257        >>> image = pipe(prompt, num_inference_steps=num_inference_steps, height=512, width=512).images[0] #value of image height/width should be consistent with 'prepare_for_ipex()'58        >>> # For BFloat1659        >>> with torch.cpu.amp.autocast(enabled=True, dtype=torch.bfloat16):60        >>>     image = pipe(prompt, num_inference_steps=num_inference_steps, height=512, width=512).images[0] #value of image height/width should be consistent with 'prepare_for_ipex()'61        ```62"""63 64 65class StableDiffusionIPEXPipeline(DiffusionPipeline, TextualInversionLoaderMixin):66    r"""67    Pipeline for text-to-image generation using Stable Diffusion on IPEX.68 69    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the70    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)71 72    Args:73        vae ([`AutoencoderKL`]):74            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.75        text_encoder ([`CLIPTextModel`]):76            Frozen text-encoder. Stable Diffusion uses the text portion of77            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically78            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.79        tokenizer (`CLIPTokenizer`):80            Tokenizer of class81            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).82        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.83        scheduler ([`SchedulerMixin`]):84            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of85            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].86        safety_checker ([`StableDiffusionSafetyChecker`]):87            Classification module that estimates whether generated images could be considered offensive or harmful.88            Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details.89        feature_extractor ([`CLIPFeatureExtractor`]):90            Model that extracts features from generated images to be used as inputs for the `safety_checker`.91    """92    _optional_components = ["safety_checker", "feature_extractor"]93 94    def __init__(95        self,96        vae: AutoencoderKL,97        text_encoder: CLIPTextModel,98        tokenizer: CLIPTokenizer,99        unet: UNet2DConditionModel,100        scheduler: KarrasDiffusionSchedulers,101        safety_checker: StableDiffusionSafetyChecker,102        feature_extractor: CLIPFeatureExtractor,103        requires_safety_checker: bool = True,104    ):105        super().__init__()106 107        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:108            deprecation_message = (109                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"110                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "111                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"112                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"113                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"114                " file"115            )116            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)117            new_config = dict(scheduler.config)118            new_config["steps_offset"] = 1119            scheduler._internal_dict = FrozenDict(new_config)120 121        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:122            deprecation_message = (123                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."124                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"125                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"126                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"127                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"128            )129            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)130            new_config = dict(scheduler.config)131            new_config["clip_sample"] = False132            scheduler._internal_dict = FrozenDict(new_config)133 134        if safety_checker is None and requires_safety_checker:135            logger.warning(136                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"137                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"138                " results in services or applications open to the public. Both the diffusers team and Hugging Face"139                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"140                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"141                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."142            )143 144        if safety_checker is not None and feature_extractor is None:145            raise ValueError(146                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"147                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."148            )149 150        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(151            version.parse(unet.config._diffusers_version).base_version152        ) < version.parse("0.9.0.dev0")153        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64154        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:155            deprecation_message = (156                "The configuration file of the unet has set the default `sample_size` to smaller than"157                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"158                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"159                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"160                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"161                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"162                " in the config might lead to incorrect results in future versions. If you have downloaded this"163                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"164                " the `unet/config.json` file"165            )166            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)167            new_config = dict(unet.config)168            new_config["sample_size"] = 64169            unet._internal_dict = FrozenDict(new_config)170 171        self.register_modules(172            vae=vae,173            text_encoder=text_encoder,174            tokenizer=tokenizer,175            unet=unet,176            scheduler=scheduler,177            safety_checker=safety_checker,178            feature_extractor=feature_extractor,179        )180        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)181        self.register_to_config(requires_safety_checker=requires_safety_checker)182 183    def get_input_example(self, prompt, height=None, width=None, guidance_scale=7.5, num_images_per_prompt=1):184        prompt_embeds = None185        negative_prompt_embeds = None186        negative_prompt = None187        callback_steps = 1188        generator = None189        latents = None190 191        # 0. Default height and width to unet192        height = height or self.unet.config.sample_size * self.vae_scale_factor193        width = width or self.unet.config.sample_size * self.vae_scale_factor194 195        # 1. Check inputs. Raise error if not correct196        self.check_inputs(197            prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds198        )199 200        # 2. Define call parameters201        if prompt is not None and isinstance(prompt, str):202            batch_size = 1203        elif prompt is not None and isinstance(prompt, list):204            batch_size = len(prompt)205 206        device = "cpu"207        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)208        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`209        # corresponds to doing no classifier free guidance.210        do_classifier_free_guidance = guidance_scale > 1.0211 212        # 3. Encode input prompt213        prompt_embeds = self._encode_prompt(214            prompt,215            device,216            num_images_per_prompt,217            do_classifier_free_guidance,218            negative_prompt,219            prompt_embeds=prompt_embeds,220            negative_prompt_embeds=negative_prompt_embeds,221        )222 223        # 5. Prepare latent variables224        latents = self.prepare_latents(225            batch_size * num_images_per_prompt,226            self.unet.in_channels,227            height,228            width,229            prompt_embeds.dtype,230            device,231            generator,232            latents,233        )234        dummy = torch.ones(1, dtype=torch.int32)235        latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents236        latent_model_input = self.scheduler.scale_model_input(latent_model_input, dummy)237 238        unet_input_example = (latent_model_input, dummy, prompt_embeds)239        vae_decoder_input_example = latents240 241        return unet_input_example, vae_decoder_input_example242 243    def prepare_for_ipex(self, promt, dtype=torch.float32, height=None, width=None, guidance_scale=7.5):244        self.unet = self.unet.to(memory_format=torch.channels_last)245        self.vae.decoder = self.vae.decoder.to(memory_format=torch.channels_last)246        self.text_encoder = self.text_encoder.to(memory_format=torch.channels_last)247        if self.safety_checker is not None:248            self.safety_checker = self.safety_checker.to(memory_format=torch.channels_last)249 250        unet_input_example, vae_decoder_input_example = self.get_input_example(promt, height, width, guidance_scale)251 252        # optimize with ipex253        if dtype == torch.bfloat16:254            self.unet = ipex.optimize(255                self.unet.eval(), dtype=torch.bfloat16, inplace=True, sample_input=unet_input_example256            )257            self.vae.decoder = ipex.optimize(self.vae.decoder.eval(), dtype=torch.bfloat16, inplace=True)258            self.text_encoder = ipex.optimize(self.text_encoder.eval(), dtype=torch.bfloat16, inplace=True)259            if self.safety_checker is not None:260                self.safety_checker = ipex.optimize(self.safety_checker.eval(), dtype=torch.bfloat16, inplace=True)261        elif dtype == torch.float32:262            self.unet = ipex.optimize(263                self.unet.eval(),264                dtype=torch.float32,265                inplace=True,266                sample_input=unet_input_example,267                level="O1",268                weights_prepack=True,269                auto_kernel_selection=False,270            )271            self.vae.decoder = ipex.optimize(272                self.vae.decoder.eval(),273                dtype=torch.float32,274                inplace=True,275                level="O1",276                weights_prepack=True,277                auto_kernel_selection=False,278            )279            self.text_encoder = ipex.optimize(280                self.text_encoder.eval(),281                dtype=torch.float32,282                inplace=True,283                level="O1",284                weights_prepack=True,285                auto_kernel_selection=False,286            )287            if self.safety_checker is not None:288                self.safety_checker = ipex.optimize(289                    self.safety_checker.eval(),290                    dtype=torch.float32,291                    inplace=True,292                    level="O1",293                    weights_prepack=True,294                    auto_kernel_selection=False,295                )296        else:297            raise ValueError(" The value of 'dtype' should be 'torch.bfloat16' or 'torch.float32' !")298 299        # trace unet model to get better performance on IPEX300        with torch.cpu.amp.autocast(enabled=dtype == torch.bfloat16), torch.no_grad():301            unet_trace_model = torch.jit.trace(self.unet, unet_input_example, check_trace=False, strict=False)302            unet_trace_model = torch.jit.freeze(unet_trace_model)303        self.unet.forward = unet_trace_model.forward304 305        # trace vae.decoder model to get better performance on IPEX306        with torch.cpu.amp.autocast(enabled=dtype == torch.bfloat16), torch.no_grad():307            ave_decoder_trace_model = torch.jit.trace(308                self.vae.decoder, vae_decoder_input_example, check_trace=False, strict=False309            )310            ave_decoder_trace_model = torch.jit.freeze(ave_decoder_trace_model)311        self.vae.decoder.forward = ave_decoder_trace_model.forward312 313    def enable_vae_slicing(self):314        r"""315        Enable sliced VAE decoding.316 317        When this option is enabled, the VAE will split the input tensor in slices to compute decoding in several318        steps. This is useful to save some memory and allow larger batch sizes.319        """320        self.vae.enable_slicing()321 322    def disable_vae_slicing(self):323        r"""324        Disable sliced VAE decoding. If `enable_vae_slicing` was previously invoked, this method will go back to325        computing decoding in one step.326        """327        self.vae.disable_slicing()328 329    def enable_vae_tiling(self):330        r"""331        Enable tiled VAE decoding.332 333        When this option is enabled, the VAE will split the input tensor into tiles to compute decoding and encoding in334        several steps. This is useful to save a large amount of memory and to allow the processing of larger images.335        """336        self.vae.enable_tiling()337 338    def disable_vae_tiling(self):339        r"""340        Disable tiled VAE decoding. If `enable_vae_tiling` was previously invoked, this method will go back to341        computing decoding in one step.342        """343        self.vae.disable_tiling()344 345    def enable_sequential_cpu_offload(self, gpu_id=0):346        r"""347        Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet,348        text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a349        `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called.350        Note that offloading happens on a submodule basis. Memory savings are higher than with351        `enable_model_cpu_offload`, but performance is lower.352        """353        if is_accelerate_available() and is_accelerate_version(">=", "0.14.0"):354            from accelerate import cpu_offload355        else:356            raise ImportError("`enable_sequential_cpu_offload` requires `accelerate v0.14.0` or higher")357 358        device = torch.device(f"cuda:{gpu_id}")359 360        if self.device.type != "cpu":361            self.to("cpu", silence_dtype_warnings=True)362            torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)363 364        for cpu_offloaded_model in [self.unet, self.text_encoder, self.vae]:365            cpu_offload(cpu_offloaded_model, device)366 367        if self.safety_checker is not None:368            cpu_offload(self.safety_checker, execution_device=device, offload_buffers=True)369 370    def enable_model_cpu_offload(self, gpu_id=0):371        r"""372        Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared373        to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward`374        method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with375        `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`.376        """377        if is_accelerate_available() and is_accelerate_version(">=", "0.17.0.dev0"):378            from accelerate import cpu_offload_with_hook379        else:380            raise ImportError("`enable_model_offload` requires `accelerate v0.17.0` or higher.")381 382        device = torch.device(f"cuda:{gpu_id}")383 384        if self.device.type != "cpu":385            self.to("cpu", silence_dtype_warnings=True)386            torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)387 388        hook = None389        for cpu_offloaded_model in [self.text_encoder, self.unet, self.vae]:390            _, hook = cpu_offload_with_hook(cpu_offloaded_model, device, prev_module_hook=hook)391 392        if self.safety_checker is not None:393            _, hook = cpu_offload_with_hook(self.safety_checker, device, prev_module_hook=hook)394 395        # We'll offload the last model manually.396        self.final_offload_hook = hook397 398    @property399    def _execution_device(self):400        r"""401        Returns the device on which the pipeline's models will be executed. After calling402        `pipeline.enable_sequential_cpu_offload()` the execution device can only be inferred from Accelerate's module403        hooks.404        """405        if not hasattr(self.unet, "_hf_hook"):406            return self.device407        for module in self.unet.modules():408            if (409                hasattr(module, "_hf_hook")410                and hasattr(module._hf_hook, "execution_device")411                and module._hf_hook.execution_device is not None412            ):413                return torch.device(module._hf_hook.execution_device)414        return self.device415 416    def _encode_prompt(417        self,418        prompt,419        device,420        num_images_per_prompt,421        do_classifier_free_guidance,422        negative_prompt=None,423        prompt_embeds: Optional[torch.FloatTensor] = None,424        negative_prompt_embeds: Optional[torch.FloatTensor] = None,425    ):426        r"""427        Encodes the prompt into text encoder hidden states.428 429        Args:430             prompt (`str` or `List[str]`, *optional*):431                prompt to be encoded432            device: (`torch.device`):433                torch device434            num_images_per_prompt (`int`):435                number of images that should be generated per prompt436            do_classifier_free_guidance (`bool`):437                whether to use classifier free guidance or not438            negative_prompt (`str` or `List[str]`, *optional*):439                The prompt or prompts not to guide the image generation. If not defined, one has to pass440                `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.441                Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).442            prompt_embeds (`torch.FloatTensor`, *optional*):443                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not444                provided, text embeddings will be generated from `prompt` input argument.445            negative_prompt_embeds (`torch.FloatTensor`, *optional*):446                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt447                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input448                argument.449        """450        if prompt is not None and isinstance(prompt, str):451            batch_size = 1452        elif prompt is not None and isinstance(prompt, list):453            batch_size = len(prompt)454        else:455            batch_size = prompt_embeds.shape[0]456 457        if prompt_embeds is None:458            # textual inversion: procecss multi-vector tokens if necessary459            if isinstance(self, TextualInversionLoaderMixin):460                prompt = self.maybe_convert_prompt(prompt, self.tokenizer)461 462            text_inputs = self.tokenizer(463                prompt,464                padding="max_length",465                max_length=self.tokenizer.model_max_length,466                truncation=True,467                return_tensors="pt",468            )469            text_input_ids = text_inputs.input_ids470            untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids471 472            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(473                text_input_ids, untruncated_ids474            ):475                removed_text = self.tokenizer.batch_decode(476                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]477                )478                logger.warning(479                    "The following part of your input was truncated because CLIP can only handle sequences up to"480                    f" {self.tokenizer.model_max_length} tokens: {removed_text}"481                )482 483            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:484                attention_mask = text_inputs.attention_mask.to(device)485            else:486                attention_mask = None487 488            prompt_embeds = self.text_encoder(489                text_input_ids.to(device),490                attention_mask=attention_mask,491            )492            prompt_embeds = prompt_embeds[0]493 494        prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)495 496        bs_embed, seq_len, _ = prompt_embeds.shape497        # duplicate text embeddings for each generation per prompt, using mps friendly method498        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)499        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)500 501        # get unconditional embeddings for classifier free guidance502        if do_classifier_free_guidance and negative_prompt_embeds is None:503            uncond_tokens: List[str]504            if negative_prompt is None:505                uncond_tokens = [""] * batch_size506            elif type(prompt) is not type(negative_prompt):507                raise TypeError(508                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="509                    f" {type(prompt)}."510                )511            elif isinstance(negative_prompt, str):512                uncond_tokens = [negative_prompt]513            elif batch_size != len(negative_prompt):514                raise ValueError(515                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"516                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"517                    " the batch size of `prompt`."518                )519            else:520                uncond_tokens = negative_prompt521 522            # textual inversion: procecss multi-vector tokens if necessary523            if isinstance(self, TextualInversionLoaderMixin):524                uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer)525 526            max_length = prompt_embeds.shape[1]527            uncond_input = self.tokenizer(528                uncond_tokens,529                padding="max_length",530                max_length=max_length,531                truncation=True,532                return_tensors="pt",533            )534 535            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:536                attention_mask = uncond_input.attention_mask.to(device)537            else:538                attention_mask = None539 540            negative_prompt_embeds = self.text_encoder(541                uncond_input.input_ids.to(device),542                attention_mask=attention_mask,543            )544            negative_prompt_embeds = negative_prompt_embeds[0]545 546        if do_classifier_free_guidance:547            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method548            seq_len = negative_prompt_embeds.shape[1]549 550            negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)551 552            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)553            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)554 555            # For classifier free guidance, we need to do two forward passes.556            # Here we concatenate the unconditional and text embeddings into a single batch557            # to avoid doing two forward passes558            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])559 560        return prompt_embeds561 562    def run_safety_checker(self, image, device, dtype):563        if self.safety_checker is not None:564            safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)565            image, has_nsfw_concept = self.safety_checker(566                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)567            )568        else:569            has_nsfw_concept = None570        return image, has_nsfw_concept571 572    def decode_latents(self, latents):573        latents = 1 / self.vae.config.scaling_factor * latents574        image = self.vae.decode(latents).sample575        image = (image / 2 + 0.5).clamp(0, 1)576        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16577        image = image.cpu().permute(0, 2, 3, 1).float().numpy()578        return image579 580    def prepare_extra_step_kwargs(self, generator, eta):581        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature582        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.583        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502584        # and should be between [0, 1]585 586        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())587        extra_step_kwargs = {}588        if accepts_eta:589            extra_step_kwargs["eta"] = eta590 591        # check if the scheduler accepts generator592        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())593        if accepts_generator:594            extra_step_kwargs["generator"] = generator595        return extra_step_kwargs596 597    def check_inputs(598        self,599        prompt,600        height,601        width,602        callback_steps,603        negative_prompt=None,604        prompt_embeds=None,605        negative_prompt_embeds=None,606    ):607        if height % 8 != 0 or width % 8 != 0:608            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")609 610        if (callback_steps is None) or (611            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)612        ):613            raise ValueError(614                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"615                f" {type(callback_steps)}."616            )617 618        if prompt is not None and prompt_embeds is not None:619            raise ValueError(620                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"621                " only forward one of the two."622            )623        elif prompt is None and prompt_embeds is None:624            raise ValueError(625                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."626            )627        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):628            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")629 630        if negative_prompt is not None and negative_prompt_embeds is not None:631            raise ValueError(632                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"633                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."634            )635 636        if prompt_embeds is not None and negative_prompt_embeds is not None:637            if prompt_embeds.shape != negative_prompt_embeds.shape:638                raise ValueError(639                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"640                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"641                    f" {negative_prompt_embeds.shape}."642                )643 644    def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):645        shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor)646        if isinstance(generator, list) and len(generator) != batch_size:647            raise ValueError(648                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"649                f" size of {batch_size}. Make sure the batch size matches the length of the generators."650            )651 652        if latents is None:653            latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)654        else:655            latents = latents.to(device)656 657        # scale the initial noise by the standard deviation required by the scheduler658        latents = latents * self.scheduler.init_noise_sigma659        return latents660 661    @torch.no_grad()662    @replace_example_docstring(EXAMPLE_DOC_STRING)663    def __call__(664        self,665        prompt: Union[str, List[str]] = None,666        height: Optional[int] = None,667        width: Optional[int] = None,668        num_inference_steps: int = 50,669        guidance_scale: float = 7.5,670        negative_prompt: Optional[Union[str, List[str]]] = None,671        num_images_per_prompt: Optional[int] = 1,672        eta: float = 0.0,673        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,674        latents: Optional[torch.FloatTensor] = None,675        prompt_embeds: Optional[torch.FloatTensor] = None,676        negative_prompt_embeds: Optional[torch.FloatTensor] = None,677        output_type: Optional[str] = "pil",678        return_dict: bool = True,679        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,680        callback_steps: int = 1,681        cross_attention_kwargs: Optional[Dict[str, Any]] = None,682    ):683        r"""684        Function invoked when calling the pipeline for generation.685 686        Args:687            prompt (`str` or `List[str]`, *optional*):688                The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.689                instead.690            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):691                The height in pixels of the generated image.692            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):693                The width in pixels of the generated image.694            num_inference_steps (`int`, *optional*, defaults to 50):695                The number of denoising steps. More denoising steps usually lead to a higher quality image at the696                expense of slower inference.697            guidance_scale (`float`, *optional*, defaults to 7.5):698                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).699                `guidance_scale` is defined as `w` of equation 2. of [Imagen700                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >701                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,702                usually at the expense of lower image quality.703            negative_prompt (`str` or `List[str]`, *optional*):704                The prompt or prompts not to guide the image generation. If not defined, one has to pass705                `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.706                Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).707            num_images_per_prompt (`int`, *optional*, defaults to 1):708                The number of images to generate per prompt.709            eta (`float`, *optional*, defaults to 0.0):710                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to711                [`schedulers.DDIMScheduler`], will be ignored for others.712            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):713                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)714                to make generation deterministic.715            latents (`torch.FloatTensor`, *optional*):716                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image717                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents718                tensor will ge generated by sampling using the supplied random `generator`.719            prompt_embeds (`torch.FloatTensor`, *optional*):720                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not721                provided, text embeddings will be generated from `prompt` input argument.722            negative_prompt_embeds (`torch.FloatTensor`, *optional*):723                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt724                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input725                argument.726            output_type (`str`, *optional*, defaults to `"pil"`):727                The output format of the generate image. Choose between728                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.729            return_dict (`bool`, *optional*, defaults to `True`):730                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a731                plain tuple.732            callback (`Callable`, *optional*):733                A function that will be called every `callback_steps` steps during inference. The function will be734                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.735            callback_steps (`int`, *optional*, defaults to 1):736                The frequency at which the `callback` function will be called. If not specified, the callback will be737                called at every step.738            cross_attention_kwargs (`dict`, *optional*):739                A kwargs dictionary that if specified is passed along to the `AttnProcessor` as defined under740                `self.processor` in741                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).742 743        Examples:744 745        Returns:746            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:747            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.748            When returning a tuple, the first element is a list with the generated images, and the second element is a749            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"750            (nsfw) content, according to the `safety_checker`.751        """752        # 0. Default height and width to unet753        height = height or self.unet.config.sample_size * self.vae_scale_factor754        width = width or self.unet.config.sample_size * self.vae_scale_factor755 756        # 1. Check inputs. Raise error if not correct757        self.check_inputs(758            prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds759        )760 761        # 2. Define call parameters762        if prompt is not None and isinstance(prompt, str):763            batch_size = 1764        elif prompt is not None and isinstance(prompt, list):765            batch_size = len(prompt)766        else:767            batch_size = prompt_embeds.shape[0]768 769        device = self._execution_device770        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)771        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`772        # corresponds to doing no classifier free guidance.773        do_classifier_free_guidance = guidance_scale > 1.0774 775        # 3. Encode input prompt776        prompt_embeds = self._encode_prompt(777            prompt,778            device,779            num_images_per_prompt,780            do_classifier_free_guidance,781            negative_prompt,782            prompt_embeds=prompt_embeds,783            negative_prompt_embeds=negative_prompt_embeds,784        )785 786        # 4. Prepare timesteps787        self.scheduler.set_timesteps(num_inference_steps, device=device)788        timesteps = self.scheduler.timesteps789 790        # 5. Prepare latent variables791        num_channels_latents = self.unet.in_channels792        latents = self.prepare_latents(793            batch_size * num_images_per_prompt,794            num_channels_latents,795            height,796            width,797            prompt_embeds.dtype,798            device,799            generator,800            latents,801        )802 803        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline804        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)805 806        # 7. Denoising loop807        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order808        with self.progress_bar(total=num_inference_steps) as progress_bar:809            for i, t in enumerate(timesteps):810                # expand the latents if we are doing classifier free guidance811                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents812                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)813 814                # predict the noise residual815                noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=prompt_embeds)["sample"]816 817                # perform guidance818                if do_classifier_free_guidance:819                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)820                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)821 822                # compute the previous noisy sample x_t -> x_t-1823                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample824 825                # call the callback, if provided826                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):827                    progress_bar.update()828                    if callback is not None and i % callback_steps == 0:829                        step_idx = i // getattr(self.scheduler, "order", 1)830                        callback(step_idx, t, latents)831 832        if output_type == "latent":833            image = latents834            has_nsfw_concept = None835        elif output_type == "pil":836            # 8. Post-processing837            image = self.decode_latents(latents)838 839            # 9. Run safety checker840            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)841 842            # 10. Convert to PIL843            image = self.numpy_to_pil(image)844        else:845            # 8. Post-processing846            image = self.decode_latents(latents)847 848            # 9. Run safety checker849            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)850 851        # Offload last model to CPU852        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:853            self.final_offload_hook.offload()854 855        if not return_dict:856            return (image, has_nsfw_concept)857 858        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)859