diffusers/community-pipelines-mirror

1# Copyright 2023 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import inspect16from typing import Any, Callable, Dict, List, Optional, Union17 18import intel_extension_for_pytorch as ipex19import torch20from packaging import version21from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer22 23from diffusers.configuration_utils import FrozenDict24from diffusers.models import AutoencoderKL, UNet2DConditionModel25from diffusers.pipeline_utils import DiffusionPipeline26from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput27from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker28from diffusers.schedulers import KarrasDiffusionSchedulers29from diffusers.utils import (30    deprecate,31    is_accelerate_available,32    is_accelerate_version,33    logging,34    randn_tensor,35    replace_example_docstring,36)37 38 39logger = logging.get_logger(__name__)  # pylint: disable=invalid-name40 41EXAMPLE_DOC_STRING = """42    Examples:43        ```py44        >>> import torch45        >>> from diffusers import StableDiffusionPipeline46 47        >>> pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", custom_pipeline="stable_diffusion_ipex")48 49        >>> # For Float3250        >>> pipe.prepare_for_ipex(prompt, dtype=torch.float32, height=512, width=512) #value of image height/width should be consistent with the pipeline inference51        >>> # For BFloat1652        >>> pipe.prepare_for_ipex(prompt, dtype=torch.bfloat16, height=512, width=512) #value of image height/width should be consistent with the pipeline inference53 54        >>> prompt = "a photo of an astronaut riding a horse on mars"55        >>> # For Float3256        >>> image = pipe(prompt, num_inference_steps=num_inference_steps, height=512, width=512).images[0] #value of image height/width should be consistent with 'prepare_for_ipex()'57        >>> # For BFloat1658        >>> with torch.cpu.amp.autocast(enabled=True, dtype=torch.bfloat16):59        >>>     image = pipe(prompt, num_inference_steps=num_inference_steps, height=512, width=512).images[0] #value of image height/width should be consistent with 'prepare_for_ipex()'60        ```61"""62 63 64class StableDiffusionIPEXPipeline(DiffusionPipeline):65    r"""66    Pipeline for text-to-image generation using Stable Diffusion on IPEX.67 68    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the69    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)70 71    Args:72        vae ([`AutoencoderKL`]):73            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.74        text_encoder ([`CLIPTextModel`]):75            Frozen text-encoder. Stable Diffusion uses the text portion of76            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically77            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.78        tokenizer (`CLIPTokenizer`):79            Tokenizer of class80            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).81        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.82        scheduler ([`SchedulerMixin`]):83            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of84            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].85        safety_checker ([`StableDiffusionSafetyChecker`]):86            Classification module that estimates whether generated images could be considered offensive or harmful.87            Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details.88        feature_extractor ([`CLIPFeatureExtractor`]):89            Model that extracts features from generated images to be used as inputs for the `safety_checker`.90    """91    _optional_components = ["safety_checker", "feature_extractor"]92 93    def __init__(94        self,95        vae: AutoencoderKL,96        text_encoder: CLIPTextModel,97        tokenizer: CLIPTokenizer,98        unet: UNet2DConditionModel,99        scheduler: KarrasDiffusionSchedulers,100        safety_checker: StableDiffusionSafetyChecker,101        feature_extractor: CLIPFeatureExtractor,102        requires_safety_checker: bool = True,103    ):104        super().__init__()105 106        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:107            deprecation_message = (108                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"109                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "110                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"111                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"112                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"113                " file"114            )115            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)116            new_config = dict(scheduler.config)117            new_config["steps_offset"] = 1118            scheduler._internal_dict = FrozenDict(new_config)119 120        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:121            deprecation_message = (122                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."123                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"124                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"125                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"126                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"127            )128            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)129            new_config = dict(scheduler.config)130            new_config["clip_sample"] = False131            scheduler._internal_dict = FrozenDict(new_config)132 133        if safety_checker is None and requires_safety_checker:134            logger.warning(135                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"136                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"137                " results in services or applications open to the public. Both the diffusers team and Hugging Face"138                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"139                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"140                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."141            )142 143        if safety_checker is not None and feature_extractor is None:144            raise ValueError(145                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"146                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."147            )148 149        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(150            version.parse(unet.config._diffusers_version).base_version151        ) < version.parse("0.9.0.dev0")152        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64153        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:154            deprecation_message = (155                "The configuration file of the unet has set the default `sample_size` to smaller than"156                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"157                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"158                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"159                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"160                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"161                " in the config might lead to incorrect results in future versions. If you have downloaded this"162                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"163                " the `unet/config.json` file"164            )165            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)166            new_config = dict(unet.config)167            new_config["sample_size"] = 64168            unet._internal_dict = FrozenDict(new_config)169 170        self.register_modules(171            vae=vae,172            text_encoder=text_encoder,173            tokenizer=tokenizer,174            unet=unet,175            scheduler=scheduler,176            safety_checker=safety_checker,177            feature_extractor=feature_extractor,178        )179        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)180        self.register_to_config(requires_safety_checker=requires_safety_checker)181 182    def get_input_example(self, prompt, height=None, width=None, guidance_scale=7.5, num_images_per_prompt=1):183        prompt_embeds = None184        negative_prompt_embeds = None185        negative_prompt = None186        callback_steps = 1187        generator = None188        latents = None189 190        # 0. Default height and width to unet191        height = height or self.unet.config.sample_size * self.vae_scale_factor192        width = width or self.unet.config.sample_size * self.vae_scale_factor193 194        # 1. Check inputs. Raise error if not correct195        self.check_inputs(196            prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds197        )198 199        # 2. Define call parameters200        if prompt is not None and isinstance(prompt, str):201            batch_size = 1202        elif prompt is not None and isinstance(prompt, list):203            batch_size = len(prompt)204 205        device = "cpu"206        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)207        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`208        # corresponds to doing no classifier free guidance.209        do_classifier_free_guidance = guidance_scale > 1.0210 211        # 3. Encode input prompt212        prompt_embeds = self._encode_prompt(213            prompt,214            device,215            num_images_per_prompt,216            do_classifier_free_guidance,217            negative_prompt,218            prompt_embeds=prompt_embeds,219            negative_prompt_embeds=negative_prompt_embeds,220        )221 222        # 5. Prepare latent variables223        latents = self.prepare_latents(224            batch_size * num_images_per_prompt,225            self.unet.in_channels,226            height,227            width,228            prompt_embeds.dtype,229            device,230            generator,231            latents,232        )233        dummy = torch.ones(1, dtype=torch.int32)234        latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents235        latent_model_input = self.scheduler.scale_model_input(latent_model_input, dummy)236 237        unet_input_example = (latent_model_input, dummy, prompt_embeds)238        vae_decoder_input_example = latents239 240        return unet_input_example, vae_decoder_input_example241 242    def prepare_for_ipex(self, promt, dtype=torch.float32, height=None, width=None, guidance_scale=7.5):243        self.unet = self.unet.to(memory_format=torch.channels_last)244        self.vae.decoder = self.vae.decoder.to(memory_format=torch.channels_last)245        self.text_encoder = self.text_encoder.to(memory_format=torch.channels_last)246        if self.safety_checker is not None:247            self.safety_checker = self.safety_checker.to(memory_format=torch.channels_last)248 249        unet_input_example, vae_decoder_input_example = self.get_input_example(promt, height, width, guidance_scale)250 251        # optimize with ipex252        if dtype == torch.bfloat16:253            self.unet = ipex.optimize(254                self.unet.eval(), dtype=torch.bfloat16, inplace=True, sample_input=unet_input_example255            )256            self.vae.decoder = ipex.optimize(self.vae.decoder.eval(), dtype=torch.bfloat16, inplace=True)257            self.text_encoder = ipex.optimize(self.text_encoder.eval(), dtype=torch.bfloat16, inplace=True)258            if self.safety_checker is not None:259                self.safety_checker = ipex.optimize(self.safety_checker.eval(), dtype=torch.bfloat16, inplace=True)260        elif dtype == torch.float32:261            self.unet = ipex.optimize(262                self.unet.eval(),263                dtype=torch.float32,264                inplace=True,265                sample_input=unet_input_example,266                level="O1",267                weights_prepack=True,268                auto_kernel_selection=False,269            )270            self.vae.decoder = ipex.optimize(271                self.vae.decoder.eval(),272                dtype=torch.float32,273                inplace=True,274                level="O1",275                weights_prepack=True,276                auto_kernel_selection=False,277            )278            self.text_encoder = ipex.optimize(279                self.text_encoder.eval(),280                dtype=torch.float32,281                inplace=True,282                level="O1",283                weights_prepack=True,284                auto_kernel_selection=False,285            )286            if self.safety_checker is not None:287                self.safety_checker = ipex.optimize(288                    self.safety_checker.eval(),289                    dtype=torch.float32,290                    inplace=True,291                    level="O1",292                    weights_prepack=True,293                    auto_kernel_selection=False,294                )295        else:296            raise ValueError(" The value of 'dtype' should be 'torch.bfloat16' or 'torch.float32' !")297 298        # trace unet model to get better performance on IPEX299        with torch.cpu.amp.autocast(enabled=dtype == torch.bfloat16), torch.no_grad():300            unet_trace_model = torch.jit.trace(self.unet, unet_input_example, check_trace=False, strict=False)301            unet_trace_model = torch.jit.freeze(unet_trace_model)302        self.unet.forward = unet_trace_model.forward303 304        # trace vae.decoder model to get better performance on IPEX305        with torch.cpu.amp.autocast(enabled=dtype == torch.bfloat16), torch.no_grad():306            ave_decoder_trace_model = torch.jit.trace(307                self.vae.decoder, vae_decoder_input_example, check_trace=False, strict=False308            )309            ave_decoder_trace_model = torch.jit.freeze(ave_decoder_trace_model)310        self.vae.decoder.forward = ave_decoder_trace_model.forward311 312    def enable_vae_slicing(self):313        r"""314        Enable sliced VAE decoding.315 316        When this option is enabled, the VAE will split the input tensor in slices to compute decoding in several317        steps. This is useful to save some memory and allow larger batch sizes.318        """319        self.vae.enable_slicing()320 321    def disable_vae_slicing(self):322        r"""323        Disable sliced VAE decoding. If `enable_vae_slicing` was previously invoked, this method will go back to324        computing decoding in one step.325        """326        self.vae.disable_slicing()327 328    def enable_vae_tiling(self):329        r"""330        Enable tiled VAE decoding.331 332        When this option is enabled, the VAE will split the input tensor into tiles to compute decoding and encoding in333        several steps. This is useful to save a large amount of memory and to allow the processing of larger images.334        """335        self.vae.enable_tiling()336 337    def disable_vae_tiling(self):338        r"""339        Disable tiled VAE decoding. If `enable_vae_tiling` was previously invoked, this method will go back to340        computing decoding in one step.341        """342        self.vae.disable_tiling()343 344    def enable_sequential_cpu_offload(self, gpu_id=0):345        r"""346        Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet,347        text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a348        `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called.349        Note that offloading happens on a submodule basis. Memory savings are higher than with350        `enable_model_cpu_offload`, but performance is lower.351        """352        if is_accelerate_available() and is_accelerate_version(">=", "0.14.0"):353            from accelerate import cpu_offload354        else:355            raise ImportError("`enable_sequential_cpu_offload` requires `accelerate v0.14.0` or higher")356 357        device = torch.device(f"cuda:{gpu_id}")358 359        if self.device.type != "cpu":360            self.to("cpu", silence_dtype_warnings=True)361            torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)362 363        for cpu_offloaded_model in [self.unet, self.text_encoder, self.vae]:364            cpu_offload(cpu_offloaded_model, device)365 366        if self.safety_checker is not None:367            cpu_offload(self.safety_checker, execution_device=device, offload_buffers=True)368 369    def enable_model_cpu_offload(self, gpu_id=0):370        r"""371        Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared372        to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward`373        method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with374        `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`.375        """376        if is_accelerate_available() and is_accelerate_version(">=", "0.17.0.dev0"):377            from accelerate import cpu_offload_with_hook378        else:379            raise ImportError("`enable_model_offload` requires `accelerate v0.17.0` or higher.")380 381        device = torch.device(f"cuda:{gpu_id}")382 383        if self.device.type != "cpu":384            self.to("cpu", silence_dtype_warnings=True)385            torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)386 387        hook = None388        for cpu_offloaded_model in [self.text_encoder, self.unet, self.vae]:389            _, hook = cpu_offload_with_hook(cpu_offloaded_model, device, prev_module_hook=hook)390 391        if self.safety_checker is not None:392            _, hook = cpu_offload_with_hook(self.safety_checker, device, prev_module_hook=hook)393 394        # We'll offload the last model manually.395        self.final_offload_hook = hook396 397    @property398    def _execution_device(self):399        r"""400        Returns the device on which the pipeline's models will be executed. After calling401        `pipeline.enable_sequential_cpu_offload()` the execution device can only be inferred from Accelerate's module402        hooks.403        """404        if not hasattr(self.unet, "_hf_hook"):405            return self.device406        for module in self.unet.modules():407            if (408                hasattr(module, "_hf_hook")409                and hasattr(module._hf_hook, "execution_device")410                and module._hf_hook.execution_device is not None411            ):412                return torch.device(module._hf_hook.execution_device)413        return self.device414 415    def _encode_prompt(416        self,417        prompt,418        device,419        num_images_per_prompt,420        do_classifier_free_guidance,421        negative_prompt=None,422        prompt_embeds: Optional[torch.FloatTensor] = None,423        negative_prompt_embeds: Optional[torch.FloatTensor] = None,424    ):425        r"""426        Encodes the prompt into text encoder hidden states.427 428        Args:429             prompt (`str` or `List[str]`, *optional*):430                prompt to be encoded431            device: (`torch.device`):432                torch device433            num_images_per_prompt (`int`):434                number of images that should be generated per prompt435            do_classifier_free_guidance (`bool`):436                whether to use classifier free guidance or not437            negative_prompt (`str` or `List[str]`, *optional*):438                The prompt or prompts not to guide the image generation. If not defined, one has to pass439                `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.440                Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).441            prompt_embeds (`torch.FloatTensor`, *optional*):442                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not443                provided, text embeddings will be generated from `prompt` input argument.444            negative_prompt_embeds (`torch.FloatTensor`, *optional*):445                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt446                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input447                argument.448        """449        if prompt is not None and isinstance(prompt, str):450            batch_size = 1451        elif prompt is not None and isinstance(prompt, list):452            batch_size = len(prompt)453        else:454            batch_size = prompt_embeds.shape[0]455 456        if prompt_embeds is None:457            text_inputs = self.tokenizer(458                prompt,459                padding="max_length",460                max_length=self.tokenizer.model_max_length,461                truncation=True,462                return_tensors="pt",463            )464            text_input_ids = text_inputs.input_ids465            untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids466 467            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(468                text_input_ids, untruncated_ids469            ):470                removed_text = self.tokenizer.batch_decode(471                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]472                )473                logger.warning(474                    "The following part of your input was truncated because CLIP can only handle sequences up to"475                    f" {self.tokenizer.model_max_length} tokens: {removed_text}"476                )477 478            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:479                attention_mask = text_inputs.attention_mask.to(device)480            else:481                attention_mask = None482 483            prompt_embeds = self.text_encoder(484                text_input_ids.to(device),485                attention_mask=attention_mask,486            )487            prompt_embeds = prompt_embeds[0]488 489        prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)490 491        bs_embed, seq_len, _ = prompt_embeds.shape492        # duplicate text embeddings for each generation per prompt, using mps friendly method493        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)494        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)495 496        # get unconditional embeddings for classifier free guidance497        if do_classifier_free_guidance and negative_prompt_embeds is None:498            uncond_tokens: List[str]499            if negative_prompt is None:500                uncond_tokens = [""] * batch_size501            elif type(prompt) is not type(negative_prompt):502                raise TypeError(503                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="504                    f" {type(prompt)}."505                )506            elif isinstance(negative_prompt, str):507                uncond_tokens = [negative_prompt]508            elif batch_size != len(negative_prompt):509                raise ValueError(510                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"511                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"512                    " the batch size of `prompt`."513                )514            else:515                uncond_tokens = negative_prompt516 517            max_length = prompt_embeds.shape[1]518            uncond_input = self.tokenizer(519                uncond_tokens,520                padding="max_length",521                max_length=max_length,522                truncation=True,523                return_tensors="pt",524            )525 526            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:527                attention_mask = uncond_input.attention_mask.to(device)528            else:529                attention_mask = None530 531            negative_prompt_embeds = self.text_encoder(532                uncond_input.input_ids.to(device),533                attention_mask=attention_mask,534            )535            negative_prompt_embeds = negative_prompt_embeds[0]536 537        if do_classifier_free_guidance:538            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method539            seq_len = negative_prompt_embeds.shape[1]540 541            negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)542 543            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)544            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)545 546            # For classifier free guidance, we need to do two forward passes.547            # Here we concatenate the unconditional and text embeddings into a single batch548            # to avoid doing two forward passes549            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])550 551        return prompt_embeds552 553    def run_safety_checker(self, image, device, dtype):554        if self.safety_checker is not None:555            safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)556            image, has_nsfw_concept = self.safety_checker(557                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)558            )559        else:560            has_nsfw_concept = None561        return image, has_nsfw_concept562 563    def decode_latents(self, latents):564        latents = 1 / self.vae.config.scaling_factor * latents565        image = self.vae.decode(latents).sample566        image = (image / 2 + 0.5).clamp(0, 1)567        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16568        image = image.cpu().permute(0, 2, 3, 1).float().numpy()569        return image570 571    def prepare_extra_step_kwargs(self, generator, eta):572        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature573        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.574        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502575        # and should be between [0, 1]576 577        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())578        extra_step_kwargs = {}579        if accepts_eta:580            extra_step_kwargs["eta"] = eta581 582        # check if the scheduler accepts generator583        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())584        if accepts_generator:585            extra_step_kwargs["generator"] = generator586        return extra_step_kwargs587 588    def check_inputs(589        self,590        prompt,591        height,592        width,593        callback_steps,594        negative_prompt=None,595        prompt_embeds=None,596        negative_prompt_embeds=None,597    ):598        if height % 8 != 0 or width % 8 != 0:599            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")600 601        if (callback_steps is None) or (602            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)603        ):604            raise ValueError(605                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"606                f" {type(callback_steps)}."607            )608 609        if prompt is not None and prompt_embeds is not None:610            raise ValueError(611                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"612                " only forward one of the two."613            )614        elif prompt is None and prompt_embeds is None:615            raise ValueError(616                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."617            )618        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):619            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")620 621        if negative_prompt is not None and negative_prompt_embeds is not None:622            raise ValueError(623                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"624                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."625            )626 627        if prompt_embeds is not None and negative_prompt_embeds is not None:628            if prompt_embeds.shape != negative_prompt_embeds.shape:629                raise ValueError(630                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"631                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"632                    f" {negative_prompt_embeds.shape}."633                )634 635    def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):636        shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor)637        if isinstance(generator, list) and len(generator) != batch_size:638            raise ValueError(639                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"640                f" size of {batch_size}. Make sure the batch size matches the length of the generators."641            )642 643        if latents is None:644            latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)645        else:646            latents = latents.to(device)647 648        # scale the initial noise by the standard deviation required by the scheduler649        latents = latents * self.scheduler.init_noise_sigma650        return latents651 652    @torch.no_grad()653    @replace_example_docstring(EXAMPLE_DOC_STRING)654    def __call__(655        self,656        prompt: Union[str, List[str]] = None,657        height: Optional[int] = None,658        width: Optional[int] = None,659        num_inference_steps: int = 50,660        guidance_scale: float = 7.5,661        negative_prompt: Optional[Union[str, List[str]]] = None,662        num_images_per_prompt: Optional[int] = 1,663        eta: float = 0.0,664        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,665        latents: Optional[torch.FloatTensor] = None,666        prompt_embeds: Optional[torch.FloatTensor] = None,667        negative_prompt_embeds: Optional[torch.FloatTensor] = None,668        output_type: Optional[str] = "pil",669        return_dict: bool = True,670        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,671        callback_steps: int = 1,672        cross_attention_kwargs: Optional[Dict[str, Any]] = None,673    ):674        r"""675        Function invoked when calling the pipeline for generation.676 677        Args:678            prompt (`str` or `List[str]`, *optional*):679                The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.680                instead.681            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):682                The height in pixels of the generated image.683            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):684                The width in pixels of the generated image.685            num_inference_steps (`int`, *optional*, defaults to 50):686                The number of denoising steps. More denoising steps usually lead to a higher quality image at the687                expense of slower inference.688            guidance_scale (`float`, *optional*, defaults to 7.5):689                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).690                `guidance_scale` is defined as `w` of equation 2. of [Imagen691                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >692                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,693                usually at the expense of lower image quality.694            negative_prompt (`str` or `List[str]`, *optional*):695                The prompt or prompts not to guide the image generation. If not defined, one has to pass696                `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.697                Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).698            num_images_per_prompt (`int`, *optional*, defaults to 1):699                The number of images to generate per prompt.700            eta (`float`, *optional*, defaults to 0.0):701                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to702                [`schedulers.DDIMScheduler`], will be ignored for others.703            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):704                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)705                to make generation deterministic.706            latents (`torch.FloatTensor`, *optional*):707                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image708                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents709                tensor will ge generated by sampling using the supplied random `generator`.710            prompt_embeds (`torch.FloatTensor`, *optional*):711                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not712                provided, text embeddings will be generated from `prompt` input argument.713            negative_prompt_embeds (`torch.FloatTensor`, *optional*):714                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt715                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input716                argument.717            output_type (`str`, *optional*, defaults to `"pil"`):718                The output format of the generate image. Choose between719                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.720            return_dict (`bool`, *optional*, defaults to `True`):721                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a722                plain tuple.723            callback (`Callable`, *optional*):724                A function that will be called every `callback_steps` steps during inference. The function will be725                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.726            callback_steps (`int`, *optional*, defaults to 1):727                The frequency at which the `callback` function will be called. If not specified, the callback will be728                called at every step.729            cross_attention_kwargs (`dict`, *optional*):730                A kwargs dictionary that if specified is passed along to the `AttnProcessor` as defined under731                `self.processor` in732                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).733 734        Examples:735 736        Returns:737            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:738            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.739            When returning a tuple, the first element is a list with the generated images, and the second element is a740            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"741            (nsfw) content, according to the `safety_checker`.742        """743        # 0. Default height and width to unet744        height = height or self.unet.config.sample_size * self.vae_scale_factor745        width = width or self.unet.config.sample_size * self.vae_scale_factor746 747        # 1. Check inputs. Raise error if not correct748        self.check_inputs(749            prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds750        )751 752        # 2. Define call parameters753        if prompt is not None and isinstance(prompt, str):754            batch_size = 1755        elif prompt is not None and isinstance(prompt, list):756            batch_size = len(prompt)757        else:758            batch_size = prompt_embeds.shape[0]759 760        device = self._execution_device761        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)762        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`763        # corresponds to doing no classifier free guidance.764        do_classifier_free_guidance = guidance_scale > 1.0765 766        # 3. Encode input prompt767        prompt_embeds = self._encode_prompt(768            prompt,769            device,770            num_images_per_prompt,771            do_classifier_free_guidance,772            negative_prompt,773            prompt_embeds=prompt_embeds,774            negative_prompt_embeds=negative_prompt_embeds,775        )776 777        # 4. Prepare timesteps778        self.scheduler.set_timesteps(num_inference_steps, device=device)779        timesteps = self.scheduler.timesteps780 781        # 5. Prepare latent variables782        num_channels_latents = self.unet.in_channels783        latents = self.prepare_latents(784            batch_size * num_images_per_prompt,785            num_channels_latents,786            height,787            width,788            prompt_embeds.dtype,789            device,790            generator,791            latents,792        )793 794        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline795        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)796 797        # 7. Denoising loop798        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order799        with self.progress_bar(total=num_inference_steps) as progress_bar:800            for i, t in enumerate(timesteps):801                # expand the latents if we are doing classifier free guidance802                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents803                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)804 805                # predict the noise residual806                noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=prompt_embeds)["sample"]807 808                # perform guidance809                if do_classifier_free_guidance:810                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)811                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)812 813                # compute the previous noisy sample x_t -> x_t-1814                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample815 816                # call the callback, if provided817                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):818                    progress_bar.update()819                    if callback is not None and i % callback_steps == 0:820                        callback(i, t, latents)821 822        if output_type == "latent":823            image = latents824            has_nsfw_concept = None825        elif output_type == "pil":826            # 8. Post-processing827            image = self.decode_latents(latents)828 829            # 9. Run safety checker830            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)831 832            # 10. Convert to PIL833            image = self.numpy_to_pil(image)834        else:835            # 8. Post-processing836            image = self.decode_latents(latents)837 838            # 9. Run safety checker839            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)840 841        # Offload last model to CPU842        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:843            self.final_offload_hook.offload()844 845        if not return_dict:846            return (image, has_nsfw_concept)847 848        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)849