diffusers/community-pipelines-mirror

1# Copyright 2023 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import inspect16from typing import Callable, List, Optional, Union17 18import torch19from packaging import version20from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer21 22from diffusers import DiffusionPipeline23from diffusers.configuration_utils import FrozenDict24from diffusers.models import AutoencoderKL, UNet2DConditionModel25from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import StableDiffusionPipelineOutput26from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker27from diffusers.schedulers import (28    DDIMScheduler,29    DPMSolverMultistepScheduler,30    EulerAncestralDiscreteScheduler,31    EulerDiscreteScheduler,32    LMSDiscreteScheduler,33    PNDMScheduler,34)35from diffusers.utils import deprecate, is_accelerate_available, logging36 37 38logger = logging.get_logger(__name__)  # pylint: disable=invalid-name39 40 41class ComposableStableDiffusionPipeline(DiffusionPipeline):42    r"""43    Pipeline for text-to-image generation using Stable Diffusion.44 45    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the46    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)47 48    Args:49        vae ([`AutoencoderKL`]):50            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.51        text_encoder ([`CLIPTextModel`]):52            Frozen text-encoder. Stable Diffusion uses the text portion of53            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically54            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.55        tokenizer (`CLIPTokenizer`):56            Tokenizer of class57            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).58        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.59        scheduler ([`SchedulerMixin`]):60            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of61            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].62        safety_checker ([`StableDiffusionSafetyChecker`]):63            Classification module that estimates whether generated images could be considered offensive or harmful.64            Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details.65        feature_extractor ([`CLIPImageProcessor`]):66            Model that extracts features from generated images to be used as inputs for the `safety_checker`.67    """68    _optional_components = ["safety_checker", "feature_extractor"]69 70    def __init__(71        self,72        vae: AutoencoderKL,73        text_encoder: CLIPTextModel,74        tokenizer: CLIPTokenizer,75        unet: UNet2DConditionModel,76        scheduler: Union[77            DDIMScheduler,78            PNDMScheduler,79            LMSDiscreteScheduler,80            EulerDiscreteScheduler,81            EulerAncestralDiscreteScheduler,82            DPMSolverMultistepScheduler,83        ],84        safety_checker: StableDiffusionSafetyChecker,85        feature_extractor: CLIPImageProcessor,86        requires_safety_checker: bool = True,87    ):88        super().__init__()89 90        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:91            deprecation_message = (92                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"93                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "94                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"95                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"96                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"97                " file"98            )99            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)100            new_config = dict(scheduler.config)101            new_config["steps_offset"] = 1102            scheduler._internal_dict = FrozenDict(new_config)103 104        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:105            deprecation_message = (106                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."107                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"108                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"109                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"110                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"111            )112            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)113            new_config = dict(scheduler.config)114            new_config["clip_sample"] = False115            scheduler._internal_dict = FrozenDict(new_config)116 117        if safety_checker is None and requires_safety_checker:118            logger.warning(119                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"120                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"121                " results in services or applications open to the public. Both the diffusers team and Hugging Face"122                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"123                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"124                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."125            )126 127        if safety_checker is not None and feature_extractor is None:128            raise ValueError(129                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"130                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."131            )132 133        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(134            version.parse(unet.config._diffusers_version).base_version135        ) < version.parse("0.9.0.dev0")136        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64137        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:138            deprecation_message = (139                "The configuration file of the unet has set the default `sample_size` to smaller than"140                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"141                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"142                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"143                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"144                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"145                " in the config might lead to incorrect results in future versions. If you have downloaded this"146                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"147                " the `unet/config.json` file"148            )149            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)150            new_config = dict(unet.config)151            new_config["sample_size"] = 64152            unet._internal_dict = FrozenDict(new_config)153 154        self.register_modules(155            vae=vae,156            text_encoder=text_encoder,157            tokenizer=tokenizer,158            unet=unet,159            scheduler=scheduler,160            safety_checker=safety_checker,161            feature_extractor=feature_extractor,162        )163        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)164        self.register_to_config(requires_safety_checker=requires_safety_checker)165 166    def enable_vae_slicing(self):167        r"""168        Enable sliced VAE decoding.169 170        When this option is enabled, the VAE will split the input tensor in slices to compute decoding in several171        steps. This is useful to save some memory and allow larger batch sizes.172        """173        self.vae.enable_slicing()174 175    def disable_vae_slicing(self):176        r"""177        Disable sliced VAE decoding. If `enable_vae_slicing` was previously invoked, this method will go back to178        computing decoding in one step.179        """180        self.vae.disable_slicing()181 182    def enable_sequential_cpu_offload(self, gpu_id=0):183        r"""184        Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet,185        text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a186        `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called.187        """188        if is_accelerate_available():189            from accelerate import cpu_offload190        else:191            raise ImportError("Please install accelerate via `pip install accelerate`")192 193        device = torch.device(f"cuda:{gpu_id}")194 195        for cpu_offloaded_model in [self.unet, self.text_encoder, self.vae]:196            if cpu_offloaded_model is not None:197                cpu_offload(cpu_offloaded_model, device)198 199        if self.safety_checker is not None:200            # TODO(Patrick) - there is currently a bug with cpu offload of nn.Parameter in accelerate201            # fix by only offloading self.safety_checker for now202            cpu_offload(self.safety_checker.vision_model, device)203 204    @property205    def _execution_device(self):206        r"""207        Returns the device on which the pipeline's models will be executed. After calling208        `pipeline.enable_sequential_cpu_offload()` the execution device can only be inferred from Accelerate's module209        hooks.210        """211        if self.device != torch.device("meta") or not hasattr(self.unet, "_hf_hook"):212            return self.device213        for module in self.unet.modules():214            if (215                hasattr(module, "_hf_hook")216                and hasattr(module._hf_hook, "execution_device")217                and module._hf_hook.execution_device is not None218            ):219                return torch.device(module._hf_hook.execution_device)220        return self.device221 222    def _encode_prompt(self, prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt):223        r"""224        Encodes the prompt into text encoder hidden states.225 226        Args:227            prompt (`str` or `list(int)`):228                prompt to be encoded229            device: (`torch.device`):230                torch device231            num_images_per_prompt (`int`):232                number of images that should be generated per prompt233            do_classifier_free_guidance (`bool`):234                whether to use classifier free guidance or not235            negative_prompt (`str` or `List[str]`):236                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored237                if `guidance_scale` is less than `1`).238        """239        batch_size = len(prompt) if isinstance(prompt, list) else 1240 241        text_inputs = self.tokenizer(242            prompt,243            padding="max_length",244            max_length=self.tokenizer.model_max_length,245            truncation=True,246            return_tensors="pt",247        )248        text_input_ids = text_inputs.input_ids249        untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids250 251        if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):252            removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1])253            logger.warning(254                "The following part of your input was truncated because CLIP can only handle sequences up to"255                f" {self.tokenizer.model_max_length} tokens: {removed_text}"256            )257 258        if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:259            attention_mask = text_inputs.attention_mask.to(device)260        else:261            attention_mask = None262 263        text_embeddings = self.text_encoder(264            text_input_ids.to(device),265            attention_mask=attention_mask,266        )267        text_embeddings = text_embeddings[0]268 269        # duplicate text embeddings for each generation per prompt, using mps friendly method270        bs_embed, seq_len, _ = text_embeddings.shape271        text_embeddings = text_embeddings.repeat(1, num_images_per_prompt, 1)272        text_embeddings = text_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1)273 274        # get unconditional embeddings for classifier free guidance275        if do_classifier_free_guidance:276            uncond_tokens: List[str]277            if negative_prompt is None:278                uncond_tokens = [""] * batch_size279            elif type(prompt) is not type(negative_prompt):280                raise TypeError(281                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="282                    f" {type(prompt)}."283                )284            elif isinstance(negative_prompt, str):285                uncond_tokens = [negative_prompt]286            elif batch_size != len(negative_prompt):287                raise ValueError(288                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"289                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"290                    " the batch size of `prompt`."291                )292            else:293                uncond_tokens = negative_prompt294 295            max_length = text_input_ids.shape[-1]296            uncond_input = self.tokenizer(297                uncond_tokens,298                padding="max_length",299                max_length=max_length,300                truncation=True,301                return_tensors="pt",302            )303 304            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:305                attention_mask = uncond_input.attention_mask.to(device)306            else:307                attention_mask = None308 309            uncond_embeddings = self.text_encoder(310                uncond_input.input_ids.to(device),311                attention_mask=attention_mask,312            )313            uncond_embeddings = uncond_embeddings[0]314 315            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method316            seq_len = uncond_embeddings.shape[1]317            uncond_embeddings = uncond_embeddings.repeat(1, num_images_per_prompt, 1)318            uncond_embeddings = uncond_embeddings.view(batch_size * num_images_per_prompt, seq_len, -1)319 320            # For classifier free guidance, we need to do two forward passes.321            # Here we concatenate the unconditional and text embeddings into a single batch322            # to avoid doing two forward passes323            text_embeddings = torch.cat([uncond_embeddings, text_embeddings])324 325        return text_embeddings326 327    def run_safety_checker(self, image, device, dtype):328        if self.safety_checker is not None:329            safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)330            image, has_nsfw_concept = self.safety_checker(331                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)332            )333        else:334            has_nsfw_concept = None335        return image, has_nsfw_concept336 337    def decode_latents(self, latents):338        latents = 1 / 0.18215 * latents339        image = self.vae.decode(latents).sample340        image = (image / 2 + 0.5).clamp(0, 1)341        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16342        image = image.cpu().permute(0, 2, 3, 1).float().numpy()343        return image344 345    def prepare_extra_step_kwargs(self, generator, eta):346        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature347        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.348        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502349        # and should be between [0, 1]350 351        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())352        extra_step_kwargs = {}353        if accepts_eta:354            extra_step_kwargs["eta"] = eta355 356        # check if the scheduler accepts generator357        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())358        if accepts_generator:359            extra_step_kwargs["generator"] = generator360        return extra_step_kwargs361 362    def check_inputs(self, prompt, height, width, callback_steps):363        if not isinstance(prompt, str) and not isinstance(prompt, list):364            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")365 366        if height % 8 != 0 or width % 8 != 0:367            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")368 369        if (callback_steps is None) or (370            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)371        ):372            raise ValueError(373                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"374                f" {type(callback_steps)}."375            )376 377    def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):378        shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor)379        if latents is None:380            if device.type == "mps":381                # randn does not work reproducibly on mps382                latents = torch.randn(shape, generator=generator, device="cpu", dtype=dtype).to(device)383            else:384                latents = torch.randn(shape, generator=generator, device=device, dtype=dtype)385        else:386            if latents.shape != shape:387                raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}")388            latents = latents.to(device)389 390        # scale the initial noise by the standard deviation required by the scheduler391        latents = latents * self.scheduler.init_noise_sigma392        return latents393 394    @torch.no_grad()395    def __call__(396        self,397        prompt: Union[str, List[str]],398        height: Optional[int] = None,399        width: Optional[int] = None,400        num_inference_steps: int = 50,401        guidance_scale: float = 7.5,402        negative_prompt: Optional[Union[str, List[str]]] = None,403        num_images_per_prompt: Optional[int] = 1,404        eta: float = 0.0,405        generator: Optional[torch.Generator] = None,406        latents: Optional[torch.FloatTensor] = None,407        output_type: Optional[str] = "pil",408        return_dict: bool = True,409        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,410        callback_steps: int = 1,411        weights: Optional[str] = "",412    ):413        r"""414        Function invoked when calling the pipeline for generation.415 416        Args:417            prompt (`str` or `List[str]`):418                The prompt or prompts to guide the image generation.419            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):420                The height in pixels of the generated image.421            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):422                The width in pixels of the generated image.423            num_inference_steps (`int`, *optional*, defaults to 50):424                The number of denoising steps. More denoising steps usually lead to a higher quality image at the425                expense of slower inference.426            guidance_scale (`float`, *optional*, defaults to 5.0):427                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).428                `guidance_scale` is defined as `w` of equation 2. of [Imagen429                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >430                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,431                usually at the expense of lower image quality.432            negative_prompt (`str` or `List[str]`, *optional*):433                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored434                if `guidance_scale` is less than `1`).435            num_images_per_prompt (`int`, *optional*, defaults to 1):436                The number of images to generate per prompt.437            eta (`float`, *optional*, defaults to 0.0):438                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to439                [`schedulers.DDIMScheduler`], will be ignored for others.440            generator (`torch.Generator`, *optional*):441                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation442                deterministic.443            latents (`torch.FloatTensor`, *optional*):444                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image445                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents446                tensor will ge generated by sampling using the supplied random `generator`.447            output_type (`str`, *optional*, defaults to `"pil"`):448                The output format of the generate image. Choose between449                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.450            return_dict (`bool`, *optional*, defaults to `True`):451                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a452                plain tuple.453            callback (`Callable`, *optional*):454                A function that will be called every `callback_steps` steps during inference. The function will be455                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.456            callback_steps (`int`, *optional*, defaults to 1):457                The frequency at which the `callback` function will be called. If not specified, the callback will be458                called at every step.459 460        Returns:461            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:462            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.463            When returning a tuple, the first element is a list with the generated images, and the second element is a464            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"465            (nsfw) content, according to the `safety_checker`.466        """467        # 0. Default height and width to unet468        height = height or self.unet.config.sample_size * self.vae_scale_factor469        width = width or self.unet.config.sample_size * self.vae_scale_factor470 471        # 1. Check inputs. Raise error if not correct472        self.check_inputs(prompt, height, width, callback_steps)473 474        # 2. Define call parameters475        batch_size = 1 if isinstance(prompt, str) else len(prompt)476        device = self._execution_device477        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)478        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`479        # corresponds to doing no classifier free guidance.480        do_classifier_free_guidance = guidance_scale > 1.0481 482        if "|" in prompt:483            prompt = [x.strip() for x in prompt.split("|")]484            print(f"composing {prompt}...")485 486            if not weights:487                # specify weights for prompts (excluding the unconditional score)488                print("using equal positive weights (conjunction) for all prompts...")489                weights = torch.tensor([guidance_scale] * len(prompt), device=self.device).reshape(-1, 1, 1, 1)490            else:491                # set prompt weight for each492                num_prompts = len(prompt) if isinstance(prompt, list) else 1493                weights = [float(w.strip()) for w in weights.split("|")]494                # guidance scale as the default495                if len(weights) < num_prompts:496                    weights.append(guidance_scale)497                else:498                    weights = weights[:num_prompts]499                assert len(weights) == len(prompt), "weights specified are not equal to the number of prompts"500                weights = torch.tensor(weights, device=self.device).reshape(-1, 1, 1, 1)501        else:502            weights = guidance_scale503 504        # 3. Encode input prompt505        text_embeddings = self._encode_prompt(506            prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt507        )508 509        # 4. Prepare timesteps510        self.scheduler.set_timesteps(num_inference_steps, device=device)511        timesteps = self.scheduler.timesteps512 513        # 5. Prepare latent variables514        num_channels_latents = self.unet.config.in_channels515        latents = self.prepare_latents(516            batch_size * num_images_per_prompt,517            num_channels_latents,518            height,519            width,520            text_embeddings.dtype,521            device,522            generator,523            latents,524        )525 526        # composable diffusion527        if isinstance(prompt, list) and batch_size == 1:528            # remove extra unconditional embedding529            # N = one unconditional embed + conditional embeds530            text_embeddings = text_embeddings[len(prompt) - 1 :]531 532        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline533        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)534 535        # 7. Denoising loop536        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order537        with self.progress_bar(total=num_inference_steps) as progress_bar:538            for i, t in enumerate(timesteps):539                # expand the latents if we are doing classifier free guidance540                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents541                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)542 543                # predict the noise residual544                noise_pred = []545                for j in range(text_embeddings.shape[0]):546                    noise_pred.append(547                        self.unet(latent_model_input[:1], t, encoder_hidden_states=text_embeddings[j : j + 1]).sample548                    )549                noise_pred = torch.cat(noise_pred, dim=0)550 551                # perform guidance552                if do_classifier_free_guidance:553                    noise_pred_uncond, noise_pred_text = noise_pred[:1], noise_pred[1:]554                    noise_pred = noise_pred_uncond + (weights * (noise_pred_text - noise_pred_uncond)).sum(555                        dim=0, keepdims=True556                    )557 558                # compute the previous noisy sample x_t -> x_t-1559                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample560 561                # call the callback, if provided562                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):563                    progress_bar.update()564                    if callback is not None and i % callback_steps == 0:565                        callback(i, t, latents)566 567        # 8. Post-processing568        image = self.decode_latents(latents)569 570        # 9. Run safety checker571        image, has_nsfw_concept = self.run_safety_checker(image, device, text_embeddings.dtype)572 573        # 10. Convert to PIL574        if output_type == "pil":575            image = self.numpy_to_pil(image)576 577        if not return_dict:578            return (image, has_nsfw_concept)579 580        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)581