diffusers/community-pipelines-mirror

1# Inspired by: https://github.com/haofanwang/ControlNet-for-Diffusers/2 3import inspect4from typing import Any, Callable, Dict, List, Optional, Tuple, Union5 6import numpy as np7import PIL.Image8import torch9from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer10 11from diffusers import AutoencoderKL, ControlNetModel, UNet2DConditionModel, logging12from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel13from diffusers.pipelines.pipeline_utils import DiffusionPipeline, StableDiffusionMixin14from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput, StableDiffusionSafetyChecker15from diffusers.schedulers import KarrasDiffusionSchedulers16from diffusers.utils import (17    PIL_INTERPOLATION,18    replace_example_docstring,19)20from diffusers.utils.torch_utils import randn_tensor21 22 23logger = logging.get_logger(__name__)  # pylint: disable=invalid-name24 25EXAMPLE_DOC_STRING = """26    Examples:27        ```py28        >>> import numpy as np29        >>> import torch30        >>> from PIL import Image31        >>> from diffusers import ControlNetModel, UniPCMultistepScheduler32        >>> from diffusers.utils import load_image33 34        >>> input_image = load_image("https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png")35 36        >>> controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)37 38        >>> pipe_controlnet = StableDiffusionControlNetImg2ImgPipeline.from_pretrained(39                "runwayml/stable-diffusion-v1-5",40                controlnet=controlnet,41                safety_checker=None,42                torch_dtype=torch.float1643                )44 45        >>> pipe_controlnet.scheduler = UniPCMultistepScheduler.from_config(pipe_controlnet.scheduler.config)46        >>> pipe_controlnet.enable_xformers_memory_efficient_attention()47        >>> pipe_controlnet.enable_model_cpu_offload()48 49        # using image with edges for our canny controlnet50        >>> control_image = load_image(51            "https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/vermeer_canny_edged.png")52 53 54        >>> result_img = pipe_controlnet(controlnet_conditioning_image=control_image,55                        image=input_image,56                        prompt="an android robot, cyberpank, digitl art masterpiece",57                        num_inference_steps=20).images[0]58 59        >>> result_img.show()60        ```61"""62 63 64def prepare_image(image):65    if isinstance(image, torch.Tensor):66        # Batch single image67        if image.ndim == 3:68            image = image.unsqueeze(0)69 70        image = image.to(dtype=torch.float32)71    else:72        # preprocess image73        if isinstance(image, (PIL.Image.Image, np.ndarray)):74            image = [image]75 76        if isinstance(image, list) and isinstance(image[0], PIL.Image.Image):77            image = [np.array(i.convert("RGB"))[None, :] for i in image]78            image = np.concatenate(image, axis=0)79        elif isinstance(image, list) and isinstance(image[0], np.ndarray):80            image = np.concatenate([i[None, :] for i in image], axis=0)81 82        image = image.transpose(0, 3, 1, 2)83        image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.084 85    return image86 87 88def prepare_controlnet_conditioning_image(89    controlnet_conditioning_image,90    width,91    height,92    batch_size,93    num_images_per_prompt,94    device,95    dtype,96    do_classifier_free_guidance,97):98    if not isinstance(controlnet_conditioning_image, torch.Tensor):99        if isinstance(controlnet_conditioning_image, PIL.Image.Image):100            controlnet_conditioning_image = [controlnet_conditioning_image]101 102        if isinstance(controlnet_conditioning_image[0], PIL.Image.Image):103            controlnet_conditioning_image = [104                np.array(i.resize((width, height), resample=PIL_INTERPOLATION["lanczos"]))[None, :]105                for i in controlnet_conditioning_image106            ]107            controlnet_conditioning_image = np.concatenate(controlnet_conditioning_image, axis=0)108            controlnet_conditioning_image = np.array(controlnet_conditioning_image).astype(np.float32) / 255.0109            controlnet_conditioning_image = controlnet_conditioning_image.transpose(0, 3, 1, 2)110            controlnet_conditioning_image = torch.from_numpy(controlnet_conditioning_image)111        elif isinstance(controlnet_conditioning_image[0], torch.Tensor):112            controlnet_conditioning_image = torch.cat(controlnet_conditioning_image, dim=0)113 114    image_batch_size = controlnet_conditioning_image.shape[0]115 116    if image_batch_size == 1:117        repeat_by = batch_size118    else:119        # image batch size is the same as prompt batch size120        repeat_by = num_images_per_prompt121 122    controlnet_conditioning_image = controlnet_conditioning_image.repeat_interleave(repeat_by, dim=0)123 124    controlnet_conditioning_image = controlnet_conditioning_image.to(device=device, dtype=dtype)125 126    if do_classifier_free_guidance:127        controlnet_conditioning_image = torch.cat([controlnet_conditioning_image] * 2)128 129    return controlnet_conditioning_image130 131 132class StableDiffusionControlNetImg2ImgPipeline(DiffusionPipeline, StableDiffusionMixin):133    """134    Inspired by: https://github.com/haofanwang/ControlNet-for-Diffusers/135    """136 137    _optional_components = ["safety_checker", "feature_extractor"]138 139    def __init__(140        self,141        vae: AutoencoderKL,142        text_encoder: CLIPTextModel,143        tokenizer: CLIPTokenizer,144        unet: UNet2DConditionModel,145        controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel],146        scheduler: KarrasDiffusionSchedulers,147        safety_checker: StableDiffusionSafetyChecker,148        feature_extractor: CLIPImageProcessor,149        requires_safety_checker: bool = True,150    ):151        super().__init__()152 153        if safety_checker is None and requires_safety_checker:154            logger.warning(155                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"156                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"157                " results in services or applications open to the public. Both the diffusers team and Hugging Face"158                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"159                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"160                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."161            )162 163        if safety_checker is not None and feature_extractor is None:164            raise ValueError(165                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"166                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."167            )168 169        if isinstance(controlnet, (list, tuple)):170            controlnet = MultiControlNetModel(controlnet)171 172        self.register_modules(173            vae=vae,174            text_encoder=text_encoder,175            tokenizer=tokenizer,176            unet=unet,177            controlnet=controlnet,178            scheduler=scheduler,179            safety_checker=safety_checker,180            feature_extractor=feature_extractor,181        )182        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)183        self.register_to_config(requires_safety_checker=requires_safety_checker)184 185    def _encode_prompt(186        self,187        prompt,188        device,189        num_images_per_prompt,190        do_classifier_free_guidance,191        negative_prompt=None,192        prompt_embeds: Optional[torch.Tensor] = None,193        negative_prompt_embeds: Optional[torch.Tensor] = None,194    ):195        r"""196        Encodes the prompt into text encoder hidden states.197 198        Args:199             prompt (`str` or `List[str]`, *optional*):200                prompt to be encoded201            device: (`torch.device`):202                torch device203            num_images_per_prompt (`int`):204                number of images that should be generated per prompt205            do_classifier_free_guidance (`bool`):206                whether to use classifier free guidance or not207            negative_prompt (`str` or `List[str]`, *optional*):208                The prompt or prompts not to guide the image generation. If not defined, one has to pass209                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).210            prompt_embeds (`torch.Tensor`, *optional*):211                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not212                provided, text embeddings will be generated from `prompt` input argument.213            negative_prompt_embeds (`torch.Tensor`, *optional*):214                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt215                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input216                argument.217        """218        if prompt is not None and isinstance(prompt, str):219            batch_size = 1220        elif prompt is not None and isinstance(prompt, list):221            batch_size = len(prompt)222        else:223            batch_size = prompt_embeds.shape[0]224 225        if prompt_embeds is None:226            text_inputs = self.tokenizer(227                prompt,228                padding="max_length",229                max_length=self.tokenizer.model_max_length,230                truncation=True,231                return_tensors="pt",232            )233            text_input_ids = text_inputs.input_ids234            untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids235 236            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(237                text_input_ids, untruncated_ids238            ):239                removed_text = self.tokenizer.batch_decode(240                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]241                )242                logger.warning(243                    "The following part of your input was truncated because CLIP can only handle sequences up to"244                    f" {self.tokenizer.model_max_length} tokens: {removed_text}"245                )246 247            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:248                attention_mask = text_inputs.attention_mask.to(device)249            else:250                attention_mask = None251 252            prompt_embeds = self.text_encoder(253                text_input_ids.to(device),254                attention_mask=attention_mask,255            )256            prompt_embeds = prompt_embeds[0]257 258        prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)259 260        bs_embed, seq_len, _ = prompt_embeds.shape261        # duplicate text embeddings for each generation per prompt, using mps friendly method262        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)263        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)264 265        # get unconditional embeddings for classifier free guidance266        if do_classifier_free_guidance and negative_prompt_embeds is None:267            uncond_tokens: List[str]268            if negative_prompt is None:269                uncond_tokens = [""] * batch_size270            elif type(prompt) is not type(negative_prompt):271                raise TypeError(272                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="273                    f" {type(prompt)}."274                )275            elif isinstance(negative_prompt, str):276                uncond_tokens = [negative_prompt]277            elif batch_size != len(negative_prompt):278                raise ValueError(279                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"280                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"281                    " the batch size of `prompt`."282                )283            else:284                uncond_tokens = negative_prompt285 286            max_length = prompt_embeds.shape[1]287            uncond_input = self.tokenizer(288                uncond_tokens,289                padding="max_length",290                max_length=max_length,291                truncation=True,292                return_tensors="pt",293            )294 295            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:296                attention_mask = uncond_input.attention_mask.to(device)297            else:298                attention_mask = None299 300            negative_prompt_embeds = self.text_encoder(301                uncond_input.input_ids.to(device),302                attention_mask=attention_mask,303            )304            negative_prompt_embeds = negative_prompt_embeds[0]305 306        if do_classifier_free_guidance:307            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method308            seq_len = negative_prompt_embeds.shape[1]309 310            negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)311 312            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)313            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)314 315            # For classifier free guidance, we need to do two forward passes.316            # Here we concatenate the unconditional and text embeddings into a single batch317            # to avoid doing two forward passes318            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])319 320        return prompt_embeds321 322    def run_safety_checker(self, image, device, dtype):323        if self.safety_checker is not None:324            safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)325            image, has_nsfw_concept = self.safety_checker(326                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)327            )328        else:329            has_nsfw_concept = None330        return image, has_nsfw_concept331 332    def decode_latents(self, latents):333        latents = 1 / self.vae.config.scaling_factor * latents334        image = self.vae.decode(latents).sample335        image = (image / 2 + 0.5).clamp(0, 1)336        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16337        image = image.cpu().permute(0, 2, 3, 1).float().numpy()338        return image339 340    def prepare_extra_step_kwargs(self, generator, eta):341        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature342        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.343        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502344        # and should be between [0, 1]345 346        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())347        extra_step_kwargs = {}348        if accepts_eta:349            extra_step_kwargs["eta"] = eta350 351        # check if the scheduler accepts generator352        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())353        if accepts_generator:354            extra_step_kwargs["generator"] = generator355        return extra_step_kwargs356 357    def check_controlnet_conditioning_image(self, image, prompt, prompt_embeds):358        image_is_pil = isinstance(image, PIL.Image.Image)359        image_is_tensor = isinstance(image, torch.Tensor)360        image_is_pil_list = isinstance(image, list) and isinstance(image[0], PIL.Image.Image)361        image_is_tensor_list = isinstance(image, list) and isinstance(image[0], torch.Tensor)362 363        if not image_is_pil and not image_is_tensor and not image_is_pil_list and not image_is_tensor_list:364            raise TypeError(365                "image must be passed and be one of PIL image, torch tensor, list of PIL images, or list of torch tensors"366            )367 368        if image_is_pil:369            image_batch_size = 1370        elif image_is_tensor:371            image_batch_size = image.shape[0]372        elif image_is_pil_list:373            image_batch_size = len(image)374        elif image_is_tensor_list:375            image_batch_size = len(image)376        else:377            raise ValueError("controlnet condition image is not valid")378 379        if prompt is not None and isinstance(prompt, str):380            prompt_batch_size = 1381        elif prompt is not None and isinstance(prompt, list):382            prompt_batch_size = len(prompt)383        elif prompt_embeds is not None:384            prompt_batch_size = prompt_embeds.shape[0]385        else:386            raise ValueError("prompt or prompt_embeds are not valid")387 388        if image_batch_size != 1 and image_batch_size != prompt_batch_size:389            raise ValueError(390                f"If image batch size is not 1, image batch size must be same as prompt batch size. image batch size: {image_batch_size}, prompt batch size: {prompt_batch_size}"391            )392 393    def check_inputs(394        self,395        prompt,396        image,397        controlnet_conditioning_image,398        height,399        width,400        callback_steps,401        negative_prompt=None,402        prompt_embeds=None,403        negative_prompt_embeds=None,404        strength=None,405        controlnet_guidance_start=None,406        controlnet_guidance_end=None,407        controlnet_conditioning_scale=None,408    ):409        if height % 8 != 0 or width % 8 != 0:410            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")411 412        if (callback_steps is None) or (413            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)414        ):415            raise ValueError(416                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"417                f" {type(callback_steps)}."418            )419 420        if prompt is not None and prompt_embeds is not None:421            raise ValueError(422                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"423                " only forward one of the two."424            )425        elif prompt is None and prompt_embeds is None:426            raise ValueError(427                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."428            )429        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):430            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")431 432        if negative_prompt is not None and negative_prompt_embeds is not None:433            raise ValueError(434                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"435                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."436            )437 438        if prompt_embeds is not None and negative_prompt_embeds is not None:439            if prompt_embeds.shape != negative_prompt_embeds.shape:440                raise ValueError(441                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"442                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"443                    f" {negative_prompt_embeds.shape}."444                )445 446        # check controlnet condition image447 448        if isinstance(self.controlnet, ControlNetModel):449            self.check_controlnet_conditioning_image(controlnet_conditioning_image, prompt, prompt_embeds)450        elif isinstance(self.controlnet, MultiControlNetModel):451            if not isinstance(controlnet_conditioning_image, list):452                raise TypeError("For multiple controlnets: `image` must be type `list`")453 454            if len(controlnet_conditioning_image) != len(self.controlnet.nets):455                raise ValueError(456                    "For multiple controlnets: `image` must have the same length as the number of controlnets."457                )458 459            for image_ in controlnet_conditioning_image:460                self.check_controlnet_conditioning_image(image_, prompt, prompt_embeds)461        else:462            assert False463 464        # Check `controlnet_conditioning_scale`465 466        if isinstance(self.controlnet, ControlNetModel):467            if not isinstance(controlnet_conditioning_scale, float):468                raise TypeError("For single controlnet: `controlnet_conditioning_scale` must be type `float`.")469        elif isinstance(self.controlnet, MultiControlNetModel):470            if isinstance(controlnet_conditioning_scale, list) and len(controlnet_conditioning_scale) != len(471                self.controlnet.nets472            ):473                raise ValueError(474                    "For multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have"475                    " the same length as the number of controlnets"476                )477        else:478            assert False479 480        if isinstance(image, torch.Tensor):481            if image.ndim != 3 and image.ndim != 4:482                raise ValueError("`image` must have 3 or 4 dimensions")483 484            if image.ndim == 3:485                image_batch_size = 1486                image_channels, image_height, image_width = image.shape487            elif image.ndim == 4:488                image_batch_size, image_channels, image_height, image_width = image.shape489            else:490                assert False491 492            if image_channels != 3:493                raise ValueError("`image` must have 3 channels")494 495            if image.min() < -1 or image.max() > 1:496                raise ValueError("`image` should be in range [-1, 1]")497 498        if self.vae.config.latent_channels != self.unet.config.in_channels:499            raise ValueError(500                f"The config of `pipeline.unet` expects {self.unet.config.in_channels} but received"501                f" latent channels: {self.vae.config.latent_channels},"502                f" Please verify the config of `pipeline.unet` and the `pipeline.vae`"503            )504 505        if strength < 0 or strength > 1:506            raise ValueError(f"The value of `strength` should in [0.0, 1.0] but is {strength}")507 508        if controlnet_guidance_start < 0 or controlnet_guidance_start > 1:509            raise ValueError(510                f"The value of `controlnet_guidance_start` should in [0.0, 1.0] but is {controlnet_guidance_start}"511            )512 513        if controlnet_guidance_end < 0 or controlnet_guidance_end > 1:514            raise ValueError(515                f"The value of `controlnet_guidance_end` should in [0.0, 1.0] but is {controlnet_guidance_end}"516            )517 518        if controlnet_guidance_start > controlnet_guidance_end:519            raise ValueError(520                "The value of `controlnet_guidance_start` should be less than `controlnet_guidance_end`, but got"521                f" `controlnet_guidance_start` {controlnet_guidance_start} >= `controlnet_guidance_end` {controlnet_guidance_end}"522            )523 524    def get_timesteps(self, num_inference_steps, strength, device):525        # get the original timestep using init_timestep526        init_timestep = min(int(num_inference_steps * strength), num_inference_steps)527 528        t_start = max(num_inference_steps - init_timestep, 0)529        timesteps = self.scheduler.timesteps[t_start:]530 531        return timesteps, num_inference_steps - t_start532 533    def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None):534        if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)):535            raise ValueError(536                f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}"537            )538 539        image = image.to(device=device, dtype=dtype)540 541        batch_size = batch_size * num_images_per_prompt542        if isinstance(generator, list) and len(generator) != batch_size:543            raise ValueError(544                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"545                f" size of {batch_size}. Make sure the batch size matches the length of the generators."546            )547 548        if isinstance(generator, list):549            init_latents = [550                self.vae.encode(image[i : i + 1]).latent_dist.sample(generator[i]) for i in range(batch_size)551            ]552            init_latents = torch.cat(init_latents, dim=0)553        else:554            init_latents = self.vae.encode(image).latent_dist.sample(generator)555 556        init_latents = self.vae.config.scaling_factor * init_latents557 558        if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0:559            raise ValueError(560                f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts."561            )562        else:563            init_latents = torch.cat([init_latents], dim=0)564 565        shape = init_latents.shape566        noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)567 568        # get latents569        init_latents = self.scheduler.add_noise(init_latents, noise, timestep)570        latents = init_latents571 572        return latents573 574    def _default_height_width(self, height, width, image):575        if isinstance(image, list):576            image = image[0]577 578        if height is None:579            if isinstance(image, PIL.Image.Image):580                height = image.height581            elif isinstance(image, torch.Tensor):582                height = image.shape[3]583 584            height = (height // 8) * 8  # round down to nearest multiple of 8585 586        if width is None:587            if isinstance(image, PIL.Image.Image):588                width = image.width589            elif isinstance(image, torch.Tensor):590                width = image.shape[2]591 592            width = (width // 8) * 8  # round down to nearest multiple of 8593 594        return height, width595 596    @torch.no_grad()597    @replace_example_docstring(EXAMPLE_DOC_STRING)598    def __call__(599        self,600        prompt: Union[str, List[str]] = None,601        image: Union[torch.Tensor, PIL.Image.Image] = None,602        controlnet_conditioning_image: Union[603            torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]604        ] = None,605        strength: float = 0.8,606        height: Optional[int] = None,607        width: Optional[int] = None,608        num_inference_steps: int = 50,609        guidance_scale: float = 7.5,610        negative_prompt: Optional[Union[str, List[str]]] = None,611        num_images_per_prompt: Optional[int] = 1,612        eta: float = 0.0,613        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,614        latents: Optional[torch.Tensor] = None,615        prompt_embeds: Optional[torch.Tensor] = None,616        negative_prompt_embeds: Optional[torch.Tensor] = None,617        output_type: Optional[str] = "pil",618        return_dict: bool = True,619        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,620        callback_steps: int = 1,621        cross_attention_kwargs: Optional[Dict[str, Any]] = None,622        controlnet_conditioning_scale: Union[float, List[float]] = 1.0,623        controlnet_guidance_start: float = 0.0,624        controlnet_guidance_end: float = 1.0,625    ):626        r"""627        Function invoked when calling the pipeline for generation.628 629        Args:630            prompt (`str` or `List[str]`, *optional*):631                The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.632                instead.633            image (`torch.Tensor` or `PIL.Image.Image`):634                `Image`, or tensor representing an image batch which will be inpainted, *i.e.* parts of the image will635                be masked out with `mask_image` and repainted according to `prompt`.636            controlnet_conditioning_image (`torch.Tensor`, `PIL.Image.Image`, `List[torch.Tensor]` or `List[PIL.Image.Image]`):637                The ControlNet input condition. ControlNet uses this input condition to generate guidance to Unet. If638                the type is specified as `torch.Tensor`, it is passed to ControlNet as is. PIL.Image.Image` can639                also be accepted as an image. The control image is automatically resized to fit the output image.640            strength (`float`, *optional*):641                Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image`642                will be used as a starting point, adding more noise to it the larger the `strength`. The number of643                denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will644                be maximum and the denoising process will run for the full number of iterations specified in645                `num_inference_steps`. A value of 1, therefore, essentially ignores `image`.646            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):647                The height in pixels of the generated image.648            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):649                The width in pixels of the generated image.650            num_inference_steps (`int`, *optional*, defaults to 50):651                The number of denoising steps. More denoising steps usually lead to a higher quality image at the652                expense of slower inference.653            guidance_scale (`float`, *optional*, defaults to 7.5):654                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).655                `guidance_scale` is defined as `w` of equation 2. of [Imagen656                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >657                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,658                usually at the expense of lower image quality.659            negative_prompt (`str` or `List[str]`, *optional*):660                The prompt or prompts not to guide the image generation. If not defined, one has to pass661                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).662            num_images_per_prompt (`int`, *optional*, defaults to 1):663                The number of images to generate per prompt.664            eta (`float`, *optional*, defaults to 0.0):665                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to666                [`schedulers.DDIMScheduler`], will be ignored for others.667            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):668                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)669                to make generation deterministic.670            latents (`torch.Tensor`, *optional*):671                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image672                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents673                tensor will ge generated by sampling using the supplied random `generator`.674            prompt_embeds (`torch.Tensor`, *optional*):675                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not676                provided, text embeddings will be generated from `prompt` input argument.677            negative_prompt_embeds (`torch.Tensor`, *optional*):678                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt679                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input680                argument.681            output_type (`str`, *optional*, defaults to `"pil"`):682                The output format of the generate image. Choose between683                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.684            return_dict (`bool`, *optional*, defaults to `True`):685                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a686                plain tuple.687            callback (`Callable`, *optional*):688                A function that will be called every `callback_steps` steps during inference. The function will be689                called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.690            callback_steps (`int`, *optional*, defaults to 1):691                The frequency at which the `callback` function will be called. If not specified, the callback will be692                called at every step.693            cross_attention_kwargs (`dict`, *optional*):694                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under695                `self.processor` in696                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).697            controlnet_conditioning_scale (`float`, *optional*, defaults to 1.0):698                The outputs of the controlnet are multiplied by `controlnet_conditioning_scale` before they are added699                to the residual in the original unet.700            controlnet_guidance_start ('float', *optional*, defaults to 0.0):701                The percentage of total steps the controlnet starts applying. Must be between 0 and 1.702            controlnet_guidance_end ('float', *optional*, defaults to 1.0):703                The percentage of total steps the controlnet ends applying. Must be between 0 and 1. Must be greater704                than `controlnet_guidance_start`.705 706        Examples:707 708        Returns:709            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:710            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.711            When returning a tuple, the first element is a list with the generated images, and the second element is a712            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"713            (nsfw) content, according to the `safety_checker`.714        """715        # 0. Default height and width to unet716        height, width = self._default_height_width(height, width, controlnet_conditioning_image)717 718        # 1. Check inputs. Raise error if not correct719        self.check_inputs(720            prompt,721            image,722            controlnet_conditioning_image,723            height,724            width,725            callback_steps,726            negative_prompt,727            prompt_embeds,728            negative_prompt_embeds,729            strength,730            controlnet_guidance_start,731            controlnet_guidance_end,732            controlnet_conditioning_scale,733        )734 735        # 2. Define call parameters736        if prompt is not None and isinstance(prompt, str):737            batch_size = 1738        elif prompt is not None and isinstance(prompt, list):739            batch_size = len(prompt)740        else:741            batch_size = prompt_embeds.shape[0]742 743        device = self._execution_device744        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)745        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`746        # corresponds to doing no classifier free guidance.747        do_classifier_free_guidance = guidance_scale > 1.0748 749        if isinstance(self.controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float):750            controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(self.controlnet.nets)751 752        # 3. Encode input prompt753        prompt_embeds = self._encode_prompt(754            prompt,755            device,756            num_images_per_prompt,757            do_classifier_free_guidance,758            negative_prompt,759            prompt_embeds=prompt_embeds,760            negative_prompt_embeds=negative_prompt_embeds,761        )762 763        # 4. Prepare image, and controlnet_conditioning_image764        image = prepare_image(image)765 766        # condition image(s)767        if isinstance(self.controlnet, ControlNetModel):768            controlnet_conditioning_image = prepare_controlnet_conditioning_image(769                controlnet_conditioning_image=controlnet_conditioning_image,770                width=width,771                height=height,772                batch_size=batch_size * num_images_per_prompt,773                num_images_per_prompt=num_images_per_prompt,774                device=device,775                dtype=self.controlnet.dtype,776                do_classifier_free_guidance=do_classifier_free_guidance,777            )778        elif isinstance(self.controlnet, MultiControlNetModel):779            controlnet_conditioning_images = []780 781            for image_ in controlnet_conditioning_image:782                image_ = prepare_controlnet_conditioning_image(783                    controlnet_conditioning_image=image_,784                    width=width,785                    height=height,786                    batch_size=batch_size * num_images_per_prompt,787                    num_images_per_prompt=num_images_per_prompt,788                    device=device,789                    dtype=self.controlnet.dtype,790                    do_classifier_free_guidance=do_classifier_free_guidance,791                )792 793                controlnet_conditioning_images.append(image_)794 795            controlnet_conditioning_image = controlnet_conditioning_images796        else:797            assert False798 799        # 5. Prepare timesteps800        self.scheduler.set_timesteps(num_inference_steps, device=device)801        timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)802        latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)803 804        # 6. Prepare latent variables805        if latents is None:806            latents = self.prepare_latents(807                image,808                latent_timestep,809                batch_size,810                num_images_per_prompt,811                prompt_embeds.dtype,812                device,813                generator,814            )815 816        # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline817        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)818 819        # 8. Denoising loop820        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order821        with self.progress_bar(total=num_inference_steps) as progress_bar:822            for i, t in enumerate(timesteps):823                # expand the latents if we are doing classifier free guidance824                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents825 826                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)827 828                # compute the percentage of total steps we are at829                current_sampling_percent = i / len(timesteps)830 831                if (832                    current_sampling_percent < controlnet_guidance_start833                    or current_sampling_percent > controlnet_guidance_end834                ):835                    # do not apply the controlnet836                    down_block_res_samples = None837                    mid_block_res_sample = None838                else:839                    # apply the controlnet840                    down_block_res_samples, mid_block_res_sample = self.controlnet(841                        latent_model_input,842                        t,843                        encoder_hidden_states=prompt_embeds,844                        controlnet_cond=controlnet_conditioning_image,845                        conditioning_scale=controlnet_conditioning_scale,846                        return_dict=False,847                    )848 849                # predict the noise residual850                noise_pred = self.unet(851                    latent_model_input,852                    t,853                    encoder_hidden_states=prompt_embeds,854                    cross_attention_kwargs=cross_attention_kwargs,855                    down_block_additional_residuals=down_block_res_samples,856                    mid_block_additional_residual=mid_block_res_sample,857                ).sample858 859                # perform guidance860                if do_classifier_free_guidance:861                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)862                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)863 864                # compute the previous noisy sample x_t -> x_t-1865                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample866 867                # call the callback, if provided868                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):869                    progress_bar.update()870                    if callback is not None and i % callback_steps == 0:871                        step_idx = i // getattr(self.scheduler, "order", 1)872                        callback(step_idx, t, latents)873 874        # If we do sequential model offloading, let's offload unet and controlnet875        # manually for max memory savings876        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:877            self.unet.to("cpu")878            self.controlnet.to("cpu")879            torch.cuda.empty_cache()880 881        if output_type == "latent":882            image = latents883            has_nsfw_concept = None884        elif output_type == "pil":885            # 8. Post-processing886            image = self.decode_latents(latents)887 888            # 9. Run safety checker889            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)890 891            # 10. Convert to PIL892            image = self.numpy_to_pil(image)893        else:894            # 8. Post-processing895            image = self.decode_latents(latents)896 897            # 9. Run safety checker898            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)899 900        # Offload last model to CPU901        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:902            self.final_offload_hook.offload()903 904        if not return_dict:905            return (image, has_nsfw_concept)906 907        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)908