junnyu/webui_controlnet_ppdiffusers
0
1# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.2# Copyright 2023 The HuggingFace Team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15#16# modified from https://github.com/AUTOMATIC1111/stable-diffusion-webui17# Here is the AGPL-3.0 license https://github.com/AUTOMATIC1111/stable-diffusion-webui/blob/master/LICENSE.txt18 19import copy20import inspect21import os22import os.path23import shutil24from pathlib import Path25from typing import Any, Callable, Dict, List, Optional, Union26 27import paddle28import paddle.nn as nn29import PIL30import PIL.Image31from huggingface_hub.file_download import _request_wrapper, hf_raise_for_status32 33from paddlenlp.transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer34from ppdiffusers.models import AutoencoderKL, ControlNetModel, UNet2DConditionModel35from ppdiffusers.pipelines.pipeline_utils import DiffusionPipeline36from ppdiffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput37from ppdiffusers.pipelines.stable_diffusion.safety_checker import (38 StableDiffusionSafetyChecker,39)40from ppdiffusers.schedulers import KarrasDiffusionSchedulers41from ppdiffusers.utils import (42 PIL_INTERPOLATION,43 PPDIFFUSERS_CACHE,44 logging,45 ppdiffusers_url_download,46 randn_tensor,47 safetensors_load,48 smart_load,49 torch_load,50)51 52 53def get_civitai_download_url(display_url, url_prefix="https://civitai.com"):54 if "api/download" in display_url:55 return display_url56 import bs457 import requests58 59 headers = {60 "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE"61 }62 r = requests.get(display_url, headers=headers)63 soup = bs4.BeautifulSoup(r.text, "lxml")64 download_url = None65 for a in soup.find_all("a", href=True):66 if "Download" in str(a):67 download_url = url_prefix + a["href"].split("?")[0]68 break69 return download_url70 71 72def http_file_name(73 url: str,74 *,75 proxies=None,76 headers: Optional[Dict[str, str]] = None,77 timeout=10.0,78 max_retries=0,79):80 """81 Get a remote file name.82 """83 headers = copy.deepcopy(headers) or {}84 r = _request_wrapper(85 method="GET",86 url=url,87 stream=True,88 proxies=proxies,89 headers=headers,90 timeout=timeout,91 max_retries=max_retries,92 )93 hf_raise_for_status(r)94 displayed_name = url95 content_disposition = r.headers.get("Content-Disposition")96 if content_disposition is not None and "filename=" in content_disposition:97 # Means file is on CDN98 displayed_name = content_disposition.split("filename=")[-1]99 return displayed_name100 101 102@paddle.no_grad()103def load_lora(104 pipeline,105 state_dict: dict,106 LORA_PREFIX_UNET: str = "lora_unet",107 LORA_PREFIX_TEXT_ENCODER: str = "lora_te",108 ratio: float = 1.0,109):110 ratio = float(ratio)111 visited = []112 for key in state_dict:113 if ".alpha" in key or ".lora_up" in key or key in visited:114 continue115 116 if "text" in key:117 tmp_layer_infos = key.split(".")[0].split(LORA_PREFIX_TEXT_ENCODER + "_")[-1].split("_")118 hf_to_ppnlp = {119 "encoder": "transformer",120 "fc1": "linear1",121 "fc2": "linear2",122 }123 layer_infos = []124 for layer_info in tmp_layer_infos:125 if layer_info == "mlp":126 continue127 layer_infos.append(hf_to_ppnlp.get(layer_info, layer_info))128 curr_layer: paddle.nn.Linear = pipeline.text_encoder129 else:130 layer_infos = key.split(".")[0].split(LORA_PREFIX_UNET + "_")[-1].split("_")131 curr_layer: paddle.nn.Linear = pipeline.unet132 133 temp_name = layer_infos.pop(0)134 while len(layer_infos) > -1:135 try:136 if temp_name == "to":137 raise ValueError()138 curr_layer = curr_layer.__getattr__(temp_name)139 if len(layer_infos) > 0:140 temp_name = layer_infos.pop(0)141 elif len(layer_infos) == 0:142 break143 except Exception:144 if len(temp_name) > 0:145 temp_name += "_" + layer_infos.pop(0)146 else:147 temp_name = layer_infos.pop(0)148 149 triplet_keys = [key, key.replace("lora_down", "lora_up"), key.replace("lora_down.weight", "alpha")]150 dtype: paddle.dtype = curr_layer.weight.dtype151 weight_down: paddle.Tensor = state_dict[triplet_keys[0]].cast(dtype)152 weight_up: paddle.Tensor = state_dict[triplet_keys[1]].cast(dtype)153 rank: float = float(weight_down.shape[0])154 if triplet_keys[2] in state_dict:155 alpha: float = state_dict[triplet_keys[2]].cast(dtype).item()156 scale: float = alpha / rank157 else:158 scale = 1.0159 160 if not hasattr(curr_layer, "backup_weights"):161 curr_layer.backup_weights = curr_layer.weight.clone()162 163 if len(weight_down.shape) == 4:164 if weight_down.shape[2:4] == [1, 1]:165 # conv2d 1x1166 curr_layer.weight.copy_(167 curr_layer.weight168 + ratio169 * paddle.matmul(weight_up.squeeze([-1, -2]), weight_down.squeeze([-1, -2])).unsqueeze([-1, -2])170 * scale,171 True,172 )173 else:174 # conv2d 3x3175 curr_layer.weight.copy_(176 curr_layer.weight177 + ratio178 * paddle.nn.functional.conv2d(weight_down.transpose([1, 0, 2, 3]), weight_up).transpose(179 [1, 0, 2, 3]180 )181 * scale,182 True,183 )184 else:185 # linear186 curr_layer.weight.copy_(curr_layer.weight + ratio * paddle.matmul(weight_up, weight_down).T * scale, True)187 188 # update visited list189 visited.extend(triplet_keys)190 return pipeline191 192 193logger = logging.get_logger(__name__) # pylint: disable=invalid-name194 195 196class WebUIStableDiffusionControlNetPipeline(DiffusionPipeline):197 r"""198 Pipeline for text-to-image generation using Stable Diffusion with ControlNet guidance.199 This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the200 library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)201 Args:202 vae ([`AutoencoderKL`]):203 Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.204 text_encoder ([`CLIPTextModel`]):205 Frozen text-encoder. Stable Diffusion uses the text portion of206 [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically207 the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.208 tokenizer (`CLIPTokenizer`):209 Tokenizer of class210 [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).211 unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.212 controlnet ([`ControlNetModel`]):213 Provides additional conditioning to the unet during the denoising process.214 scheduler ([`SchedulerMixin`]):215 A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of216 [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].217 safety_checker ([`StableDiffusionSafetyChecker`]):218 Classification module that estimates whether generated images could be considered offensive or harmful.219 Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details.220 feature_extractor ([`CLIPFeatureExtractor`]):221 Model that extracts features from generated images to be used as inputs for the `safety_checker`.222 """223 _optional_components = ["safety_checker", "feature_extractor"]224 enable_emphasis = True225 comma_padding_backtrack = 20226 227 LORA_DIR = os.path.join(PPDIFFUSERS_CACHE, "lora")228 TI_DIR = os.path.join(PPDIFFUSERS_CACHE, "textual_inversion")229 230 def __init__(231 self,232 vae: AutoencoderKL,233 text_encoder: CLIPTextModel,234 tokenizer: CLIPTokenizer,235 unet: UNet2DConditionModel,236 controlnet: ControlNetModel,237 scheduler: KarrasDiffusionSchedulers,238 safety_checker: StableDiffusionSafetyChecker,239 feature_extractor: CLIPFeatureExtractor,240 requires_safety_checker: bool = True,241 ):242 super().__init__()243 244 if safety_checker is None and requires_safety_checker:245 logger.warning(246 f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"247 " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"248 " results in services or applications open to the public. PaddleNLP team, diffusers team and Hugging Face"249 " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"250 " it only for use-cases that involve analyzing network behavior or auditing its results. For more"251 " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."252 )253 254 if safety_checker is not None and feature_extractor is None:255 raise ValueError(256 f"Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"257 " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."258 )259 260 self.register_modules(261 vae=vae,262 text_encoder=text_encoder,263 tokenizer=tokenizer,264 unet=unet,265 controlnet=controlnet,266 scheduler=scheduler,267 safety_checker=safety_checker,268 feature_extractor=feature_extractor,269 )270 self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)271 self.register_to_config(requires_safety_checker=requires_safety_checker)272 273 # custom data274 clip_model = FrozenCLIPEmbedder(text_encoder, tokenizer)275 self.sj = StableDiffusionModelHijack(clip_model)276 self.orginal_scheduler_config = self.scheduler.config277 self.supported_scheduler = [278 "pndm",279 "lms",280 "euler",281 "euler-ancestral",282 "dpm-multi",283 "dpm-single",284 "unipc-multi",285 "ddim",286 "ddpm",287 "deis-multi",288 "heun",289 "kdpm2-ancestral",290 "kdpm2",291 ]292 self.weights_has_changed = False293 294 # register_state_dict_hook to fix text_encoder, when we save_pretrained text model.295 def map_to(state_dict, *args, **kwargs):296 if "text_model.token_embedding.wrapped.weight" in state_dict:297 state_dict["text_model.token_embedding.weight"] = state_dict.pop(298 "text_model.token_embedding.wrapped.weight"299 )300 return state_dict301 302 self.text_encoder.register_state_dict_hook(map_to)303 304 def add_ti_embedding_dir(self, embeddings_dir=None):305 self.sj.embedding_db.add_embedding_dir(embeddings_dir)306 self.sj.embedding_db.load_textual_inversion_embeddings()307 308 def clear_ti_embedding(self):309 self.sj.embedding_db.clear_embedding_dirs()310 self.sj.embedding_db.load_textual_inversion_embeddings(True)311 312 def download_civitai_lora_file(self, url):313 if os.path.isfile(url):314 dst = os.path.join(self.LORA_DIR, os.path.basename(url))315 shutil.copyfile(url, dst)316 return dst317 318 download_url = get_civitai_download_url(url) or url319 file_path = ppdiffusers_url_download(320 download_url, cache_dir=self.LORA_DIR, filename=http_file_name(download_url).strip('"')321 )322 return file_path323 324 def download_civitai_ti_file(self, url):325 if os.path.isfile(url):326 dst = os.path.join(self.TI_DIR, os.path.basename(url))327 shutil.copyfile(url, dst)328 return dst329 330 download_url = get_civitai_download_url(url) or url331 file_path = ppdiffusers_url_download(332 download_url, cache_dir=self.TI_DIR, filename=http_file_name(download_url).strip('"')333 )334 return file_path335 336 def change_scheduler(self, scheduler_type="ddim"):337 self.switch_scheduler(scheduler_type)338 339 def switch_scheduler(self, scheduler_type="ddim"):340 scheduler_type = scheduler_type.lower()341 from ppdiffusers import (342 DDIMScheduler,343 DDPMScheduler,344 DEISMultistepScheduler,345 DPMSolverMultistepScheduler,346 DPMSolverSinglestepScheduler,347 EulerAncestralDiscreteScheduler,348 EulerDiscreteScheduler,349 HeunDiscreteScheduler,350 KDPM2AncestralDiscreteScheduler,351 KDPM2DiscreteScheduler,352 LMSDiscreteScheduler,353 PNDMScheduler,354 UniPCMultistepScheduler,355 )356 357 if scheduler_type == "pndm":358 scheduler = PNDMScheduler.from_config(self.orginal_scheduler_config, skip_prk_steps=True)359 elif scheduler_type == "lms":360 scheduler = LMSDiscreteScheduler.from_config(self.orginal_scheduler_config)361 elif scheduler_type == "heun":362 scheduler = HeunDiscreteScheduler.from_config(self.orginal_scheduler_config)363 elif scheduler_type == "euler":364 scheduler = EulerDiscreteScheduler.from_config(self.orginal_scheduler_config)365 elif scheduler_type == "euler-ancestral":366 scheduler = EulerAncestralDiscreteScheduler.from_config(self.orginal_scheduler_config)367 elif scheduler_type == "dpm-multi":368 scheduler = DPMSolverMultistepScheduler.from_config(self.orginal_scheduler_config)369 elif scheduler_type == "dpm-single":370 scheduler = DPMSolverSinglestepScheduler.from_config(self.orginal_scheduler_config)371 elif scheduler_type == "kdpm2-ancestral":372 scheduler = KDPM2AncestralDiscreteScheduler.from_config(self.orginal_scheduler_config)373 elif scheduler_type == "kdpm2":374 scheduler = KDPM2DiscreteScheduler.from_config(self.orginal_scheduler_config)375 elif scheduler_type == "unipc-multi":376 scheduler = UniPCMultistepScheduler.from_config(self.orginal_scheduler_config)377 elif scheduler_type == "ddim":378 scheduler = DDIMScheduler.from_config(379 self.orginal_scheduler_config,380 steps_offset=1,381 clip_sample=False,382 set_alpha_to_one=False,383 )384 elif scheduler_type == "ddpm":385 scheduler = DDPMScheduler.from_config(386 self.orginal_scheduler_config,387 )388 elif scheduler_type == "deis-multi":389 scheduler = DEISMultistepScheduler.from_config(390 self.orginal_scheduler_config,391 )392 else:393 raise ValueError(394 f"Scheduler of type {scheduler_type} doesn't exist! Please choose in {self.supported_scheduler}!"395 )396 self.scheduler = scheduler397 398 @paddle.no_grad()399 def _encode_prompt(400 self,401 prompt: str,402 do_classifier_free_guidance: float = 7.5,403 negative_prompt: str = None,404 num_inference_steps: int = 50,405 ):406 if do_classifier_free_guidance:407 assert isinstance(negative_prompt, str)408 negative_prompt = [negative_prompt]409 uc = get_learned_conditioning(self.sj.clip, negative_prompt, num_inference_steps)410 else:411 uc = None412 413 c = get_multicond_learned_conditioning(self.sj.clip, prompt, num_inference_steps)414 return c, uc415 416 def run_safety_checker(self, image, dtype):417 if self.safety_checker is not None:418 safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pd")419 image, has_nsfw_concept = self.safety_checker(420 images=image, clip_input=safety_checker_input.pixel_values.cast(dtype)421 )422 else:423 has_nsfw_concept = None424 return image, has_nsfw_concept425 426 def decode_latents(self, latents):427 latents = 1 / self.vae.config.scaling_factor * latents428 image = self.vae.decode(latents).sample429 image = (image / 2 + 0.5).clip(0, 1)430 # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16431 image = image.transpose([0, 2, 3, 1]).cast("float32").numpy()432 return image433 434 def prepare_extra_step_kwargs(self, generator, eta):435 # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature436 # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.437 # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502438 # and should be between [0, 1]439 440 accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())441 extra_step_kwargs = {}442 if accepts_eta:443 extra_step_kwargs["eta"] = eta444 445 # check if the scheduler accepts generator446 accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())447 if accepts_generator:448 extra_step_kwargs["generator"] = generator449 return extra_step_kwargs450 451 def check_inputs(452 self,453 prompt,454 image,455 height,456 width,457 callback_steps,458 negative_prompt=None,459 controlnet_conditioning_scale=1.0,460 ):461 if height % 8 != 0 or width % 8 != 0:462 raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")463 464 if (callback_steps is None) or (465 callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)466 ):467 raise ValueError(468 f"`callback_steps` has to be a positive integer but is {callback_steps} of type"469 f" {type(callback_steps)}."470 )471 472 if prompt is not None and not isinstance(prompt, str):473 raise ValueError(f"`prompt` has to be of type `str` but is {type(prompt)}")474 475 if negative_prompt is not None and not isinstance(negative_prompt, str):476 raise ValueError(f"`negative_prompt` has to be of type `str` but is {type(negative_prompt)}")477 478 # Check `image`479 480 if isinstance(self.controlnet, ControlNetModel):481 self.check_image(image, prompt)482 else:483 assert False484 485 # Check `controlnet_conditioning_scale`486 if isinstance(self.controlnet, ControlNetModel):487 if not isinstance(controlnet_conditioning_scale, (float, list, tuple)):488 raise TypeError(489 "For single controlnet: `controlnet_conditioning_scale` must be type `float, list(float) or tuple(float)`."490 )491 492 def check_image(self, image, prompt):493 image_is_pil = isinstance(image, PIL.Image.Image)494 image_is_tensor = isinstance(image, paddle.Tensor)495 image_is_pil_list = isinstance(image, list) and isinstance(image[0], PIL.Image.Image)496 image_is_tensor_list = isinstance(image, list) and isinstance(image[0], paddle.Tensor)497 498 if not image_is_pil and not image_is_tensor and not image_is_pil_list and not image_is_tensor_list:499 raise TypeError(500 "image must be one of PIL image, paddle tensor, list of PIL images, or list of paddle tensors"501 )502 503 if image_is_pil:504 image_batch_size = 1505 elif image_is_tensor:506 image_batch_size = image.shape[0]507 elif image_is_pil_list:508 image_batch_size = len(image)509 elif image_is_tensor_list:510 image_batch_size = len(image)511 512 if prompt is not None and isinstance(prompt, str):513 prompt_batch_size = 1514 elif prompt is not None and isinstance(prompt, list):515 prompt_batch_size = len(prompt)516 517 if image_batch_size != 1 and image_batch_size != prompt_batch_size:518 raise ValueError(519 f"If image batch size is not 1, image batch size must be same as prompt batch size. image batch size: {image_batch_size}, prompt batch size: {prompt_batch_size}"520 )521 522 def prepare_image(self, image, width, height, dtype):523 if not isinstance(image, paddle.Tensor):524 if isinstance(image, PIL.Image.Image):525 image = [image]526 527 if isinstance(image[0], PIL.Image.Image):528 images = []529 for image_ in image:530 image_ = image_.convert("RGB")531 image_ = image_.resize((width, height), resample=PIL_INTERPOLATION["lanczos"])532 image_ = np.array(image_)533 image_ = image_[None, :]534 images.append(image_)535 536 image = np.concatenate(images, axis=0)537 image = np.array(image).astype(np.float32) / 255.0538 image = image.transpose(0, 3, 1, 2)539 image = paddle.to_tensor(image)540 elif isinstance(image[0], paddle.Tensor):541 image = paddle.concat(image, axis=0)542 543 image = image.cast(dtype)544 return image545 546 def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, generator, latents=None):547 shape = [batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor]548 if isinstance(generator, list) and len(generator) != batch_size:549 raise ValueError(550 f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"551 f" size of {batch_size}. Make sure the batch size matches the length of the generators."552 )553 554 if latents is None:555 latents = randn_tensor(shape, generator=generator, dtype=dtype)556 557 # scale the initial noise by the standard deviation required by the scheduler558 latents = latents * self.scheduler.init_noise_sigma559 return latents560 561 def _default_height_width(self, height, width, image):562 while isinstance(image, list):563 image = image[0]564 565 if height is None:566 if isinstance(image, PIL.Image.Image):567 height = image.height568 elif isinstance(image, paddle.Tensor):569 height = image.shape[3]570 571 height = (height // 8) * 8 # round down to nearest multiple of 8572 573 if width is None:574 if isinstance(image, PIL.Image.Image):575 width = image.width576 elif isinstance(image, paddle.Tensor):577 width = image.shape[2]578 579 width = (width // 8) * 8 # round down to nearest multiple of 8580 581 return height, width582 583 @paddle.no_grad()584 def __call__(585 self,586 prompt: str = None,587 image: PIL.Image.Image = None,588 height: Optional[int] = None,589 width: Optional[int] = None,590 num_inference_steps: int = 50,591 guidance_scale: float = 7.5,592 negative_prompt: str = None,593 eta: float = 0.0,594 generator: Optional[Union[paddle.Generator, List[paddle.Generator]]] = None,595 latents: Optional[paddle.Tensor] = None,596 output_type: Optional[str] = "pil",597 return_dict: bool = True,598 callback: Optional[Callable[[int, int, paddle.Tensor], None]] = None,599 callback_steps: Optional[int] = 1,600 cross_attention_kwargs: Optional[Dict[str, Any]] = None,601 clip_skip: int = 1,602 controlnet_conditioning_scale: Union[float, List[float]] = 1.0,603 enable_lora: bool = True,604 ):605 r"""606 Function invoked when calling the pipeline for generation.607 608 Args:609 prompt (`str`, *optional*):610 The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.611 instead.612 image (`paddle.Tensor`, `PIL.Image.Image`):613 The ControlNet input condition. ControlNet uses this input condition to generate guidance to Unet. If614 the type is specified as `paddle.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can615 also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If616 height and/or width are passed, `image` is resized according to them. If multiple ControlNets are617 specified in init, images must be passed as a list such that each element of the list can be correctly618 batched for input to a single controlnet.619 height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):620 The height in pixels of the generated image.621 width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):622 The width in pixels of the generated image.623 num_inference_steps (`int`, *optional*, defaults to 50):624 The number of denoising steps. More denoising steps usually lead to a higher quality image at the625 expense of slower inference.626 guidance_scale (`float`, *optional*, defaults to 7.5):627 Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).628 `guidance_scale` is defined as `w` of equation 2. of [Imagen629 Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >630 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,631 usually at the expense of lower image quality.632 negative_prompt (`str`, *optional*):633 The prompt or prompts not to guide the image generation. If not defined, one has to pass634 `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.635 Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).636 eta (`float`, *optional*, defaults to 0.0):637 Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to638 [`schedulers.DDIMScheduler`], will be ignored for others.639 generator (`paddle.Generator` or `List[paddle.Generator]`, *optional*):640 One or a list of paddle generator(s) to make generation deterministic.641 latents (`paddle.Tensor`, *optional*):642 Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image643 generation. Can be used to tweak the same generation with different prompts. If not provided, a latents644 tensor will ge generated by sampling using the supplied random `generator`.645 output_type (`str`, *optional*, defaults to `"pil"`):646 The output format of the generate image. Choose between647 [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.648 return_dict (`bool`, *optional*, defaults to `True`):649 Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a650 plain tuple.651 callback (`Callable`, *optional*):652 A function that will be called every `callback_steps` steps during inference. The function will be653 called with the following arguments: `callback(step: int, timestep: int, latents: paddle.Tensor)`.654 callback_steps (`int`, *optional*, defaults to 1):655 The frequency at which the `callback` function will be called. If not specified, the callback will be656 called at every step.657 cross_attention_kwargs (`dict`, *optional*):658 A kwargs dictionary that if specified is passed along to the `AttnProcessor` as defined under659 `self.processor` in660 [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py).661 clip_skip (`int`, *optional*, defaults to 1):662 CLIP_stop_at_last_layers, if clip_skip <= 1, we will use the last_hidden_state from text_encoder.663 controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0):664 The outputs of the controlnet are multiplied by `controlnet_conditioning_scale` before they are added665 to the residual in the original unet. If multiple ControlNets are specified in init, you can set the666 corresponding scale as a list.667 Examples:668 669 Returns:670 [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:671 [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.672 When returning a tuple, the first element is a list with the generated images, and the second element is a673 list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"674 (nsfw) content, according to the `safety_checker`.675 """676 self.add_ti_embedding_dir(self.TI_DIR)677 678 try:679 # 0. Default height and width to unet680 height, width = self._default_height_width(height, width, image)681 682 # 1. Check inputs. Raise error if not correct683 self.check_inputs(684 prompt,685 image,686 height,687 width,688 callback_steps,689 negative_prompt,690 controlnet_conditioning_scale,691 )692 693 batch_size = 1694 695 image = self.prepare_image(696 image=image,697 width=width,698 height=height,699 dtype=self.controlnet.dtype,700 )701 702 # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)703 # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`704 # corresponds to doing no classifier free guidance.705 do_classifier_free_guidance = guidance_scale > 1.0706 707 prompts, extra_network_data = parse_prompts([prompt])708 709 if enable_lora and self.LORA_DIR is not None:710 if os.path.exists(self.LORA_DIR):711 lora_mapping = {p.stem: p.absolute() for p in Path(self.LORA_DIR).glob("*.safetensors")}712 for params in extra_network_data["lora"]:713 assert len(params.items) > 0714 name = params.items[0]715 if name in lora_mapping:716 ratio = float(params.items[1]) if len(params.items) > 1 else 1.0717 lora_state_dict = smart_load(lora_mapping[name], map_location=paddle.get_device())718 self.weights_has_changed = True719 load_lora(self, state_dict=lora_state_dict, ratio=ratio)720 del lora_state_dict721 else:722 print(f"We can't find lora weight: {name}! Please make sure that exists!")723 else:724 if len(extra_network_data["lora"]) > 0:725 print(f"{self.LORA_DIR} not exists, so we cant load loras!")726 727 self.sj.clip.CLIP_stop_at_last_layers = clip_skip728 # 3. Encode input prompt729 prompt_embeds, negative_prompt_embeds = self._encode_prompt(730 prompts,731 do_classifier_free_guidance,732 negative_prompt,733 num_inference_steps=num_inference_steps,734 )735 736 # 4. Prepare timesteps737 self.scheduler.set_timesteps(num_inference_steps)738 timesteps = self.scheduler.timesteps739 740 # 5. Prepare latent variables741 num_channels_latents = self.unet.in_channels742 latents = self.prepare_latents(743 batch_size,744 num_channels_latents,745 height,746 width,747 self.unet.dtype,748 generator,749 latents,750 )751 752 # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline753 extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)754 755 # 7. Denoising loop756 num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order757 with self.progress_bar(total=num_inference_steps) as progress_bar:758 for i, t in enumerate(timesteps):759 step = i // self.scheduler.order760 do_batch = False761 conds_list, cond_tensor = reconstruct_multicond_batch(prompt_embeds, step)762 try:763 weight = conds_list[0][0][1]764 except Exception:765 weight = 1.0766 if do_classifier_free_guidance:767 uncond_tensor = reconstruct_cond_batch(negative_prompt_embeds, step)768 do_batch = cond_tensor.shape[1] == uncond_tensor.shape[1]769 770 # expand the latents if we are doing classifier free guidance771 latent_model_input = paddle.concat([latents] * 2) if do_batch else latents772 latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)773 774 if do_batch:775 encoder_hidden_states = paddle.concat([uncond_tensor, cond_tensor])776 down_block_res_samples, mid_block_res_sample = self.controlnet(777 latent_model_input,778 t,779 encoder_hidden_states=encoder_hidden_states,780 controlnet_cond=paddle.concat([image, image]),781 conditioning_scale=controlnet_conditioning_scale,782 return_dict=False,783 )784 noise_pred = self.unet(785 latent_model_input,786 t,787 encoder_hidden_states=encoder_hidden_states,788 cross_attention_kwargs=cross_attention_kwargs,789 down_block_additional_residuals=down_block_res_samples,790 mid_block_additional_residual=mid_block_res_sample,791 ).sample792 noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)793 noise_pred = noise_pred_uncond + weight * guidance_scale * (794 noise_pred_text - noise_pred_uncond795 )796 else:797 down_block_res_samples, mid_block_res_sample = self.controlnet(798 latent_model_input,799 t,800 encoder_hidden_states=cond_tensor,801 controlnet_cond=image,802 conditioning_scale=controlnet_conditioning_scale,803 return_dict=False,804 )805 noise_pred = self.unet(806 latent_model_input,807 t,808 encoder_hidden_states=cond_tensor,809 cross_attention_kwargs=cross_attention_kwargs,810 down_block_additional_residuals=down_block_res_samples,811 mid_block_additional_residual=mid_block_res_sample,812 ).sample813 814 if do_classifier_free_guidance:815 down_block_res_samples, mid_block_res_sample = self.controlnet(816 latent_model_input,817 t,818 encoder_hidden_states=uncond_tensor,819 controlnet_cond=image,820 conditioning_scale=controlnet_conditioning_scale,821 return_dict=False,822 )823 noise_pred_uncond = self.unet(824 latent_model_input,825 t,826 encoder_hidden_states=uncond_tensor,827 cross_attention_kwargs=cross_attention_kwargs,828 down_block_additional_residuals=down_block_res_samples,829 mid_block_additional_residual=mid_block_res_sample,830 ).sample831 noise_pred = noise_pred_uncond + weight * guidance_scale * (noise_pred - noise_pred_uncond)832 833 # compute the previous noisy sample x_t -> x_t-1834 latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample835 836 # call the callback, if provided837 if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):838 progress_bar.update()839 if callback is not None and i % callback_steps == 0:840 callback(i, t, latents)841 842 if output_type == "latent":843 image = latents844 has_nsfw_concept = None845 elif output_type == "pil":846 # 8. Post-processing847 image = self.decode_latents(latents)848 849 # 9. Run safety checker850 image, has_nsfw_concept = self.run_safety_checker(image, self.unet.dtype)851 852 # 10. Convert to PIL853 image = self.numpy_to_pil(image)854 else:855 # 8. Post-processing856 image = self.decode_latents(latents)857 858 # 9. Run safety checker859 image, has_nsfw_concept = self.run_safety_checker(image, self.unet.dtype)860 861 if not return_dict:862 return (image, has_nsfw_concept)863 864 return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)865 except Exception as e:866 raise ValueError(e)867 finally:868 if enable_lora and self.weights_has_changed:869 for sub_layer in self.text_encoder.sublayers(include_self=True):870 if hasattr(sub_layer, "backup_weights"):871 sub_layer.weight.copy_(sub_layer.backup_weights, True)872 for sub_layer in self.unet.sublayers(include_self=True):873 if hasattr(sub_layer, "backup_weights"):874 sub_layer.weight.copy_(sub_layer.backup_weights, True)875 self.weights_has_changed = False876 877 878# clip.py879import math880from collections import namedtuple881 882 883class PromptChunk:884 """885 This object contains token ids, weight (multipliers:1.4) and textual inversion embedding info for a chunk of prompt.886 If a prompt is short, it is represented by one PromptChunk, otherwise, multiple are necessary.887 Each PromptChunk contains an exact amount of tokens - 77, which includes one for start and end token,888 so just 75 tokens from prompt.889 """890 891 def __init__(self):892 self.tokens = []893 self.multipliers = []894 self.fixes = []895 896 897PromptChunkFix = namedtuple("PromptChunkFix", ["offset", "embedding"])898"""An object of this type is a marker showing that textual inversion embedding's vectors have to placed at offset in the prompt899chunk. Thos objects are found in PromptChunk.fixes and, are placed into FrozenCLIPEmbedderWithCustomWordsBase.hijack.fixes, and finally900are applied by sd_hijack.EmbeddingsWithFixes's forward function."""901 902 903class FrozenCLIPEmbedder(nn.Layer):904 """Uses the CLIP transformer encoder for text (from huggingface)"""905 906 LAYERS = ["last", "pooled", "hidden"]907 908 def __init__(self, text_encoder, tokenizer, freeze=True, layer="last", layer_idx=None):909 super().__init__()910 assert layer in self.LAYERS911 self.tokenizer = tokenizer912 self.text_encoder = text_encoder913 if freeze:914 self.freeze()915 self.layer = layer916 self.layer_idx = layer_idx917 if layer == "hidden":918 assert layer_idx is not None919 assert 0 <= abs(layer_idx) <= 12920 921 def freeze(self):922 self.text_encoder.eval()923 for param in self.parameters():924 param.stop_gradient = False925 926 def forward(self, text):927 batch_encoding = self.tokenizer(928 text,929 truncation=True,930 max_length=self.tokenizer.model_max_length,931 padding="max_length",932 return_tensors="pd",933 )934 tokens = batch_encoding["input_ids"]935 outputs = self.text_encoder(input_ids=tokens, output_hidden_states=self.layer == "hidden", return_dict=True)936 if self.layer == "last":937 z = outputs.last_hidden_state938 elif self.layer == "pooled":939 z = outputs.pooler_output[:, None, :]940 else:941 z = outputs.hidden_states[self.layer_idx]942 return z943 944 def encode(self, text):945 return self(text)946 947 948class FrozenCLIPEmbedderWithCustomWordsBase(nn.Layer):949 """A pytorch module that is a wrapper for FrozenCLIPEmbedder module. it enhances FrozenCLIPEmbedder, making it possible to950 have unlimited prompt length and assign weights to tokens in prompt.951 """952 953 def __init__(self, wrapped, hijack):954 super().__init__()955 956 self.wrapped = wrapped957 """Original FrozenCLIPEmbedder module; can also be FrozenOpenCLIPEmbedder or xlmr.BertSeriesModelWithTransformation,958 depending on model."""959 960 self.hijack = hijack961 self.chunk_length = 75962 963 def empty_chunk(self):964 """creates an empty PromptChunk and returns it"""965 966 chunk = PromptChunk()967 chunk.tokens = [self.id_start] + [self.id_end] * (self.chunk_length + 1)968 chunk.multipliers = [1.0] * (self.chunk_length + 2)969 return chunk970 971 def get_target_prompt_token_count(self, token_count):972 """returns the maximum number of tokens a prompt of a known length can have before it requires one more PromptChunk to be represented"""973 974 return math.ceil(max(token_count, 1) / self.chunk_length) * self.chunk_length975 976 def tokenize(self, texts):977 """Converts a batch of texts into a batch of token ids"""978 979 raise NotImplementedError980 981 def encode_with_text_encoder(self, tokens):982 """983 converts a batch of token ids (in python lists) into a single tensor with numeric respresentation of those tokens;984 All python lists with tokens are assumed to have same length, usually 77.985 if input is a list with B elements and each element has T tokens, expected output shape is (B, T, C), where C depends on986 model - can be 768 and 1024.987 Among other things, this call will read self.hijack.fixes, apply it to its inputs, and clear it (setting it to None).988 """989 990 raise NotImplementedError991 992 def encode_embedding_init_text(self, init_text, nvpt):993 """Converts text into a tensor with this text's tokens' embeddings. Note that those are embeddings before they are passed through994 transformers. nvpt is used as a maximum length in tokens. If text produces less teokens than nvpt, only this many is returned."""995 996 raise NotImplementedError997 998 def tokenize_line(self, line):999 """1000 this transforms a single prompt into a list of PromptChunk objects - as many as needed to1001 represent the prompt.1002 Returns the list and the total number of tokens in the prompt.1003 """1004 1005 if WebUIStableDiffusionControlNetPipeline.enable_emphasis:1006 parsed = parse_prompt_attention(line)1007 else:1008 parsed = [[line, 1.0]]1009 1010 tokenized = self.tokenize([text for text, _ in parsed])1011 1012 chunks = []1013 chunk = PromptChunk()1014 token_count = 01015 last_comma = -11016 1017 def next_chunk(is_last=False):1018 """puts current chunk into the list of results and produces the next one - empty;1019 if is_last is true, tokens <end-of-text> tokens at the end won't add to token_count"""1020 nonlocal token_count1021 nonlocal last_comma1022 nonlocal chunk1023 1024 if is_last:1025 token_count += len(chunk.tokens)1026 else:1027 token_count += self.chunk_length1028 1029 to_add = self.chunk_length - len(chunk.tokens)1030 if to_add > 0:1031 chunk.tokens += [self.id_end] * to_add1032 chunk.multipliers += [1.0] * to_add1033 1034 chunk.tokens = [self.id_start] + chunk.tokens + [self.id_end]1035 chunk.multipliers = [1.0] + chunk.multipliers + [1.0]1036 1037 last_comma = -11038 chunks.append(chunk)1039 chunk = PromptChunk()1040 1041 for tokens, (text, weight) in zip(tokenized, parsed):1042 if text == "BREAK" and weight == -1:1043 next_chunk()1044 continue1045 1046 position = 01047 while position < len(tokens):1048 token = tokens[position]1049 1050 if token == self.comma_token:1051 last_comma = len(chunk.tokens)1052 1053 # this is when we are at the end of alloted 75 tokens for the current chunk, and the current token is not a comma. opts.comma_padding_backtrack1054 # is a setting that specifies that if there is a comma nearby, the text after the comma should be moved out of this chunk and into the next.1055 elif (1056 WebUIStableDiffusionControlNetPipeline.comma_padding_backtrack != 01057 and len(chunk.tokens) == self.chunk_length1058 and last_comma != -11059 and len(chunk.tokens) - last_comma1060 <= WebUIStableDiffusionControlNetPipeline.comma_padding_backtrack1061 ):1062 break_location = last_comma + 11063 1064 reloc_tokens = chunk.tokens[break_location:]1065 reloc_mults = chunk.multipliers[break_location:]1066 1067 chunk.tokens = chunk.tokens[:break_location]1068 chunk.multipliers = chunk.multipliers[:break_location]1069 1070 next_chunk()1071 chunk.tokens = reloc_tokens1072 chunk.multipliers = reloc_mults1073 1074 if len(chunk.tokens) == self.chunk_length:1075 next_chunk()1076 1077 embedding, embedding_length_in_tokens = self.hijack.embedding_db.find_embedding_at_position(1078 tokens, position1079 )1080 if embedding is None:1081 chunk.tokens.append(token)1082 chunk.multipliers.append(weight)1083 position += 11084 continue1085 1086 emb_len = int(embedding.vec.shape[0])1087 if len(chunk.tokens) + emb_len > self.chunk_length:1088 next_chunk()1089 1090 chunk.fixes.append(PromptChunkFix(len(chunk.tokens), embedding))1091 1092 chunk.tokens += [0] * emb_len1093 chunk.multipliers += [weight] * emb_len1094 position += embedding_length_in_tokens1095 1096 if len(chunk.tokens) > 0 or len(chunks) == 0:1097 next_chunk(is_last=True)1098 1099 return chunks, token_count1100 1101 def process_texts(self, texts):1102 """1103 Accepts a list of texts and calls tokenize_line() on each, with cache. Returns the list of results and maximum1104 length, in tokens, of all texts.1105 """1106 1107 token_count = 01108 1109 cache = {}1110 batch_chunks = []1111 for line in texts:1112 if line in cache:1113 chunks = cache[line]1114 else:1115 chunks, current_token_count = self.tokenize_line(line)1116 token_count = max(current_token_count, token_count)1117 1118 cache[line] = chunks1119 1120 batch_chunks.append(chunks)1121 1122 return batch_chunks, token_count1123 1124 def forward(self, texts):1125 """1126 Accepts an array of texts; Passes texts through transformers network to create a tensor with numerical representation of those texts.1127 Returns a tensor with shape of (B, T, C), where B is length of the array; T is length, in tokens, of texts (including padding) - T will1128 be a multiple of 77; and C is dimensionality of each token - for SD1 it's 768, and for SD2 it's 1024.1129 An example shape returned by this function can be: (2, 77, 768).1130 Webui usually sends just one text at a time through this function - the only time when texts is an array with more than one elemenet1131 is when you do prompt editing: "a picture of a [cat:dog:0.4] eating ice cream"1132 """1133 1134 batch_chunks, token_count = self.process_texts(texts)1135 1136 used_embeddings = {}1137 chunk_count = max([len(x) for x in batch_chunks])1138 1139 zs = []1140 for i in range(chunk_count):1141 batch_chunk = [chunks[i] if i < len(chunks) else self.empty_chunk() for chunks in batch_chunks]1142 1143 tokens = [x.tokens for x in batch_chunk]1144 multipliers = [x.multipliers for x in batch_chunk]1145 self.hijack.fixes = [x.fixes for x in batch_chunk]1146 1147 for fixes in self.hijack.fixes:1148 for position, embedding in fixes:1149 used_embeddings[embedding.name] = embedding1150 1151 z = self.process_tokens(tokens, multipliers)1152 zs.append(z)1153 1154 if len(used_embeddings) > 0:1155 embeddings_list = ", ".join(1156 [f"{name} [{embedding.checksum()}]" for name, embedding in used_embeddings.items()]1157 )1158 self.hijack.comments.append(f"Used embeddings: {embeddings_list}")1159 1160 return paddle.concat(zs, axis=1)1161 1162 def process_tokens(self, remade_batch_tokens, batch_multipliers):1163 """1164 sends one single prompt chunk to be encoded by transformers neural network.1165 remade_batch_tokens is a batch of tokens - a list, where every element is a list of tokens; usually1166 there are exactly 77 tokens in the list. batch_multipliers is the same but for multipliers instead of tokens.1167 Multipliers are used to give more or less weight to the outputs of transformers network. Each multiplier1168 corresponds to one token.1169 """1170 tokens = paddle.to_tensor(remade_batch_tokens)1171 1172 # this is for SD2: SD1 uses the same token for padding and end of text, while SD2 uses different ones.1173 if self.id_end != self.id_pad:1174 for batch_pos in range(len(remade_batch_tokens)):1175 index = remade_batch_tokens[batch_pos].index(self.id_end)1176 tokens[batch_pos, index + 1 : tokens.shape[1]] = self.id_pad1177 1178 z = self.encode_with_text_encoder(tokens)1179 1180 # restoring original mean is likely not correct, but it seems to work well to prevent artifacts that happen otherwise1181 batch_multipliers = paddle.to_tensor(batch_multipliers)1182 original_mean = z.mean()1183 z = z * batch_multipliers.reshape(1184 batch_multipliers.shape1185 + [1186 1,1187 ]1188 ).expand(z.shape)1189 new_mean = z.mean()1190 z = z * (original_mean / new_mean)1191 1192 return z1193 1194 1195class FrozenCLIPEmbedderWithCustomWords(FrozenCLIPEmbedderWithCustomWordsBase):1196 def __init__(self, wrapped, hijack, CLIP_stop_at_last_layers=-1):1197 super().__init__(wrapped, hijack)1198 self.CLIP_stop_at_last_layers = CLIP_stop_at_last_layers1199 self.tokenizer = wrapped.tokenizer1200