diffusers/community-pipelines-mirror
Community Pipeline Examples For more information about community pipelines, please have a look at this issue. Community pipeline examples consist pipelines that have been added by the community. Please have a look at the following tables to get an overview of all community examples. Click on the Code Example to get a copy-and-paste ready code example that you can try out. If a community pipeline doesn't work as expected, please open an issue and ping the author on it. Please… See the full description on the dataset page: https://huggingface.co/datasets/diffusers/community-pipelines-mirror.
922k
1## ----------------------------------------------------------2# A SDXL pipeline can take unlimited weighted prompt3#4# Author: Andrew Zhu5# Github: https://github.com/xhinker6# Medium: https://medium.com/@xhinker7## -----------------------------------------------------------8 9import inspect10import os11from typing import Any, Callable, Dict, List, Optional, Tuple, Union12 13import torch14from PIL import Image15from transformers import (16 CLIPImageProcessor,17 CLIPTextModel,18 CLIPTextModelWithProjection,19 CLIPTokenizer,20 CLIPVisionModelWithProjection,21)22 23from diffusers import DiffusionPipeline, StableDiffusionXLPipeline24from diffusers.image_processor import PipelineImageInput, VaeImageProcessor25from diffusers.loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin26from diffusers.models import AutoencoderKL, ImageProjection, UNet2DConditionModel27from diffusers.models.attention_processor import (28 AttnProcessor2_0,29 LoRAAttnProcessor2_0,30 LoRAXFormersAttnProcessor,31 XFormersAttnProcessor,32)33from diffusers.pipelines.pipeline_utils import StableDiffusionMixin34from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput35from diffusers.schedulers import KarrasDiffusionSchedulers36from diffusers.utils import (37 deprecate,38 is_accelerate_available,39 is_accelerate_version,40 is_invisible_watermark_available,41 logging,42 replace_example_docstring,43)44from diffusers.utils.torch_utils import randn_tensor45 46 47if is_invisible_watermark_available():48 from diffusers.pipelines.stable_diffusion_xl.watermark import StableDiffusionXLWatermarker49 50 51def parse_prompt_attention(text):52 """53 Parses a string with attention tokens and returns a list of pairs: text and its associated weight.54 Accepted tokens are:55 (abc) - increases attention to abc by a multiplier of 1.156 (abc:3.12) - increases attention to abc by a multiplier of 3.1257 [abc] - decreases attention to abc by a multiplier of 1.158 \\( - literal character '('59 \\[ - literal character '['60 \\) - literal character ')'61 \\] - literal character ']'62 \\ - literal character '\'63 anything else - just text64 65 >>> parse_prompt_attention('normal text')66 [['normal text', 1.0]]67 >>> parse_prompt_attention('an (important) word')68 [['an ', 1.0], ['important', 1.1], [' word', 1.0]]69 >>> parse_prompt_attention('(unbalanced')70 [['unbalanced', 1.1]]71 >>> parse_prompt_attention('\\(literal\\]')72 [['(literal]', 1.0]]73 >>> parse_prompt_attention('(unnecessary)(parens)')74 [['unnecessaryparens', 1.1]]75 >>> parse_prompt_attention('a (((house:1.3)) [on] a (hill:0.5), sun, (((sky))).')76 [['a ', 1.0],77 ['house', 1.5730000000000004],78 [' ', 1.1],79 ['on', 1.0],80 [' a ', 1.1],81 ['hill', 0.55],82 [', sun, ', 1.1],83 ['sky', 1.4641000000000006],84 ['.', 1.1]]85 """86 import re87 88 re_attention = re.compile(89 r"""90 \\\(|\\\)|\\\[|\\]|\\\\|\\|\(|\[|:([+-]?[.\d]+)\)|91 \)|]|[^\\()\[\]:]+|:92 """,93 re.X,94 )95 96 re_break = re.compile(r"\s*\bBREAK\b\s*", re.S)97 98 res = []99 round_brackets = []100 square_brackets = []101 102 round_bracket_multiplier = 1.1103 square_bracket_multiplier = 1 / 1.1104 105 def multiply_range(start_position, multiplier):106 for p in range(start_position, len(res)):107 res[p][1] *= multiplier108 109 for m in re_attention.finditer(text):110 text = m.group(0)111 weight = m.group(1)112 113 if text.startswith("\\"):114 res.append([text[1:], 1.0])115 elif text == "(":116 round_brackets.append(len(res))117 elif text == "[":118 square_brackets.append(len(res))119 elif weight is not None and len(round_brackets) > 0:120 multiply_range(round_brackets.pop(), float(weight))121 elif text == ")" and len(round_brackets) > 0:122 multiply_range(round_brackets.pop(), round_bracket_multiplier)123 elif text == "]" and len(square_brackets) > 0:124 multiply_range(square_brackets.pop(), square_bracket_multiplier)125 else:126 parts = re.split(re_break, text)127 for i, part in enumerate(parts):128 if i > 0:129 res.append(["BREAK", -1])130 res.append([part, 1.0])131 132 for pos in round_brackets:133 multiply_range(pos, round_bracket_multiplier)134 135 for pos in square_brackets:136 multiply_range(pos, square_bracket_multiplier)137 138 if len(res) == 0:139 res = [["", 1.0]]140 141 # merge runs of identical weights142 i = 0143 while i + 1 < len(res):144 if res[i][1] == res[i + 1][1]:145 res[i][0] += res[i + 1][0]146 res.pop(i + 1)147 else:148 i += 1149 150 return res151 152 153def get_prompts_tokens_with_weights(clip_tokenizer: CLIPTokenizer, prompt: str):154 """155 Get prompt token ids and weights, this function works for both prompt and negative prompt156 157 Args:158 pipe (CLIPTokenizer)159 A CLIPTokenizer160 prompt (str)161 A prompt string with weights162 163 Returns:164 text_tokens (list)165 A list contains token ids166 text_weight (list)167 A list contains the correspondent weight of token ids168 169 Example:170 import torch171 from transformers import CLIPTokenizer172 173 clip_tokenizer = CLIPTokenizer.from_pretrained(174 "stablediffusionapi/deliberate-v2"175 , subfolder = "tokenizer"176 , dtype = torch.float16177 )178 179 token_id_list, token_weight_list = get_prompts_tokens_with_weights(180 clip_tokenizer = clip_tokenizer181 ,prompt = "a (red:1.5) cat"*70182 )183 """184 texts_and_weights = parse_prompt_attention(prompt)185 text_tokens, text_weights = [], []186 for word, weight in texts_and_weights:187 # tokenize and discard the starting and the ending token188 token = clip_tokenizer(word, truncation=False).input_ids[1:-1] # so that tokenize whatever length prompt189 # the returned token is a 1d list: [320, 1125, 539, 320]190 191 # merge the new tokens to the all tokens holder: text_tokens192 text_tokens = [*text_tokens, *token]193 194 # each token chunk will come with one weight, like ['red cat', 2.0]195 # need to expand weight for each token.196 chunk_weights = [weight] * len(token)197 198 # append the weight back to the weight holder: text_weights199 text_weights = [*text_weights, *chunk_weights]200 return text_tokens, text_weights201 202 203def group_tokens_and_weights(token_ids: list, weights: list, pad_last_block=False):204 """205 Produce tokens and weights in groups and pad the missing tokens206 207 Args:208 token_ids (list)209 The token ids from tokenizer210 weights (list)211 The weights list from function get_prompts_tokens_with_weights212 pad_last_block (bool)213 Control if fill the last token list to 75 tokens with eos214 Returns:215 new_token_ids (2d list)216 new_weights (2d list)217 218 Example:219 token_groups,weight_groups = group_tokens_and_weights(220 token_ids = token_id_list221 , weights = token_weight_list222 )223 """224 bos, eos = 49406, 49407225 226 # this will be a 2d list227 new_token_ids = []228 new_weights = []229 while len(token_ids) >= 75:230 # get the first 75 tokens231 head_75_tokens = [token_ids.pop(0) for _ in range(75)]232 head_75_weights = [weights.pop(0) for _ in range(75)]233 234 # extract token ids and weights235 temp_77_token_ids = [bos] + head_75_tokens + [eos]236 temp_77_weights = [1.0] + head_75_weights + [1.0]237 238 # add 77 token and weights chunk to the holder list239 new_token_ids.append(temp_77_token_ids)240 new_weights.append(temp_77_weights)241 242 # padding the left243 if len(token_ids) > 0:244 padding_len = 75 - len(token_ids) if pad_last_block else 0245 246 temp_77_token_ids = [bos] + token_ids + [eos] * padding_len + [eos]247 new_token_ids.append(temp_77_token_ids)248 249 temp_77_weights = [1.0] + weights + [1.0] * padding_len + [1.0]250 new_weights.append(temp_77_weights)251 252 return new_token_ids, new_weights253 254 255def get_weighted_text_embeddings_sdxl(256 pipe: StableDiffusionXLPipeline,257 prompt: str = "",258 prompt_2: str = None,259 neg_prompt: str = "",260 neg_prompt_2: str = None,261 num_images_per_prompt: int = 1,262 device: Optional[torch.device] = None,263 clip_skip: Optional[int] = None,264):265 """266 This function can process long prompt with weights, no length limitation267 for Stable Diffusion XL268 269 Args:270 pipe (StableDiffusionPipeline)271 prompt (str)272 prompt_2 (str)273 neg_prompt (str)274 neg_prompt_2 (str)275 num_images_per_prompt (int)276 device (torch.device)277 clip_skip (int)278 Returns:279 prompt_embeds (torch.Tensor)280 neg_prompt_embeds (torch.Tensor)281 """282 device = device or pipe._execution_device283 284 if prompt_2:285 prompt = f"{prompt} {prompt_2}"286 287 if neg_prompt_2:288 neg_prompt = f"{neg_prompt} {neg_prompt_2}"289 290 prompt_t1 = prompt_t2 = prompt291 neg_prompt_t1 = neg_prompt_t2 = neg_prompt292 293 if isinstance(pipe, TextualInversionLoaderMixin):294 prompt_t1 = pipe.maybe_convert_prompt(prompt_t1, pipe.tokenizer)295 neg_prompt_t1 = pipe.maybe_convert_prompt(neg_prompt_t1, pipe.tokenizer)296 prompt_t2 = pipe.maybe_convert_prompt(prompt_t2, pipe.tokenizer_2)297 neg_prompt_t2 = pipe.maybe_convert_prompt(neg_prompt_t2, pipe.tokenizer_2)298 299 eos = pipe.tokenizer.eos_token_id300 301 # tokenizer 1302 prompt_tokens, prompt_weights = get_prompts_tokens_with_weights(pipe.tokenizer, prompt_t1)303 neg_prompt_tokens, neg_prompt_weights = get_prompts_tokens_with_weights(pipe.tokenizer, neg_prompt_t1)304 305 # tokenizer 2306 prompt_tokens_2, prompt_weights_2 = get_prompts_tokens_with_weights(pipe.tokenizer_2, prompt_t2)307 neg_prompt_tokens_2, neg_prompt_weights_2 = get_prompts_tokens_with_weights(pipe.tokenizer_2, neg_prompt_t2)308 309 # padding the shorter one for prompt set 1310 prompt_token_len = len(prompt_tokens)311 neg_prompt_token_len = len(neg_prompt_tokens)312 313 if prompt_token_len > neg_prompt_token_len:314 # padding the neg_prompt with eos token315 neg_prompt_tokens = neg_prompt_tokens + [eos] * abs(prompt_token_len - neg_prompt_token_len)316 neg_prompt_weights = neg_prompt_weights + [1.0] * abs(prompt_token_len - neg_prompt_token_len)317 else:318 # padding the prompt319 prompt_tokens = prompt_tokens + [eos] * abs(prompt_token_len - neg_prompt_token_len)320 prompt_weights = prompt_weights + [1.0] * abs(prompt_token_len - neg_prompt_token_len)321 322 # padding the shorter one for token set 2323 prompt_token_len_2 = len(prompt_tokens_2)324 neg_prompt_token_len_2 = len(neg_prompt_tokens_2)325 326 if prompt_token_len_2 > neg_prompt_token_len_2:327 # padding the neg_prompt with eos token328 neg_prompt_tokens_2 = neg_prompt_tokens_2 + [eos] * abs(prompt_token_len_2 - neg_prompt_token_len_2)329 neg_prompt_weights_2 = neg_prompt_weights_2 + [1.0] * abs(prompt_token_len_2 - neg_prompt_token_len_2)330 else:331 # padding the prompt332 prompt_tokens_2 = prompt_tokens_2 + [eos] * abs(prompt_token_len_2 - neg_prompt_token_len_2)333 prompt_weights_2 = prompt_weights + [1.0] * abs(prompt_token_len_2 - neg_prompt_token_len_2)334 335 embeds = []336 neg_embeds = []337 338 prompt_token_groups, prompt_weight_groups = group_tokens_and_weights(prompt_tokens.copy(), prompt_weights.copy())339 340 neg_prompt_token_groups, neg_prompt_weight_groups = group_tokens_and_weights(341 neg_prompt_tokens.copy(), neg_prompt_weights.copy()342 )343 344 prompt_token_groups_2, prompt_weight_groups_2 = group_tokens_and_weights(345 prompt_tokens_2.copy(), prompt_weights_2.copy()346 )347 348 neg_prompt_token_groups_2, neg_prompt_weight_groups_2 = group_tokens_and_weights(349 neg_prompt_tokens_2.copy(), neg_prompt_weights_2.copy()350 )351 352 # get prompt embeddings one by one is not working.353 for i in range(len(prompt_token_groups)):354 # get positive prompt embeddings with weights355 token_tensor = torch.tensor([prompt_token_groups[i]], dtype=torch.long, device=device)356 weight_tensor = torch.tensor(prompt_weight_groups[i], dtype=torch.float16, device=device)357 358 token_tensor_2 = torch.tensor([prompt_token_groups_2[i]], dtype=torch.long, device=device)359 360 # use first text encoder361 prompt_embeds_1 = pipe.text_encoder(token_tensor.to(device), output_hidden_states=True)362 363 # use second text encoder364 prompt_embeds_2 = pipe.text_encoder_2(token_tensor_2.to(device), output_hidden_states=True)365 pooled_prompt_embeds = prompt_embeds_2[0]366 367 if clip_skip is None:368 prompt_embeds_1_hidden_states = prompt_embeds_1.hidden_states[-2]369 prompt_embeds_2_hidden_states = prompt_embeds_2.hidden_states[-2]370 else:371 # "2" because SDXL always indexes from the penultimate layer.372 prompt_embeds_1_hidden_states = prompt_embeds_1.hidden_states[-(clip_skip + 2)]373 prompt_embeds_2_hidden_states = prompt_embeds_2.hidden_states[-(clip_skip + 2)]374 375 prompt_embeds_list = [prompt_embeds_1_hidden_states, prompt_embeds_2_hidden_states]376 token_embedding = torch.concat(prompt_embeds_list, dim=-1).squeeze(0)377 378 for j in range(len(weight_tensor)):379 if weight_tensor[j] != 1.0:380 token_embedding[j] = (381 token_embedding[-1] + (token_embedding[j] - token_embedding[-1]) * weight_tensor[j]382 )383 384 token_embedding = token_embedding.unsqueeze(0)385 embeds.append(token_embedding)386 387 # get negative prompt embeddings with weights388 neg_token_tensor = torch.tensor([neg_prompt_token_groups[i]], dtype=torch.long, device=device)389 neg_token_tensor_2 = torch.tensor([neg_prompt_token_groups_2[i]], dtype=torch.long, device=device)390 neg_weight_tensor = torch.tensor(neg_prompt_weight_groups[i], dtype=torch.float16, device=device)391 392 # use first text encoder393 neg_prompt_embeds_1 = pipe.text_encoder(neg_token_tensor.to(device), output_hidden_states=True)394 neg_prompt_embeds_1_hidden_states = neg_prompt_embeds_1.hidden_states[-2]395 396 # use second text encoder397 neg_prompt_embeds_2 = pipe.text_encoder_2(neg_token_tensor_2.to(device), output_hidden_states=True)398 neg_prompt_embeds_2_hidden_states = neg_prompt_embeds_2.hidden_states[-2]399 negative_pooled_prompt_embeds = neg_prompt_embeds_2[0]400 401 neg_prompt_embeds_list = [neg_prompt_embeds_1_hidden_states, neg_prompt_embeds_2_hidden_states]402 neg_token_embedding = torch.concat(neg_prompt_embeds_list, dim=-1).squeeze(0)403 404 for z in range(len(neg_weight_tensor)):405 if neg_weight_tensor[z] != 1.0:406 neg_token_embedding[z] = (407 neg_token_embedding[-1] + (neg_token_embedding[z] - neg_token_embedding[-1]) * neg_weight_tensor[z]408 )409 410 neg_token_embedding = neg_token_embedding.unsqueeze(0)411 neg_embeds.append(neg_token_embedding)412 413 prompt_embeds = torch.cat(embeds, dim=1)414 negative_prompt_embeds = torch.cat(neg_embeds, dim=1)415 416 bs_embed, seq_len, _ = prompt_embeds.shape417 # duplicate text embeddings for each generation per prompt, using mps friendly method418 prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)419 prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)420 421 seq_len = negative_prompt_embeds.shape[1]422 negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)423 negative_prompt_embeds = negative_prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)424 425 pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt, 1).view(426 bs_embed * num_images_per_prompt, -1427 )428 negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt, 1).view(429 bs_embed * num_images_per_prompt, -1430 )431 432 return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds433 434 435# -------------------------------------------------------------------------------------------------------------------------------436# reuse the backbone code from StableDiffusionXLPipeline437# -------------------------------------------------------------------------------------------------------------------------------438 439logger = logging.get_logger(__name__) # pylint: disable=invalid-name440 441EXAMPLE_DOC_STRING = """442 Examples:443 ```py444 from diffusers import DiffusionPipeline445 import torch446 447 pipe = DiffusionPipeline.from_pretrained(448 "stabilityai/stable-diffusion-xl-base-1.0"449 , torch_dtype = torch.float16450 , use_safetensors = True451 , variant = "fp16"452 , custom_pipeline = "lpw_stable_diffusion_xl",453 )454 455 prompt = "a white cat running on the grass"*20456 prompt2 = "play a football"*20457 prompt = f"{prompt},{prompt2}"458 neg_prompt = "blur, low quality"459 460 pipe.to("cuda")461 images = pipe(462 prompt = prompt463 , negative_prompt = neg_prompt464 ).images[0]465 466 pipe.to("cpu")467 torch.cuda.empty_cache()468 images469 ```470"""471 472 473# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg474def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):475 """476 Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and477 Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4478 """479 std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)480 std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)481 # rescale the results from guidance (fixes overexposure)482 noise_pred_rescaled = noise_cfg * (std_text / std_cfg)483 # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images484 noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg485 return noise_cfg486 487 488# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents489def retrieve_latents(490 encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample"491):492 if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":493 return encoder_output.latent_dist.sample(generator)494 elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":495 return encoder_output.latent_dist.mode()496 elif hasattr(encoder_output, "latents"):497 return encoder_output.latents498 else:499 raise AttributeError("Could not access latents of provided encoder_output")500 501 502# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps503def retrieve_timesteps(504 scheduler,505 num_inference_steps: Optional[int] = None,506 device: Optional[Union[str, torch.device]] = None,507 timesteps: Optional[List[int]] = None,508 **kwargs,509):510 """511 Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles512 custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.513 514 Args:515 scheduler (`SchedulerMixin`):516 The scheduler to get timesteps from.517 num_inference_steps (`int`):518 The number of diffusion steps used when generating samples with a pre-trained model. If used,519 `timesteps` must be `None`.520 device (`str` or `torch.device`, *optional*):521 The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.522 timesteps (`List[int]`, *optional*):523 Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default524 timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps`525 must be `None`.526 527 Returns:528 `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the529 second element is the number of inference steps.530 """531 if timesteps is not None:532 accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())533 if not accepts_timesteps:534 raise ValueError(535 f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"536 f" timestep schedules. Please check whether you are using the correct scheduler."537 )538 scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)539 timesteps = scheduler.timesteps540 num_inference_steps = len(timesteps)541 else:542 scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)543 timesteps = scheduler.timesteps544 return timesteps, num_inference_steps545 546 547class SDXLLongPromptWeightingPipeline(548 DiffusionPipeline,549 StableDiffusionMixin,550 FromSingleFileMixin,551 IPAdapterMixin,552 LoraLoaderMixin,553 TextualInversionLoaderMixin,554):555 r"""556 Pipeline for text-to-image generation using Stable Diffusion XL.557 558 This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods559 implemented for all pipelines (downloading, saving, running on a particular device, etc.).560 561 The pipeline also inherits the following loading methods:562 - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files563 - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters564 - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights565 - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights566 - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings567 568 Args:569 vae ([`AutoencoderKL`]):570 Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.571 text_encoder ([`CLIPTextModel`]):572 Frozen text-encoder. Stable Diffusion XL uses the text portion of573 [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically574 the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.575 text_encoder_2 ([` CLIPTextModelWithProjection`]):576 Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of577 [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection),578 specifically the579 [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)580 variant.581 tokenizer (`CLIPTokenizer`):582 Tokenizer of class583 [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).584 tokenizer_2 (`CLIPTokenizer`):585 Second Tokenizer of class586 [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).587 unet ([`UNet2DConditionModel`]):588 Conditional U-Net architecture to denoise the encoded image latents.589 scheduler ([`SchedulerMixin`]):590 A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of591 [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].592 feature_extractor ([`~transformers.CLIPImageProcessor`]):593 A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`.594 """595 596 model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae"597 _optional_components = [598 "tokenizer",599 "tokenizer_2",600 "text_encoder",601 "text_encoder_2",602 "image_encoder",603 "feature_extractor",604 ]605 _callback_tensor_inputs = [606 "latents",607 "prompt_embeds",608 "negative_prompt_embeds",609 "add_text_embeds",610 "add_time_ids",611 "negative_pooled_prompt_embeds",612 "negative_add_time_ids",613 ]614 615 def __init__(616 self,617 vae: AutoencoderKL,618 text_encoder: CLIPTextModel,619 text_encoder_2: CLIPTextModelWithProjection,620 tokenizer: CLIPTokenizer,621 tokenizer_2: CLIPTokenizer,622 unet: UNet2DConditionModel,623 scheduler: KarrasDiffusionSchedulers,624 feature_extractor: Optional[CLIPImageProcessor] = None,625 image_encoder: Optional[CLIPVisionModelWithProjection] = None,626 force_zeros_for_empty_prompt: bool = True,627 add_watermarker: Optional[bool] = None,628 ):629 super().__init__()630 631 self.register_modules(632 vae=vae,633 text_encoder=text_encoder,634 text_encoder_2=text_encoder_2,635 tokenizer=tokenizer,636 tokenizer_2=tokenizer_2,637 unet=unet,638 scheduler=scheduler,639 feature_extractor=feature_extractor,640 image_encoder=image_encoder,641 )642 self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt)643 self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)644 self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)645 self.mask_processor = VaeImageProcessor(646 vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True647 )648 self.default_sample_size = self.unet.config.sample_size649 650 add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available()651 652 if add_watermarker:653 self.watermark = StableDiffusionXLWatermarker()654 else:655 self.watermark = None656 657 def enable_model_cpu_offload(self, gpu_id=0):658 r"""659 Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared660 to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward`661 method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with662 `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`.663 """664 if is_accelerate_available() and is_accelerate_version(">=", "0.17.0.dev0"):665 from accelerate import cpu_offload_with_hook666 else:667 raise ImportError("`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher.")668 669 device = torch.device(f"cuda:{gpu_id}")670 671 if self.device.type != "cpu":672 self.to("cpu", silence_dtype_warnings=True)673 torch.cuda.empty_cache() # otherwise we don't see the memory savings (but they probably exist)674 675 model_sequence = (676 [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]677 )678 model_sequence.extend([self.unet, self.vae])679 680 hook = None681 for cpu_offloaded_model in model_sequence:682 _, hook = cpu_offload_with_hook(cpu_offloaded_model, device, prev_module_hook=hook)683 684 # We'll offload the last model manually.685 self.final_offload_hook = hook686 687 # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt688 def encode_prompt(689 self,690 prompt: str,691 prompt_2: Optional[str] = None,692 device: Optional[torch.device] = None,693 num_images_per_prompt: int = 1,694 do_classifier_free_guidance: bool = True,695 negative_prompt: Optional[str] = None,696 negative_prompt_2: Optional[str] = None,697 prompt_embeds: Optional[torch.Tensor] = None,698 negative_prompt_embeds: Optional[torch.Tensor] = None,699 pooled_prompt_embeds: Optional[torch.Tensor] = None,700 negative_pooled_prompt_embeds: Optional[torch.Tensor] = None,701 lora_scale: Optional[float] = None,702 ):703 r"""704 Encodes the prompt into text encoder hidden states.705 706 Args:707 prompt (`str` or `List[str]`, *optional*):708 prompt to be encoded709 prompt_2 (`str` or `List[str]`, *optional*):710 The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is711 used in both text-encoders712 device: (`torch.device`):713 torch device714 num_images_per_prompt (`int`):715 number of images that should be generated per prompt716 do_classifier_free_guidance (`bool`):717 whether to use classifier free guidance or not718 negative_prompt (`str` or `List[str]`, *optional*):719 The prompt or prompts not to guide the image generation. If not defined, one has to pass720 `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is721 less than `1`).722 negative_prompt_2 (`str` or `List[str]`, *optional*):723 The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and724 `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders725 prompt_embeds (`torch.Tensor`, *optional*):726 Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not727 provided, text embeddings will be generated from `prompt` input argument.728 negative_prompt_embeds (`torch.Tensor`, *optional*):729 Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt730 weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input731 argument.732 pooled_prompt_embeds (`torch.Tensor`, *optional*):733 Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.734 If not provided, pooled text embeddings will be generated from `prompt` input argument.735 negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):736 Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt737 weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`738 input argument.739 lora_scale (`float`, *optional*):740 A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.741 """742 device = device or self._execution_device743 744 # set lora scale so that monkey patched LoRA745 # function of text encoder can correctly access it746 if lora_scale is not None and isinstance(self, LoraLoaderMixin):747 self._lora_scale = lora_scale748 749 if prompt is not None and isinstance(prompt, str):750 batch_size = 1751 elif prompt is not None and isinstance(prompt, list):752 batch_size = len(prompt)753 else:754 batch_size = prompt_embeds.shape[0]755 756 # Define tokenizers and text encoders757 tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2]758 text_encoders = (759 [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]760 )761 762 if prompt_embeds is None:763 prompt_2 = prompt_2 or prompt764 # textual inversion: process multi-vector tokens if necessary765 prompt_embeds_list = []766 prompts = [prompt, prompt_2]767 for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders):768 if isinstance(self, TextualInversionLoaderMixin):769 prompt = self.maybe_convert_prompt(prompt, tokenizer)770 771 text_inputs = tokenizer(772 prompt,773 padding="max_length",774 max_length=tokenizer.model_max_length,775 truncation=True,776 return_tensors="pt",777 )778 779 text_input_ids = text_inputs.input_ids780 untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids781 782 if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(783 text_input_ids, untruncated_ids784 ):785 removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1])786 logger.warning(787 "The following part of your input was truncated because CLIP can only handle sequences up to"788 f" {tokenizer.model_max_length} tokens: {removed_text}"789 )790 791 prompt_embeds = text_encoder(792 text_input_ids.to(device),793 output_hidden_states=True,794 )795 796 # We are only ALWAYS interested in the pooled output of the final text encoder797 pooled_prompt_embeds = prompt_embeds[0]798 prompt_embeds = prompt_embeds.hidden_states[-2]799 800 prompt_embeds_list.append(prompt_embeds)801 802 prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)803 804 # get unconditional embeddings for classifier free guidance805 zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt806 if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt:807 negative_prompt_embeds = torch.zeros_like(prompt_embeds)808 negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds)809 elif do_classifier_free_guidance and negative_prompt_embeds is None:810 negative_prompt = negative_prompt or ""811 negative_prompt_2 = negative_prompt_2 or negative_prompt812 813 uncond_tokens: List[str]814 if prompt is not None and type(prompt) is not type(negative_prompt):815 raise TypeError(816 f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="817 f" {type(prompt)}."818 )819 elif isinstance(negative_prompt, str):820 uncond_tokens = [negative_prompt, negative_prompt_2]821 elif batch_size != len(negative_prompt):822 raise ValueError(823 f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"824 f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"825 " the batch size of `prompt`."826 )827 else:828 uncond_tokens = [negative_prompt, negative_prompt_2]829 830 negative_prompt_embeds_list = []831 for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders):832 if isinstance(self, TextualInversionLoaderMixin):833 negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer)834 835 max_length = prompt_embeds.shape[1]836 uncond_input = tokenizer(837 negative_prompt,838 padding="max_length",839 max_length=max_length,840 truncation=True,841 return_tensors="pt",842 )843 844 negative_prompt_embeds = text_encoder(845 uncond_input.input_ids.to(device),846 output_hidden_states=True,847 )848 # We are only ALWAYS interested in the pooled output of the final text encoder849 negative_pooled_prompt_embeds = negative_prompt_embeds[0]850 negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2]851 852 negative_prompt_embeds_list.append(negative_prompt_embeds)853 854 negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1)855 856 prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)857 bs_embed, seq_len, _ = prompt_embeds.shape858 # duplicate text embeddings for each generation per prompt, using mps friendly method859 prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)860 prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)861 862 if do_classifier_free_guidance:863 # duplicate unconditional embeddings for each generation per prompt, using mps friendly method864 seq_len = negative_prompt_embeds.shape[1]865 negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)866 negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)867 negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)868 869 pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(870 bs_embed * num_images_per_prompt, -1871 )872 if do_classifier_free_guidance:873 negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(874 bs_embed * num_images_per_prompt, -1875 )876 877 return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds878 879 # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image880 def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):881 dtype = next(self.image_encoder.parameters()).dtype882 883 if not isinstance(image, torch.Tensor):884 image = self.feature_extractor(image, return_tensors="pt").pixel_values885 886 image = image.to(device=device, dtype=dtype)887 if output_hidden_states:888 image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]889 image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)890 uncond_image_enc_hidden_states = self.image_encoder(891 torch.zeros_like(image), output_hidden_states=True892 ).hidden_states[-2]893 uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(894 num_images_per_prompt, dim=0895 )896 return image_enc_hidden_states, uncond_image_enc_hidden_states897 else:898 image_embeds = self.image_encoder(image).image_embeds899 image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)900 uncond_image_embeds = torch.zeros_like(image_embeds)901 902 return image_embeds, uncond_image_embeds903 904 # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs905 def prepare_extra_step_kwargs(self, generator, eta):906 # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature907 # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.908 # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502909 # and should be between [0, 1]910 911 accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())912 extra_step_kwargs = {}913 if accepts_eta:914 extra_step_kwargs["eta"] = eta915 916 # check if the scheduler accepts generator917 accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())918 if accepts_generator:919 extra_step_kwargs["generator"] = generator920 return extra_step_kwargs921 922 def check_inputs(923 self,924 prompt,925 prompt_2,926 height,927 width,928 strength,929 callback_steps,930 negative_prompt=None,931 negative_prompt_2=None,932 prompt_embeds=None,933 negative_prompt_embeds=None,934 pooled_prompt_embeds=None,935 negative_pooled_prompt_embeds=None,936 callback_on_step_end_tensor_inputs=None,937 ):938 if height % 8 != 0 or width % 8 != 0:939 raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")940 941 if strength < 0 or strength > 1:942 raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")943 944 if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):945 raise ValueError(946 f"`callback_steps` has to be a positive integer but is {callback_steps} of type"947 f" {type(callback_steps)}."948 )949 950 if callback_on_step_end_tensor_inputs is not None and not all(951 k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs952 ):953 raise ValueError(954 f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"955 )956 957 if prompt is not None and prompt_embeds is not None:958 raise ValueError(959 f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"960 " only forward one of the two."961 )962 elif prompt_2 is not None and prompt_embeds is not None:963 raise ValueError(964 f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"965 " only forward one of the two."966 )967 elif prompt is None and prompt_embeds is None:968 raise ValueError(969 "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."970 )971 elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):972 raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")973 elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):974 raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")975 976 if negative_prompt is not None and negative_prompt_embeds is not None:977 raise ValueError(978 f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"979 f" {negative_prompt_embeds}. Please make sure to only forward one of the two."980 )981 elif negative_prompt_2 is not None and negative_prompt_embeds is not None:982 raise ValueError(983 f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:"984 f" {negative_prompt_embeds}. Please make sure to only forward one of the two."985 )986 987 if prompt_embeds is not None and negative_prompt_embeds is not None:988 if prompt_embeds.shape != negative_prompt_embeds.shape:989 raise ValueError(990 "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"991 f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"992 f" {negative_prompt_embeds.shape}."993 )994 995 if prompt_embeds is not None and pooled_prompt_embeds is None:996 raise ValueError(997 "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`."998 )999 1000 if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None:1001 raise ValueError(1002 "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`."1003 )1004 1005 def get_timesteps(self, num_inference_steps, strength, device, denoising_start=None):1006 # get the original timestep using init_timestep1007 if denoising_start is None:1008 init_timestep = min(int(num_inference_steps * strength), num_inference_steps)1009 t_start = max(num_inference_steps - init_timestep, 0)1010 else:1011 t_start = 01012 1013 timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]1014 1015 # Strength is irrelevant if we directly request a timestep to start at;1016 # that is, strength is determined by the denoising_start instead.1017 if denoising_start is not None:1018 discrete_timestep_cutoff = int(1019 round(1020 self.scheduler.config.num_train_timesteps1021 - (denoising_start * self.scheduler.config.num_train_timesteps)1022 )1023 )1024 1025 num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item()1026 if self.scheduler.order == 2 and num_inference_steps % 2 == 0:1027 # if the scheduler is a 2nd order scheduler we might have to do +11028 # because `num_inference_steps` might be even given that every timestep1029 # (except the highest one) is duplicated. If `num_inference_steps` is even it would1030 # mean that we cut the timesteps in the middle of the denoising step1031 # (between 1st and 2nd derivative) which leads to incorrect results. By adding 11032 # we ensure that the denoising process always ends after the 2nd derivate step of the scheduler1033 num_inference_steps = num_inference_steps + 11034 1035 # because t_n+1 >= t_n, we slice the timesteps starting from the end1036 timesteps = timesteps[-num_inference_steps:]1037 return timesteps, num_inference_steps1038 1039 return timesteps, num_inference_steps - t_start1040 1041 def prepare_latents(1042 self,1043 image,1044 mask,1045 width,1046 height,1047 num_channels_latents,1048 timestep,1049 batch_size,1050 num_images_per_prompt,1051 dtype,1052 device,1053 generator=None,1054 add_noise=True,1055 latents=None,1056 is_strength_max=True,1057 return_noise=False,1058 return_image_latents=False,1059 ):1060 batch_size *= num_images_per_prompt1061 1062 if image is None:1063 shape = (1064 batch_size,1065 num_channels_latents,1066 int(height) // self.vae_scale_factor,1067 int(width) // self.vae_scale_factor,1068 )1069 if isinstance(generator, list) and len(generator) != batch_size:1070 raise ValueError(1071 f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"1072 f" size of {batch_size}. Make sure the batch size matches the length of the generators."1073 )1074 1075 if latents is None:1076 latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)1077 else:1078 latents = latents.to(device)1079 1080 # scale the initial noise by the standard deviation required by the scheduler1081 latents = latents * self.scheduler.init_noise_sigma1082 return latents1083 1084 elif mask is None:1085 if not isinstance(image, (torch.Tensor, Image.Image, list)):1086 raise ValueError(1087 f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}"1088 )1089 1090 # Offload text encoder if `enable_model_cpu_offload` was enabled1091 if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:1092 self.text_encoder_2.to("cpu")1093 torch.cuda.empty_cache()1094 1095 image = image.to(device=device, dtype=dtype)1096 1097 if image.shape[1] == 4:1098 init_latents = image1099 1100 else:1101 # make sure the VAE is in float32 mode, as it overflows in float161102 if self.vae.config.force_upcast:1103 image = image.float()1104 self.vae.to(dtype=torch.float32)1105 1106 if isinstance(generator, list) and len(generator) != batch_size:1107 raise ValueError(1108 f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"1109 f" size of {batch_size}. Make sure the batch size matches the length of the generators."1110 )1111 1112 elif isinstance(generator, list):1113 init_latents = [1114 retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i])1115 for i in range(batch_size)1116 ]1117 init_latents = torch.cat(init_latents, dim=0)1118 else:1119 init_latents = retrieve_latents(self.vae.encode(image), generator=generator)1120 1121 if self.vae.config.force_upcast:1122 self.vae.to(dtype)1123 1124 init_latents = init_latents.to(dtype)1125 init_latents = self.vae.config.scaling_factor * init_latents1126 1127 if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0:1128 # expand init_latents for batch_size1129 additional_image_per_prompt = batch_size // init_latents.shape[0]1130 init_latents = torch.cat([init_latents] * additional_image_per_prompt, dim=0)1131 elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0:1132 raise ValueError(1133 f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts."1134 )1135 else:1136 init_latents = torch.cat([init_latents], dim=0)1137 1138 if add_noise:1139 shape = init_latents.shape1140 noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)1141 # get latents1142 init_latents = self.scheduler.add_noise(init_latents, noise, timestep)1143 1144 latents = init_latents1145 return latents1146 1147 else:1148 shape = (1149 batch_size,1150 num_channels_latents,1151 int(height) // self.vae_scale_factor,1152 int(width) // self.vae_scale_factor,1153 )1154 if isinstance(generator, list) and len(generator) != batch_size:1155 raise ValueError(1156 f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"1157 f" size of {batch_size}. Make sure the batch size matches the length of the generators."1158 )1159 1160 if (image is None or timestep is None) and not is_strength_max:1161 raise ValueError(1162 "Since strength < 1. initial latents are to be initialised as a combination of Image + Noise."1163 "However, either the image or the noise timestep has not been provided."1164 )1165 1166 if image.shape[1] == 4:1167 image_latents = image.to(device=device, dtype=dtype)1168 image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1)1169 elif return_image_latents or (latents is None and not is_strength_max):1170 image = image.to(device=device, dtype=dtype)1171 image_latents = self._encode_vae_image(image=image, generator=generator)1172 image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1)1173 1174 if latents is None and add_noise:1175 noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)1176 # if strength is 1. then initialise the latents to noise, else initial to image + noise1177 latents = noise if is_strength_max else self.scheduler.add_noise(image_latents, noise, timestep)1178 # if pure noise then scale the initial latents by the Scheduler's init sigma1179 latents = latents * self.scheduler.init_noise_sigma if is_strength_max else latents1180 elif add_noise:1181 noise = latents.to(device)1182 latents = noise * self.scheduler.init_noise_sigma1183 else:1184 noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)1185 latents = image_latents.to(device)1186 1187 outputs = (latents,)1188 1189 if return_noise:1190 outputs += (noise,)1191 1192 if return_image_latents:1193 outputs += (image_latents,)1194 1195 return outputs1196 1197 def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator):1198 dtype = image.dtype1199 if self.vae.config.force_upcast:1200 image = image.float()