tsi-org/tango
0
1import inspect2import re3from typing import Callable, List, Optional, Union4 5import numpy as np6import PIL7import torch8from packaging import version9from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer10 11import diffusers12from diffusers import SchedulerMixin, StableDiffusionPipeline13from diffusers.models import AutoencoderKL, UNet2DConditionModel14from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput, StableDiffusionSafetyChecker15from diffusers.utils import logging16 17 18try:19 from diffusers.utils import PIL_INTERPOLATION20except ImportError:21 if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"):22 PIL_INTERPOLATION = {23 "linear": PIL.Image.Resampling.BILINEAR,24 "bilinear": PIL.Image.Resampling.BILINEAR,25 "bicubic": PIL.Image.Resampling.BICUBIC,26 "lanczos": PIL.Image.Resampling.LANCZOS,27 "nearest": PIL.Image.Resampling.NEAREST,28 }29 else:30 PIL_INTERPOLATION = {31 "linear": PIL.Image.LINEAR,32 "bilinear": PIL.Image.BILINEAR,33 "bicubic": PIL.Image.BICUBIC,34 "lanczos": PIL.Image.LANCZOS,35 "nearest": PIL.Image.NEAREST,36 }37# ------------------------------------------------------------------------------38 39logger = logging.get_logger(__name__) # pylint: disable=invalid-name40 41re_attention = re.compile(42 r"""43\\\(|44\\\)|45\\\[|46\\]|47\\\\|48\\|49\(|50\[|51:([+-]?[.\d]+)\)|52\)|53]|54[^\\()\[\]:]+|55:56""",57 re.X,58)59 60 61def parse_prompt_attention(text):62 """63 Parses a string with attention tokens and returns a list of pairs: text and its associated weight.64 Accepted tokens are:65 (abc) - increases attention to abc by a multiplier of 1.166 (abc:3.12) - increases attention to abc by a multiplier of 3.1267 [abc] - decreases attention to abc by a multiplier of 1.168 \( - literal character '('69 \[ - literal character '['70 \) - literal character ')'71 \] - literal character ']'72 \\ - literal character '\'73 anything else - just text74 >>> parse_prompt_attention('normal text')75 [['normal text', 1.0]]76 >>> parse_prompt_attention('an (important) word')77 [['an ', 1.0], ['important', 1.1], [' word', 1.0]]78 >>> parse_prompt_attention('(unbalanced')79 [['unbalanced', 1.1]]80 >>> parse_prompt_attention('\(literal\]')81 [['(literal]', 1.0]]82 >>> parse_prompt_attention('(unnecessary)(parens)')83 [['unnecessaryparens', 1.1]]84 >>> parse_prompt_attention('a (((house:1.3)) [on] a (hill:0.5), sun, (((sky))).')85 [['a ', 1.0],86 ['house', 1.5730000000000004],87 [' ', 1.1],88 ['on', 1.0],89 [' a ', 1.1],90 ['hill', 0.55],91 [', sun, ', 1.1],92 ['sky', 1.4641000000000006],93 ['.', 1.1]]94 """95 96 res = []97 round_brackets = []98 square_brackets = []99 100 round_bracket_multiplier = 1.1101 square_bracket_multiplier = 1 / 1.1102 103 def multiply_range(start_position, multiplier):104 for p in range(start_position, len(res)):105 res[p][1] *= multiplier106 107 for m in re_attention.finditer(text):108 text = m.group(0)109 weight = m.group(1)110 111 if text.startswith("\\"):112 res.append([text[1:], 1.0])113 elif text == "(":114 round_brackets.append(len(res))115 elif text == "[":116 square_brackets.append(len(res))117 elif weight is not None and len(round_brackets) > 0:118 multiply_range(round_brackets.pop(), float(weight))119 elif text == ")" and len(round_brackets) > 0:120 multiply_range(round_brackets.pop(), round_bracket_multiplier)121 elif text == "]" and len(square_brackets) > 0:122 multiply_range(square_brackets.pop(), square_bracket_multiplier)123 else:124 res.append([text, 1.0])125 126 for pos in round_brackets:127 multiply_range(pos, round_bracket_multiplier)128 129 for pos in square_brackets:130 multiply_range(pos, square_bracket_multiplier)131 132 if len(res) == 0:133 res = [["", 1.0]]134 135 # merge runs of identical weights136 i = 0137 while i + 1 < len(res):138 if res[i][1] == res[i + 1][1]:139 res[i][0] += res[i + 1][0]140 res.pop(i + 1)141 else:142 i += 1143 144 return res145 146 147def get_prompts_with_weights(pipe: StableDiffusionPipeline, prompt: List[str], max_length: int):148 r"""149 Tokenize a list of prompts and return its tokens with weights of each token.150 151 No padding, starting or ending token is included.152 """153 tokens = []154 weights = []155 truncated = False156 for text in prompt:157 texts_and_weights = parse_prompt_attention(text)158 text_token = []159 text_weight = []160 for word, weight in texts_and_weights:161 # tokenize and discard the starting and the ending token162 token = pipe.tokenizer(word).input_ids[1:-1]163 text_token += token164 # copy the weight by length of token165 text_weight += [weight] * len(token)166 # stop if the text is too long (longer than truncation limit)167 if len(text_token) > max_length:168 truncated = True169 break170 # truncate171 if len(text_token) > max_length:172 truncated = True173 text_token = text_token[:max_length]174 text_weight = text_weight[:max_length]175 tokens.append(text_token)176 weights.append(text_weight)177 if truncated:178 logger.warning("Prompt was truncated. Try to shorten the prompt or increase max_embeddings_multiples")179 return tokens, weights180 181 182def pad_tokens_and_weights(tokens, weights, max_length, bos, eos, pad, no_boseos_middle=True, chunk_length=77):183 r"""184 Pad the tokens (with starting and ending tokens) and weights (with 1.0) to max_length.185 """186 max_embeddings_multiples = (max_length - 2) // (chunk_length - 2)187 weights_length = max_length if no_boseos_middle else max_embeddings_multiples * chunk_length188 for i in range(len(tokens)):189 tokens[i] = [bos] + tokens[i] + [pad] * (max_length - 1 - len(tokens[i]) - 1) + [eos]190 if no_boseos_middle:191 weights[i] = [1.0] + weights[i] + [1.0] * (max_length - 1 - len(weights[i]))192 else:193 w = []194 if len(weights[i]) == 0:195 w = [1.0] * weights_length196 else:197 for j in range(max_embeddings_multiples):198 w.append(1.0) # weight for starting token in this chunk199 w += weights[i][j * (chunk_length - 2) : min(len(weights[i]), (j + 1) * (chunk_length - 2))]200 w.append(1.0) # weight for ending token in this chunk201 w += [1.0] * (weights_length - len(w))202 weights[i] = w[:]203 204 return tokens, weights205 206 207def get_unweighted_text_embeddings(208 pipe: StableDiffusionPipeline,209 text_input: torch.Tensor,210 chunk_length: int,211 no_boseos_middle: Optional[bool] = True,212):213 """214 When the length of tokens is a multiple of the capacity of the text encoder,215 it should be split into chunks and sent to the text encoder individually.216 """217 max_embeddings_multiples = (text_input.shape[1] - 2) // (chunk_length - 2)218 if max_embeddings_multiples > 1:219 text_embeddings = []220 for i in range(max_embeddings_multiples):221 # extract the i-th chunk222 text_input_chunk = text_input[:, i * (chunk_length - 2) : (i + 1) * (chunk_length - 2) + 2].clone()223 224 # cover the head and the tail by the starting and the ending tokens225 text_input_chunk[:, 0] = text_input[0, 0]226 text_input_chunk[:, -1] = text_input[0, -1]227 text_embedding = pipe.text_encoder(text_input_chunk)[0]228 229 if no_boseos_middle:230 if i == 0:231 # discard the ending token232 text_embedding = text_embedding[:, :-1]233 elif i == max_embeddings_multiples - 1:234 # discard the starting token235 text_embedding = text_embedding[:, 1:]236 else:237 # discard both starting and ending tokens238 text_embedding = text_embedding[:, 1:-1]239 240 text_embeddings.append(text_embedding)241 text_embeddings = torch.concat(text_embeddings, axis=1)242 else:243 text_embeddings = pipe.text_encoder(text_input)[0]244 return text_embeddings245 246 247def get_weighted_text_embeddings(248 pipe: StableDiffusionPipeline,249 prompt: Union[str, List[str]],250 uncond_prompt: Optional[Union[str, List[str]]] = None,251 max_embeddings_multiples: Optional[int] = 3,252 no_boseos_middle: Optional[bool] = False,253 skip_parsing: Optional[bool] = False,254 skip_weighting: Optional[bool] = False,255):256 r"""257 Prompts can be assigned with local weights using brackets. For example,258 prompt 'A (very beautiful) masterpiece' highlights the words 'very beautiful',259 and the embedding tokens corresponding to the words get multiplied by a constant, 1.1.260 261 Also, to regularize of the embedding, the weighted embedding would be scaled to preserve the original mean.262 263 Args:264 pipe (`StableDiffusionPipeline`):265 Pipe to provide access to the tokenizer and the text encoder.266 prompt (`str` or `List[str]`):267 The prompt or prompts to guide the image generation.268 uncond_prompt (`str` or `List[str]`):269 The unconditional prompt or prompts for guide the image generation. If unconditional prompt270 is provided, the embeddings of prompt and uncond_prompt are concatenated.271 max_embeddings_multiples (`int`, *optional*, defaults to `3`):272 The max multiple length of prompt embeddings compared to the max output length of text encoder.273 no_boseos_middle (`bool`, *optional*, defaults to `False`):274 If the length of text token is multiples of the capacity of text encoder, whether reserve the starting and275 ending token in each of the chunk in the middle.276 skip_parsing (`bool`, *optional*, defaults to `False`):277 Skip the parsing of brackets.278 skip_weighting (`bool`, *optional*, defaults to `False`):279 Skip the weighting. When the parsing is skipped, it is forced True.280 """281 max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2282 if isinstance(prompt, str):283 prompt = [prompt]284 285 if not skip_parsing:286 prompt_tokens, prompt_weights = get_prompts_with_weights(pipe, prompt, max_length - 2)287 if uncond_prompt is not None:288 if isinstance(uncond_prompt, str):289 uncond_prompt = [uncond_prompt]290 uncond_tokens, uncond_weights = get_prompts_with_weights(pipe, uncond_prompt, max_length - 2)291 else:292 prompt_tokens = [293 token[1:-1] for token in pipe.tokenizer(prompt, max_length=max_length, truncation=True).input_ids294 ]295 prompt_weights = [[1.0] * len(token) for token in prompt_tokens]296 if uncond_prompt is not None:297 if isinstance(uncond_prompt, str):298 uncond_prompt = [uncond_prompt]299 uncond_tokens = [300 token[1:-1]301 for token in pipe.tokenizer(uncond_prompt, max_length=max_length, truncation=True).input_ids302 ]303 uncond_weights = [[1.0] * len(token) for token in uncond_tokens]304 305 # round up the longest length of tokens to a multiple of (model_max_length - 2)306 max_length = max([len(token) for token in prompt_tokens])307 if uncond_prompt is not None:308 max_length = max(max_length, max([len(token) for token in uncond_tokens]))309 310 max_embeddings_multiples = min(311 max_embeddings_multiples,312 (max_length - 1) // (pipe.tokenizer.model_max_length - 2) + 1,313 )314 max_embeddings_multiples = max(1, max_embeddings_multiples)315 max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2316 317 # pad the length of tokens and weights318 bos = pipe.tokenizer.bos_token_id319 eos = pipe.tokenizer.eos_token_id320 pad = getattr(pipe.tokenizer, "pad_token_id", eos)321 prompt_tokens, prompt_weights = pad_tokens_and_weights(322 prompt_tokens,323 prompt_weights,324 max_length,325 bos,326 eos,327 pad,328 no_boseos_middle=no_boseos_middle,329 chunk_length=pipe.tokenizer.model_max_length,330 )331 prompt_tokens = torch.tensor(prompt_tokens, dtype=torch.long, device=pipe.device)332 if uncond_prompt is not None:333 uncond_tokens, uncond_weights = pad_tokens_and_weights(334 uncond_tokens,335 uncond_weights,336 max_length,337 bos,338 eos,339 pad,340 no_boseos_middle=no_boseos_middle,341 chunk_length=pipe.tokenizer.model_max_length,342 )343 uncond_tokens = torch.tensor(uncond_tokens, dtype=torch.long, device=pipe.device)344 345 # get the embeddings346 text_embeddings = get_unweighted_text_embeddings(347 pipe,348 prompt_tokens,349 pipe.tokenizer.model_max_length,350 no_boseos_middle=no_boseos_middle,351 )352 prompt_weights = torch.tensor(prompt_weights, dtype=text_embeddings.dtype, device=pipe.device)353 if uncond_prompt is not None:354 uncond_embeddings = get_unweighted_text_embeddings(355 pipe,356 uncond_tokens,357 pipe.tokenizer.model_max_length,358 no_boseos_middle=no_boseos_middle,359 )360 uncond_weights = torch.tensor(uncond_weights, dtype=uncond_embeddings.dtype, device=pipe.device)361 362 # assign weights to the prompts and normalize in the sense of mean363 # TODO: should we normalize by chunk or in a whole (current implementation)?364 if (not skip_parsing) and (not skip_weighting):365 previous_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)366 text_embeddings *= prompt_weights.unsqueeze(-1)367 current_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)368 text_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)369 if uncond_prompt is not None:370 previous_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)371 uncond_embeddings *= uncond_weights.unsqueeze(-1)372 current_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)373 uncond_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)374 375 if uncond_prompt is not None:376 return text_embeddings, uncond_embeddings377 return text_embeddings, None378 379 380def preprocess_image(image):381 w, h = image.size382 w, h = (x - x % 32 for x in (w, h)) # resize to integer multiple of 32383 image = image.resize((w, h), resample=PIL_INTERPOLATION["lanczos"])384 image = np.array(image).astype(np.float32) / 255.0385 image = image[None].transpose(0, 3, 1, 2)386 image = torch.from_numpy(image)387 return 2.0 * image - 1.0388 389 390def preprocess_mask(mask, scale_factor=8):391 mask = mask.convert("L")392 w, h = mask.size393 w, h = (x - x % 32 for x in (w, h)) # resize to integer multiple of 32394 mask = mask.resize((w // scale_factor, h // scale_factor), resample=PIL_INTERPOLATION["nearest"])395 mask = np.array(mask).astype(np.float32) / 255.0396 mask = np.tile(mask, (4, 1, 1))397 mask = mask[None].transpose(0, 1, 2, 3) # what does this step do?398 mask = 1 - mask # repaint white, keep black399 mask = torch.from_numpy(mask)400 return mask401 402 403class StableDiffusionLongPromptWeightingPipeline(StableDiffusionPipeline):404 r"""405 Pipeline for text-to-image generation using Stable Diffusion without tokens length limit, and support parsing406 weighting in prompt.407 408 This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the409 library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)410 411 Args:412 vae ([`AutoencoderKL`]):413 Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.414 text_encoder ([`CLIPTextModel`]):415 Frozen text-encoder. Stable Diffusion uses the text portion of416 [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically417 the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.418 tokenizer (`CLIPTokenizer`):419 Tokenizer of class420 [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).421 unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.422 scheduler ([`SchedulerMixin`]):423 A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of424 [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].425 safety_checker ([`StableDiffusionSafetyChecker`]):426 Classification module that estimates whether generated images could be considered offensive or harmful.427 Please, refer to the [model card](https://huggingface.co/CompVis/stable-diffusion-v1-4) for details.428 feature_extractor ([`CLIPImageProcessor`]):429 Model that extracts features from generated images to be used as inputs for the `safety_checker`.430 """431 432 if version.parse(version.parse(diffusers.__version__).base_version) >= version.parse("0.9.0"):433 434 def __init__(435 self,436 vae: AutoencoderKL,437 text_encoder: CLIPTextModel,438 tokenizer: CLIPTokenizer,439 unet: UNet2DConditionModel,440 scheduler: SchedulerMixin,441 safety_checker: StableDiffusionSafetyChecker,442 feature_extractor: CLIPImageProcessor,443 requires_safety_checker: bool = True,444 ):445 super().__init__(446 vae=vae,447 text_encoder=text_encoder,448 tokenizer=tokenizer,449 unet=unet,450 scheduler=scheduler,451 safety_checker=safety_checker,452 feature_extractor=feature_extractor,453 requires_safety_checker=requires_safety_checker,454 )455 self.__init__additional__()456 457 else:458 459 def __init__(460 self,461 vae: AutoencoderKL,462 text_encoder: CLIPTextModel,463 tokenizer: CLIPTokenizer,464 unet: UNet2DConditionModel,465 scheduler: SchedulerMixin,466 safety_checker: StableDiffusionSafetyChecker,467 feature_extractor: CLIPImageProcessor,468 ):469 super().__init__(470 vae=vae,471 text_encoder=text_encoder,472 tokenizer=tokenizer,473 unet=unet,474 scheduler=scheduler,475 safety_checker=safety_checker,476 feature_extractor=feature_extractor,477 )478 self.__init__additional__()479 480 def __init__additional__(self):481 if not hasattr(self, "vae_scale_factor"):482 setattr(self, "vae_scale_factor", 2 ** (len(self.vae.config.block_out_channels) - 1))483 484 @property485 def _execution_device(self):486 r"""487 Returns the device on which the pipeline's models will be executed. After calling488 `pipeline.enable_sequential_cpu_offload()` the execution device can only be inferred from Accelerate's module489 hooks.490 """491 if self.device != torch.device("meta") or not hasattr(self.unet, "_hf_hook"):492 return self.device493 for module in self.unet.modules():494 if (495 hasattr(module, "_hf_hook")496 and hasattr(module._hf_hook, "execution_device")497 and module._hf_hook.execution_device is not None498 ):499 return torch.device(module._hf_hook.execution_device)500 return self.device501 502 def _encode_prompt(503 self,504 prompt,505 device,506 num_images_per_prompt,507 do_classifier_free_guidance,508 negative_prompt,509 max_embeddings_multiples,510 ):511 r"""512 Encodes the prompt into text encoder hidden states.513 514 Args:515 prompt (`str` or `list(int)`):516 prompt to be encoded517 device: (`torch.device`):518 torch device519 num_images_per_prompt (`int`):520 number of images that should be generated per prompt521 do_classifier_free_guidance (`bool`):522 whether to use classifier free guidance or not523 negative_prompt (`str` or `List[str]`):524 The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored525 if `guidance_scale` is less than `1`).526 max_embeddings_multiples (`int`, *optional*, defaults to `3`):527 The max multiple length of prompt embeddings compared to the max output length of text encoder.528 """529 batch_size = len(prompt) if isinstance(prompt, list) else 1530 531 if negative_prompt is None:532 negative_prompt = [""] * batch_size533 elif isinstance(negative_prompt, str):534 negative_prompt = [negative_prompt] * batch_size535 if batch_size != len(negative_prompt):536 raise ValueError(537 f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"538 f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"539 " the batch size of `prompt`."540 )541 542 text_embeddings, uncond_embeddings = get_weighted_text_embeddings(543 pipe=self,544 prompt=prompt,545 uncond_prompt=negative_prompt if do_classifier_free_guidance else None,546 max_embeddings_multiples=max_embeddings_multiples,547 )548 bs_embed, seq_len, _ = text_embeddings.shape549 text_embeddings = text_embeddings.repeat(1, num_images_per_prompt, 1)550 text_embeddings = text_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1)551 552 if do_classifier_free_guidance:553 bs_embed, seq_len, _ = uncond_embeddings.shape554 uncond_embeddings = uncond_embeddings.repeat(1, num_images_per_prompt, 1)555 uncond_embeddings = uncond_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1)556 text_embeddings = torch.cat([uncond_embeddings, text_embeddings])557 558 return text_embeddings559 560 def check_inputs(self, prompt, height, width, strength, callback_steps):561 if not isinstance(prompt, str) and not isinstance(prompt, list):562 raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")563 564 if strength < 0 or strength > 1:565 raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")566 567 if height % 8 != 0 or width % 8 != 0:568 raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")569 570 if (callback_steps is None) or (571 callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)572 ):573 raise ValueError(574 f"`callback_steps` has to be a positive integer but is {callback_steps} of type"575 f" {type(callback_steps)}."576 )577 578 def get_timesteps(self, num_inference_steps, strength, device, is_text2img):579 if is_text2img:580 return self.scheduler.timesteps.to(device), num_inference_steps581 else:582 # get the original timestep using init_timestep583 offset = self.scheduler.config.get("steps_offset", 0)584 init_timestep = int(num_inference_steps * strength) + offset585 init_timestep = min(init_timestep, num_inference_steps)586 587 t_start = max(num_inference_steps - init_timestep + offset, 0)588 timesteps = self.scheduler.timesteps[t_start:].to(device)589 return timesteps, num_inference_steps - t_start590 591 def run_safety_checker(self, image, device, dtype):592 if self.safety_checker is not None:593 safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)594 image, has_nsfw_concept = self.safety_checker(595 images=image, clip_input=safety_checker_input.pixel_values.to(dtype)596 )597 else:598 has_nsfw_concept = None599 return image, has_nsfw_concept600 601 def decode_latents(self, latents):602 latents = 1 / 0.18215 * latents603 image = self.vae.decode(latents).sample604 image = (image / 2 + 0.5).clamp(0, 1)605 # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16606 image = image.cpu().permute(0, 2, 3, 1).float().numpy()607 return image608 609 def prepare_extra_step_kwargs(self, generator, eta):610 # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature611 # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.612 # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502613 # and should be between [0, 1]614 615 accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())616 extra_step_kwargs = {}617 if accepts_eta:618 extra_step_kwargs["eta"] = eta619 620 # check if the scheduler accepts generator621 accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())622 if accepts_generator:623 extra_step_kwargs["generator"] = generator624 return extra_step_kwargs625 626 def prepare_latents(self, image, timestep, batch_size, height, width, dtype, device, generator, latents=None):627 if image is None:628 shape = (629 batch_size,630 self.unet.in_channels,631 height // self.vae_scale_factor,632 width // self.vae_scale_factor,633 )634 635 if latents is None:636 if device.type == "mps":637 # randn does not work reproducibly on mps638 latents = torch.randn(shape, generator=generator, device="cpu", dtype=dtype).to(device)639 else:640 latents = torch.randn(shape, generator=generator, device=device, dtype=dtype)641 else:642 if latents.shape != shape:643 raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}")644 latents = latents.to(device)645 646 # scale the initial noise by the standard deviation required by the scheduler647 latents = latents * self.scheduler.init_noise_sigma648 return latents, None, None649 else:650 init_latent_dist = self.vae.encode(image).latent_dist651 init_latents = init_latent_dist.sample(generator=generator)652 init_latents = 0.18215 * init_latents653 init_latents = torch.cat([init_latents] * batch_size, dim=0)654 init_latents_orig = init_latents655 shape = init_latents.shape656 657 # add noise to latents using the timesteps658 if device.type == "mps":659 noise = torch.randn(shape, generator=generator, device="cpu", dtype=dtype).to(device)660 else:661 noise = torch.randn(shape, generator=generator, device=device, dtype=dtype)662 latents = self.scheduler.add_noise(init_latents, noise, timestep)663 return latents, init_latents_orig, noise664 665 @torch.no_grad()666 def __call__(667 self,668 prompt: Union[str, List[str]],669 negative_prompt: Optional[Union[str, List[str]]] = None,670 image: Union[torch.FloatTensor, PIL.Image.Image] = None,671 mask_image: Union[torch.FloatTensor, PIL.Image.Image] = None,672 height: int = 512,673 width: int = 512,674 num_inference_steps: int = 50,675 guidance_scale: float = 7.5,676 strength: float = 0.8,677 num_images_per_prompt: Optional[int] = 1,678 eta: float = 0.0,679 generator: Optional[torch.Generator] = None,680 latents: Optional[torch.FloatTensor] = None,681 max_embeddings_multiples: Optional[int] = 3,682 output_type: Optional[str] = "pil",683 return_dict: bool = True,684 callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,685 is_cancelled_callback: Optional[Callable[[], bool]] = None,686 callback_steps: int = 1,687 ):688 r"""689 Function invoked when calling the pipeline for generation.690 691 Args:692 prompt (`str` or `List[str]`):693 The prompt or prompts to guide the image generation.694 negative_prompt (`str` or `List[str]`, *optional*):695 The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored696 if `guidance_scale` is less than `1`).697 image (`torch.FloatTensor` or `PIL.Image.Image`):698 `Image`, or tensor representing an image batch, that will be used as the starting point for the699 process.700 mask_image (`torch.FloatTensor` or `PIL.Image.Image`):701 `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be702 replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a703 PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should704 contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.705 height (`int`, *optional*, defaults to 512):706 The height in pixels of the generated image.707 width (`int`, *optional*, defaults to 512):708 The width in pixels of the generated image.709 num_inference_steps (`int`, *optional*, defaults to 50):710 The number of denoising steps. More denoising steps usually lead to a higher quality image at the711 expense of slower inference.712 guidance_scale (`float`, *optional*, defaults to 7.5):713 Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).714 `guidance_scale` is defined as `w` of equation 2. of [Imagen715 Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >716 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,717 usually at the expense of lower image quality.718 strength (`float`, *optional*, defaults to 0.8):719 Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1.720 `image` will be used as a starting point, adding more noise to it the larger the `strength`. The721 number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added722 noise will be maximum and the denoising process will run for the full number of iterations specified in723 `num_inference_steps`. A value of 1, therefore, essentially ignores `image`.724 num_images_per_prompt (`int`, *optional*, defaults to 1):725 The number of images to generate per prompt.726 eta (`float`, *optional*, defaults to 0.0):727 Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to728 [`schedulers.DDIMScheduler`], will be ignored for others.729 generator (`torch.Generator`, *optional*):730 A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation731 deterministic.732 latents (`torch.FloatTensor`, *optional*):733 Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image734 generation. Can be used to tweak the same generation with different prompts. If not provided, a latents735 tensor will ge generated by sampling using the supplied random `generator`.736 max_embeddings_multiples (`int`, *optional*, defaults to `3`):737 The max multiple length of prompt embeddings compared to the max output length of text encoder.738 output_type (`str`, *optional*, defaults to `"pil"`):739 The output format of the generate image. Choose between740 [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.741 return_dict (`bool`, *optional*, defaults to `True`):742 Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a743 plain tuple.744 callback (`Callable`, *optional*):745 A function that will be called every `callback_steps` steps during inference. The function will be746 called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.747 is_cancelled_callback (`Callable`, *optional*):748 A function that will be called every `callback_steps` steps during inference. If the function returns749 `True`, the inference will be cancelled.750 callback_steps (`int`, *optional*, defaults to 1):751 The frequency at which the `callback` function will be called. If not specified, the callback will be752 called at every step.753 754 Returns:755 `None` if cancelled by `is_cancelled_callback`,756 [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:757 [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.758 When returning a tuple, the first element is a list with the generated images, and the second element is a759 list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"760 (nsfw) content, according to the `safety_checker`.761 """762 # 0. Default height and width to unet763 height = height or self.unet.config.sample_size * self.vae_scale_factor764 width = width or self.unet.config.sample_size * self.vae_scale_factor765 766 # 1. Check inputs. Raise error if not correct767 self.check_inputs(prompt, height, width, strength, callback_steps)768 769 # 2. Define call parameters770 batch_size = 1 if isinstance(prompt, str) else len(prompt)771 device = self._execution_device772 # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)773 # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`774 # corresponds to doing no classifier free guidance.775 do_classifier_free_guidance = guidance_scale > 1.0776 777 # 3. Encode input prompt778 text_embeddings = self._encode_prompt(779 prompt,780 device,781 num_images_per_prompt,782 do_classifier_free_guidance,783 negative_prompt,784 max_embeddings_multiples,785 )786 dtype = text_embeddings.dtype787 788 # 4. Preprocess image and mask789 if isinstance(image, PIL.Image.Image):790 image = preprocess_image(image)791 if image is not None:792 image = image.to(device=self.device, dtype=dtype)793 if isinstance(mask_image, PIL.Image.Image):794 mask_image = preprocess_mask(mask_image, self.vae_scale_factor)795 if mask_image is not None:796 mask = mask_image.to(device=self.device, dtype=dtype)797 mask = torch.cat([mask] * batch_size * num_images_per_prompt)798 else:799 mask = None800 801 # 5. set timesteps802 self.scheduler.set_timesteps(num_inference_steps, device=device)803 timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device, image is None)804 latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)805 806 # 6. Prepare latent variables807 latents, init_latents_orig, noise = self.prepare_latents(808 image,809 latent_timestep,810 batch_size * num_images_per_prompt,811 height,812 width,813 dtype,814 device,815 generator,816 latents,817 )818 819 # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline820 extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)821 822 # 8. Denoising loop823 for i, t in enumerate(self.progress_bar(timesteps)):824 # expand the latents if we are doing classifier free guidance825 latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents826 latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)827 828 # predict the noise residual829 noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample830 831 # perform guidance832 if do_classifier_free_guidance:833 noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)834 noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)835 836 # compute the previous noisy sample x_t -> x_t-1837 latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample838 839 if mask is not None:840 # masking841 init_latents_proper = self.scheduler.add_noise(init_latents_orig, noise, torch.tensor([t]))842 latents = (init_latents_proper * mask) + (latents * (1 - mask))843 844 # call the callback, if provided845 if i % callback_steps == 0:846 if callback is not None:847 callback(i, t, latents)848 if is_cancelled_callback is not None and is_cancelled_callback():849 return None850 851 # 9. Post-processing852 image = self.decode_latents(latents)853 854 # 10. Run safety checker855 image, has_nsfw_concept = self.run_safety_checker(image, device, text_embeddings.dtype)856 857 # 11. Convert to PIL858 if output_type == "pil":859 image = self.numpy_to_pil(image)860 861 if not return_dict:862 return image, has_nsfw_concept863 864 return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)865 866 def text2img(867 self,868 prompt: Union[str, List[str]],869 negative_prompt: Optional[Union[str, List[str]]] = None,870 height: int = 512,871 width: int = 512,872 num_inference_steps: int = 50,873 guidance_scale: float = 7.5,874 num_images_per_prompt: Optional[int] = 1,875 eta: float = 0.0,876 generator: Optional[torch.Generator] = None,877 latents: Optional[torch.FloatTensor] = None,878 max_embeddings_multiples: Optional[int] = 3,879 output_type: Optional[str] = "pil",880 return_dict: bool = True,881 callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,882 is_cancelled_callback: Optional[Callable[[], bool]] = None,883 callback_steps: int = 1,884 ):885 r"""886 Function for text-to-image generation.887 Args:888 prompt (`str` or `List[str]`):889 The prompt or prompts to guide the image generation.890 negative_prompt (`str` or `List[str]`, *optional*):891 The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored892 if `guidance_scale` is less than `1`).893 height (`int`, *optional*, defaults to 512):894 The height in pixels of the generated image.895 width (`int`, *optional*, defaults to 512):896 The width in pixels of the generated image.897 num_inference_steps (`int`, *optional*, defaults to 50):898 The number of denoising steps. More denoising steps usually lead to a higher quality image at the899 expense of slower inference.900 guidance_scale (`float`, *optional*, defaults to 7.5):901 Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).902 `guidance_scale` is defined as `w` of equation 2. of [Imagen903 Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >904 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,905 usually at the expense of lower image quality.906 num_images_per_prompt (`int`, *optional*, defaults to 1):907 The number of images to generate per prompt.908 eta (`float`, *optional*, defaults to 0.0):909 Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to910 [`schedulers.DDIMScheduler`], will be ignored for others.911 generator (`torch.Generator`, *optional*):912 A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation913 deterministic.914 latents (`torch.FloatTensor`, *optional*):915 Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image916 generation. Can be used to tweak the same generation with different prompts. If not provided, a latents917 tensor will ge generated by sampling using the supplied random `generator`.918 max_embeddings_multiples (`int`, *optional*, defaults to `3`):919 The max multiple length of prompt embeddings compared to the max output length of text encoder.920 output_type (`str`, *optional*, defaults to `"pil"`):921 The output format of the generate image. Choose between922 [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.923 return_dict (`bool`, *optional*, defaults to `True`):924 Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a925 plain tuple.926 callback (`Callable`, *optional*):927 A function that will be called every `callback_steps` steps during inference. The function will be928 called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.929 is_cancelled_callback (`Callable`, *optional*):930 A function that will be called every `callback_steps` steps during inference. If the function returns931 `True`, the inference will be cancelled.932 callback_steps (`int`, *optional*, defaults to 1):933 The frequency at which the `callback` function will be called. If not specified, the callback will be934 called at every step.935 Returns:936 [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:937 [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.938 When returning a tuple, the first element is a list with the generated images, and the second element is a939 list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"940 (nsfw) content, according to the `safety_checker`.941 """942 return self.__call__(943 prompt=prompt,944 negative_prompt=negative_prompt,945 height=height,946 width=width,947 num_inference_steps=num_inference_steps,948 guidance_scale=guidance_scale,949 num_images_per_prompt=num_images_per_prompt,950 eta=eta,951 generator=generator,952 latents=latents,953 max_embeddings_multiples=max_embeddings_multiples,954 output_type=output_type,955 return_dict=return_dict,956 callback=callback,957 is_cancelled_callback=is_cancelled_callback,958 callback_steps=callback_steps,959 )960 961 def img2img(962 self,963 image: Union[torch.FloatTensor, PIL.Image.Image],964 prompt: Union[str, List[str]],965 negative_prompt: Optional[Union[str, List[str]]] = None,966 strength: float = 0.8,967 num_inference_steps: Optional[int] = 50,968 guidance_scale: Optional[float] = 7.5,969 num_images_per_prompt: Optional[int] = 1,970 eta: Optional[float] = 0.0,971 generator: Optional[torch.Generator] = None,972 max_embeddings_multiples: Optional[int] = 3,973 output_type: Optional[str] = "pil",974 return_dict: bool = True,975 callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,976 is_cancelled_callback: Optional[Callable[[], bool]] = None,977 callback_steps: int = 1,978 ):979 r"""980 Function for image-to-image generation.981 Args:982 image (`torch.FloatTensor` or `PIL.Image.Image`):983 `Image`, or tensor representing an image batch, that will be used as the starting point for the984 process.985 prompt (`str` or `List[str]`):986 The prompt or prompts to guide the image generation.987 negative_prompt (`str` or `List[str]`, *optional*):988 The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored989 if `guidance_scale` is less than `1`).990 strength (`float`, *optional*, defaults to 0.8):991 Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1.992 `image` will be used as a starting point, adding more noise to it the larger the `strength`. The993 number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added994 noise will be maximum and the denoising process will run for the full number of iterations specified in995 `num_inference_steps`. A value of 1, therefore, essentially ignores `image`.996 num_inference_steps (`int`, *optional*, defaults to 50):997 The number of denoising steps. More denoising steps usually lead to a higher quality image at the998 expense of slower inference. This parameter will be modulated by `strength`.999 guidance_scale (`float`, *optional*, defaults to 7.5):1000 Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).1001 `guidance_scale` is defined as `w` of equation 2. of [Imagen1002 Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >1003 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,1004 usually at the expense of lower image quality.1005 num_images_per_prompt (`int`, *optional*, defaults to 1):1006 The number of images to generate per prompt.1007 eta (`float`, *optional*, defaults to 0.0):1008 Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to1009 [`schedulers.DDIMScheduler`], will be ignored for others.1010 generator (`torch.Generator`, *optional*):1011 A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation1012 deterministic.1013 max_embeddings_multiples (`int`, *optional*, defaults to `3`):1014 The max multiple length of prompt embeddings compared to the max output length of text encoder.1015 output_type (`str`, *optional*, defaults to `"pil"`):1016 The output format of the generate image. Choose between1017 [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.1018 return_dict (`bool`, *optional*, defaults to `True`):1019 Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a1020 plain tuple.1021 callback (`Callable`, *optional*):1022 A function that will be called every `callback_steps` steps during inference. The function will be1023 called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.1024 is_cancelled_callback (`Callable`, *optional*):1025 A function that will be called every `callback_steps` steps during inference. If the function returns1026 `True`, the inference will be cancelled.1027 callback_steps (`int`, *optional*, defaults to 1):1028 The frequency at which the `callback` function will be called. If not specified, the callback will be1029 called at every step.1030 Returns:1031 [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:1032 [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.1033 When returning a tuple, the first element is a list with the generated images, and the second element is a1034 list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"1035 (nsfw) content, according to the `safety_checker`.1036 """1037 return self.__call__(1038 prompt=prompt,1039 negative_prompt=negative_prompt,1040 image=image,1041 num_inference_steps=num_inference_steps,1042 guidance_scale=guidance_scale,1043 strength=strength,1044 num_images_per_prompt=num_images_per_prompt,1045 eta=eta,1046 generator=generator,1047 max_embeddings_multiples=max_embeddings_multiples,1048 output_type=output_type,1049 return_dict=return_dict,1050 callback=callback,1051 is_cancelled_callback=is_cancelled_callback,1052 callback_steps=callback_steps,1053 )1054 1055 def inpaint(1056 self,1057 image: Union[torch.FloatTensor, PIL.Image.Image],1058 mask_image: Union[torch.FloatTensor, PIL.Image.Image],1059 prompt: Union[str, List[str]],1060 negative_prompt: Optional[Union[str, List[str]]] = None,1061 strength: float = 0.8,1062 num_inference_steps: Optional[int] = 50,1063 guidance_scale: Optional[float] = 7.5,1064 num_images_per_prompt: Optional[int] = 1,1065 eta: Optional[float] = 0.0,1066 generator: Optional[torch.Generator] = None,1067 max_embeddings_multiples: Optional[int] = 3,1068 output_type: Optional[str] = "pil",1069 return_dict: bool = True,1070 callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,1071 is_cancelled_callback: Optional[Callable[[], bool]] = None,1072 callback_steps: int = 1,1073 ):1074 r"""1075 Function for inpaint.1076 Args:1077 image (`torch.FloatTensor` or `PIL.Image.Image`):1078 `Image`, or tensor representing an image batch, that will be used as the starting point for the1079 process. This is the image whose masked region will be inpainted.1080 mask_image (`torch.FloatTensor` or `PIL.Image.Image`):1081 `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be1082 replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a1083 PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should1084 contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.1085 prompt (`str` or `List[str]`):1086 The prompt or prompts to guide the image generation.1087 negative_prompt (`str` or `List[str]`, *optional*):1088 The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored1089 if `guidance_scale` is less than `1`).1090 strength (`float`, *optional*, defaults to 0.8):1091 Conceptually, indicates how much to inpaint the masked area. Must be between 0 and 1. When `strength`1092 is 1, the denoising process will be run on the masked area for the full number of iterations specified1093 in `num_inference_steps`. `image` will be used as a reference for the masked area, adding more1094 noise to that region the larger the `strength`. If `strength` is 0, no inpainting will occur.1095 num_inference_steps (`int`, *optional*, defaults to 50):1096 The reference number of denoising steps. More denoising steps usually lead to a higher quality image at1097 the expense of slower inference. This parameter will be modulated by `strength`, as explained above.1098 guidance_scale (`float`, *optional*, defaults to 7.5):1099 Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).1100 `guidance_scale` is defined as `w` of equation 2. of [Imagen1101 Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >1102 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,1103 usually at the expense of lower image quality.1104 num_images_per_prompt (`int`, *optional*, defaults to 1):1105 The number of images to generate per prompt.1106 eta (`float`, *optional*, defaults to 0.0):1107 Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to1108 [`schedulers.DDIMScheduler`], will be ignored for others.1109 generator (`torch.Generator`, *optional*):1110 A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation1111 deterministic.1112 max_embeddings_multiples (`int`, *optional*, defaults to `3`):1113 The max multiple length of prompt embeddings compared to the max output length of text encoder.1114 output_type (`str`, *optional*, defaults to `"pil"`):1115 The output format of the generate image. Choose between1116 [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.1117 return_dict (`bool`, *optional*, defaults to `True`):1118 Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a1119 plain tuple.1120 callback (`Callable`, *optional*):1121 A function that will be called every `callback_steps` steps during inference. The function will be1122 called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.1123 is_cancelled_callback (`Callable`, *optional*):1124 A function that will be called every `callback_steps` steps during inference. If the function returns1125 `True`, the inference will be cancelled.1126 callback_steps (`int`, *optional*, defaults to 1):1127 The frequency at which the `callback` function will be called. If not specified, the callback will be1128 called at every step.1129 Returns:1130 [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:1131 [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.1132 When returning a tuple, the first element is a list with the generated images, and the second element is a1133 list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"1134 (nsfw) content, according to the `safety_checker`.1135 """1136 return self.__call__(1137 prompt=prompt,1138 negative_prompt=negative_prompt,1139 image=image,1140 mask_image=mask_image,1141 num_inference_steps=num_inference_steps,1142 guidance_scale=guidance_scale,1143 strength=strength,1144 num_images_per_prompt=num_images_per_prompt,1145 eta=eta,1146 generator=generator,1147 max_embeddings_multiples=max_embeddings_multiples,1148 output_type=output_type,1149 return_dict=return_dict,1150 callback=callback,1151 is_cancelled_callback=is_cancelled_callback,1152 callback_steps=callback_steps,1153 )1154