diffusers/community-pipelines-mirror

1import inspect2import re3from typing import Any, Callable, Dict, List, Optional, Union4 5import numpy as np6import PIL.Image7import torch8from packaging import version9from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer10 11from diffusers import DiffusionPipeline12from diffusers.configuration_utils import FrozenDict13from diffusers.image_processor import VaeImageProcessor14from diffusers.loaders import FromSingleFileMixin, StableDiffusionLoraLoaderMixin, TextualInversionLoaderMixin15from diffusers.models import AutoencoderKL, UNet2DConditionModel16from diffusers.models.lora import adjust_lora_scale_text_encoder17from diffusers.pipelines.pipeline_utils import StableDiffusionMixin18from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput, StableDiffusionSafetyChecker19from diffusers.schedulers import KarrasDiffusionSchedulers20from diffusers.utils import (21    PIL_INTERPOLATION,22    USE_PEFT_BACKEND,23    deprecate,24    logging,25    scale_lora_layers,26    unscale_lora_layers,27)28from diffusers.utils.torch_utils import randn_tensor29 30 31# ------------------------------------------------------------------------------32 33logger = logging.get_logger(__name__)  # pylint: disable=invalid-name34 35re_attention = re.compile(36    r"""37\\\(|38\\\)|39\\\[|40\\]|41\\\\|42\\|43\(|44\[|45:([+-]?[.\d]+)\)|46\)|47]|48[^\\()\[\]:]+|49:50""",51    re.X,52)53 54 55def parse_prompt_attention(text):56    """57    Parses a string with attention tokens and returns a list of pairs: text and its associated weight.58    Accepted tokens are:59      (abc) - increases attention to abc by a multiplier of 1.160      (abc:3.12) - increases attention to abc by a multiplier of 3.1261      [abc] - decreases attention to abc by a multiplier of 1.162      \\( - literal character '('63      \\[ - literal character '['64      \\) - literal character ')'65      \\] - literal character ']'66      \\ - literal character '\'67      anything else - just text68    >>> parse_prompt_attention('normal text')69    [['normal text', 1.0]]70    >>> parse_prompt_attention('an (important) word')71    [['an ', 1.0], ['important', 1.1], [' word', 1.0]]72    >>> parse_prompt_attention('(unbalanced')73    [['unbalanced', 1.1]]74    >>> parse_prompt_attention('\\(literal\\]')75    [['(literal]', 1.0]]76    >>> parse_prompt_attention('(unnecessary)(parens)')77    [['unnecessaryparens', 1.1]]78    >>> parse_prompt_attention('a (((house:1.3)) [on] a (hill:0.5), sun, (((sky))).')79    [['a ', 1.0],80     ['house', 1.5730000000000004],81     [' ', 1.1],82     ['on', 1.0],83     [' a ', 1.1],84     ['hill', 0.55],85     [', sun, ', 1.1],86     ['sky', 1.4641000000000006],87     ['.', 1.1]]88    """89 90    res = []91    round_brackets = []92    square_brackets = []93 94    round_bracket_multiplier = 1.195    square_bracket_multiplier = 1 / 1.196 97    def multiply_range(start_position, multiplier):98        for p in range(start_position, len(res)):99            res[p][1] *= multiplier100 101    for m in re_attention.finditer(text):102        text = m.group(0)103        weight = m.group(1)104 105        if text.startswith("\\"):106            res.append([text[1:], 1.0])107        elif text == "(":108            round_brackets.append(len(res))109        elif text == "[":110            square_brackets.append(len(res))111        elif weight is not None and len(round_brackets) > 0:112            multiply_range(round_brackets.pop(), float(weight))113        elif text == ")" and len(round_brackets) > 0:114            multiply_range(round_brackets.pop(), round_bracket_multiplier)115        elif text == "]" and len(square_brackets) > 0:116            multiply_range(square_brackets.pop(), square_bracket_multiplier)117        else:118            res.append([text, 1.0])119 120    for pos in round_brackets:121        multiply_range(pos, round_bracket_multiplier)122 123    for pos in square_brackets:124        multiply_range(pos, square_bracket_multiplier)125 126    if len(res) == 0:127        res = [["", 1.0]]128 129    # merge runs of identical weights130    i = 0131    while i + 1 < len(res):132        if res[i][1] == res[i + 1][1]:133            res[i][0] += res[i + 1][0]134            res.pop(i + 1)135        else:136            i += 1137 138    return res139 140 141def get_prompts_with_weights(pipe: DiffusionPipeline, prompt: List[str], max_length: int):142    r"""143    Tokenize a list of prompts and return its tokens with weights of each token.144 145    No padding, starting or ending token is included.146    """147    tokens = []148    weights = []149    truncated = False150    for text in prompt:151        texts_and_weights = parse_prompt_attention(text)152        text_token = []153        text_weight = []154        for word, weight in texts_and_weights:155            # tokenize and discard the starting and the ending token156            token = pipe.tokenizer(word).input_ids[1:-1]157            text_token += token158            # copy the weight by length of token159            text_weight += [weight] * len(token)160            # stop if the text is too long (longer than truncation limit)161            if len(text_token) > max_length:162                truncated = True163                break164        # truncate165        if len(text_token) > max_length:166            truncated = True167            text_token = text_token[:max_length]168            text_weight = text_weight[:max_length]169        tokens.append(text_token)170        weights.append(text_weight)171    if truncated:172        logger.warning("Prompt was truncated. Try to shorten the prompt or increase max_embeddings_multiples")173    return tokens, weights174 175 176def pad_tokens_and_weights(tokens, weights, max_length, bos, eos, pad, no_boseos_middle=True, chunk_length=77):177    r"""178    Pad the tokens (with starting and ending tokens) and weights (with 1.0) to max_length.179    """180    max_embeddings_multiples = (max_length - 2) // (chunk_length - 2)181    weights_length = max_length if no_boseos_middle else max_embeddings_multiples * chunk_length182    for i in range(len(tokens)):183        tokens[i] = [bos] + tokens[i] + [pad] * (max_length - 1 - len(tokens[i]) - 1) + [eos]184        if no_boseos_middle:185            weights[i] = [1.0] + weights[i] + [1.0] * (max_length - 1 - len(weights[i]))186        else:187            w = []188            if len(weights[i]) == 0:189                w = [1.0] * weights_length190            else:191                for j in range(max_embeddings_multiples):192                    w.append(1.0)  # weight for starting token in this chunk193                    w += weights[i][j * (chunk_length - 2) : min(len(weights[i]), (j + 1) * (chunk_length - 2))]194                    w.append(1.0)  # weight for ending token in this chunk195                w += [1.0] * (weights_length - len(w))196            weights[i] = w[:]197 198    return tokens, weights199 200 201def get_unweighted_text_embeddings(202    pipe: DiffusionPipeline,203    text_input: torch.Tensor,204    chunk_length: int,205    no_boseos_middle: Optional[bool] = True,206    clip_skip: Optional[int] = None,207):208    """209    When the length of tokens is a multiple of the capacity of the text encoder,210    it should be split into chunks and sent to the text encoder individually.211    """212    max_embeddings_multiples = (text_input.shape[1] - 2) // (chunk_length - 2)213    if max_embeddings_multiples > 1:214        text_embeddings = []215        for i in range(max_embeddings_multiples):216            # extract the i-th chunk217            text_input_chunk = text_input[:, i * (chunk_length - 2) : (i + 1) * (chunk_length - 2) + 2].clone()218 219            # cover the head and the tail by the starting and the ending tokens220            text_input_chunk[:, 0] = text_input[0, 0]221            text_input_chunk[:, -1] = text_input[0, -1]222            if clip_skip is None:223                prompt_embeds = pipe.text_encoder(text_input_chunk.to(pipe.device))224                text_embedding = prompt_embeds[0]225            else:226                prompt_embeds = pipe.text_encoder(text_input_chunk.to(pipe.device), output_hidden_states=True)227                # Access the `hidden_states` first, that contains a tuple of228                # all the hidden states from the encoder layers. Then index into229                # the tuple to access the hidden states from the desired layer.230                prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)]231                # We also need to apply the final LayerNorm here to not mess with the232                # representations. The `last_hidden_states` that we typically use for233                # obtaining the final prompt representations passes through the LayerNorm234                # layer.235                text_embedding = pipe.text_encoder.text_model.final_layer_norm(prompt_embeds)236 237            if no_boseos_middle:238                if i == 0:239                    # discard the ending token240                    text_embedding = text_embedding[:, :-1]241                elif i == max_embeddings_multiples - 1:242                    # discard the starting token243                    text_embedding = text_embedding[:, 1:]244                else:245                    # discard both starting and ending tokens246                    text_embedding = text_embedding[:, 1:-1]247 248            text_embeddings.append(text_embedding)249        text_embeddings = torch.concat(text_embeddings, axis=1)250    else:251        if clip_skip is None:252            clip_skip = 0253        prompt_embeds = pipe.text_encoder(text_input, output_hidden_states=True)[-1][-(clip_skip + 1)]254        text_embeddings = pipe.text_encoder.text_model.final_layer_norm(prompt_embeds)255    return text_embeddings256 257 258def get_weighted_text_embeddings(259    pipe: DiffusionPipeline,260    prompt: Union[str, List[str]],261    uncond_prompt: Optional[Union[str, List[str]]] = None,262    max_embeddings_multiples: Optional[int] = 3,263    no_boseos_middle: Optional[bool] = False,264    skip_parsing: Optional[bool] = False,265    skip_weighting: Optional[bool] = False,266    clip_skip=None,267    lora_scale=None,268):269    r"""270    Prompts can be assigned with local weights using brackets. For example,271    prompt 'A (very beautiful) masterpiece' highlights the words 'very beautiful',272    and the embedding tokens corresponding to the words get multiplied by a constant, 1.1.273 274    Also, to regularize of the embedding, the weighted embedding would be scaled to preserve the original mean.275 276    Args:277        pipe (`DiffusionPipeline`):278            Pipe to provide access to the tokenizer and the text encoder.279        prompt (`str` or `List[str]`):280            The prompt or prompts to guide the image generation.281        uncond_prompt (`str` or `List[str]`):282            The unconditional prompt or prompts for guide the image generation. If unconditional prompt283            is provided, the embeddings of prompt and uncond_prompt are concatenated.284        max_embeddings_multiples (`int`, *optional*, defaults to `3`):285            The max multiple length of prompt embeddings compared to the max output length of text encoder.286        no_boseos_middle (`bool`, *optional*, defaults to `False`):287            If the length of text token is multiples of the capacity of text encoder, whether reserve the starting and288            ending token in each of the chunk in the middle.289        skip_parsing (`bool`, *optional*, defaults to `False`):290            Skip the parsing of brackets.291        skip_weighting (`bool`, *optional*, defaults to `False`):292            Skip the weighting. When the parsing is skipped, it is forced True.293    """294    # set lora scale so that monkey patched LoRA295    # function of text encoder can correctly access it296    if lora_scale is not None and isinstance(pipe, StableDiffusionLoraLoaderMixin):297        pipe._lora_scale = lora_scale298 299        # dynamically adjust the LoRA scale300        if not USE_PEFT_BACKEND:301            adjust_lora_scale_text_encoder(pipe.text_encoder, lora_scale)302        else:303            scale_lora_layers(pipe.text_encoder, lora_scale)304    max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2305    if isinstance(prompt, str):306        prompt = [prompt]307 308    if not skip_parsing:309        prompt_tokens, prompt_weights = get_prompts_with_weights(pipe, prompt, max_length - 2)310        if uncond_prompt is not None:311            if isinstance(uncond_prompt, str):312                uncond_prompt = [uncond_prompt]313            uncond_tokens, uncond_weights = get_prompts_with_weights(pipe, uncond_prompt, max_length - 2)314    else:315        prompt_tokens = [316            token[1:-1] for token in pipe.tokenizer(prompt, max_length=max_length, truncation=True).input_ids317        ]318        prompt_weights = [[1.0] * len(token) for token in prompt_tokens]319        if uncond_prompt is not None:320            if isinstance(uncond_prompt, str):321                uncond_prompt = [uncond_prompt]322            uncond_tokens = [323                token[1:-1]324                for token in pipe.tokenizer(uncond_prompt, max_length=max_length, truncation=True).input_ids325            ]326            uncond_weights = [[1.0] * len(token) for token in uncond_tokens]327 328    # round up the longest length of tokens to a multiple of (model_max_length - 2)329    max_length = max([len(token) for token in prompt_tokens])330    if uncond_prompt is not None:331        max_length = max(max_length, max([len(token) for token in uncond_tokens]))332 333    max_embeddings_multiples = min(334        max_embeddings_multiples,335        (max_length - 1) // (pipe.tokenizer.model_max_length - 2) + 1,336    )337    max_embeddings_multiples = max(1, max_embeddings_multiples)338    max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2339 340    # pad the length of tokens and weights341    bos = pipe.tokenizer.bos_token_id342    eos = pipe.tokenizer.eos_token_id343    pad = getattr(pipe.tokenizer, "pad_token_id", eos)344    prompt_tokens, prompt_weights = pad_tokens_and_weights(345        prompt_tokens,346        prompt_weights,347        max_length,348        bos,349        eos,350        pad,351        no_boseos_middle=no_boseos_middle,352        chunk_length=pipe.tokenizer.model_max_length,353    )354    prompt_tokens = torch.tensor(prompt_tokens, dtype=torch.long, device=pipe.device)355    if uncond_prompt is not None:356        uncond_tokens, uncond_weights = pad_tokens_and_weights(357            uncond_tokens,358            uncond_weights,359            max_length,360            bos,361            eos,362            pad,363            no_boseos_middle=no_boseos_middle,364            chunk_length=pipe.tokenizer.model_max_length,365        )366        uncond_tokens = torch.tensor(uncond_tokens, dtype=torch.long, device=pipe.device)367 368    # get the embeddings369    text_embeddings = get_unweighted_text_embeddings(370        pipe, prompt_tokens, pipe.tokenizer.model_max_length, no_boseos_middle=no_boseos_middle, clip_skip=clip_skip371    )372    prompt_weights = torch.tensor(prompt_weights, dtype=text_embeddings.dtype, device=text_embeddings.device)373    if uncond_prompt is not None:374        uncond_embeddings = get_unweighted_text_embeddings(375            pipe,376            uncond_tokens,377            pipe.tokenizer.model_max_length,378            no_boseos_middle=no_boseos_middle,379            clip_skip=clip_skip,380        )381        uncond_weights = torch.tensor(uncond_weights, dtype=uncond_embeddings.dtype, device=uncond_embeddings.device)382 383    # assign weights to the prompts and normalize in the sense of mean384    # TODO: should we normalize by chunk or in a whole (current implementation)?385    if (not skip_parsing) and (not skip_weighting):386        previous_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)387        text_embeddings *= prompt_weights.unsqueeze(-1)388        current_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)389        text_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)390        if uncond_prompt is not None:391            previous_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)392            uncond_embeddings *= uncond_weights.unsqueeze(-1)393            current_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)394            uncond_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)395 396    if pipe.text_encoder is not None:397        if isinstance(pipe, StableDiffusionLoraLoaderMixin) and USE_PEFT_BACKEND:398            # Retrieve the original scale by scaling back the LoRA layers399            unscale_lora_layers(pipe.text_encoder, lora_scale)400 401    if uncond_prompt is not None:402        return text_embeddings, uncond_embeddings403    return text_embeddings, None404 405 406def preprocess_image(image, batch_size):407    w, h = image.size408    w, h = (x - x % 8 for x in (w, h))  # resize to integer multiple of 8409    image = image.resize((w, h), resample=PIL_INTERPOLATION["lanczos"])410    image = np.array(image).astype(np.float32) / 255.0411    image = np.vstack([image[None].transpose(0, 3, 1, 2)] * batch_size)412    image = torch.from_numpy(image)413    return 2.0 * image - 1.0414 415 416def preprocess_mask(mask, batch_size, scale_factor=8):417    if not isinstance(mask, torch.Tensor):418        mask = mask.convert("L")419        w, h = mask.size420        w, h = (x - x % 8 for x in (w, h))  # resize to integer multiple of 8421        mask = mask.resize((w // scale_factor, h // scale_factor), resample=PIL_INTERPOLATION["nearest"])422        mask = np.array(mask).astype(np.float32) / 255.0423        mask = np.tile(mask, (4, 1, 1))424        mask = np.vstack([mask[None]] * batch_size)425        mask = 1 - mask  # repaint white, keep black426        mask = torch.from_numpy(mask)427        return mask428 429    else:430        valid_mask_channel_sizes = [1, 3]431        # if mask channel is fourth tensor dimension, permute dimensions to pytorch standard (B, C, H, W)432        if mask.shape[3] in valid_mask_channel_sizes:433            mask = mask.permute(0, 3, 1, 2)434        elif mask.shape[1] not in valid_mask_channel_sizes:435            raise ValueError(436                f"Mask channel dimension of size in {valid_mask_channel_sizes} should be second or fourth dimension,"437                f" but received mask of shape {tuple(mask.shape)}"438            )439        # (potentially) reduce mask channel dimension from 3 to 1 for broadcasting to latent shape440        mask = mask.mean(dim=1, keepdim=True)441        h, w = mask.shape[-2:]442        h, w = (x - x % 8 for x in (h, w))  # resize to integer multiple of 8443        mask = torch.nn.functional.interpolate(mask, (h // scale_factor, w // scale_factor))444        return mask445 446 447class StableDiffusionLongPromptWeightingPipeline(448    DiffusionPipeline,449    StableDiffusionMixin,450    TextualInversionLoaderMixin,451    StableDiffusionLoraLoaderMixin,452    FromSingleFileMixin,453):454    r"""455    Pipeline for text-to-image generation using Stable Diffusion without tokens length limit, and support parsing456    weighting in prompt.457 458    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the459    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)460 461    Args:462        vae ([`AutoencoderKL`]):463            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.464        text_encoder ([`CLIPTextModel`]):465            Frozen text-encoder. Stable Diffusion uses the text portion of466            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically467            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.468        tokenizer (`CLIPTokenizer`):469            Tokenizer of class470            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).471        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.472        scheduler ([`SchedulerMixin`]):473            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of474            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].475        safety_checker ([`StableDiffusionSafetyChecker`]):476            Classification module that estimates whether generated images could be considered offensive or harmful.477            Please, refer to the [model card](https://huggingface.co/CompVis/stable-diffusion-v1-4) for details.478        feature_extractor ([`CLIPImageProcessor`]):479            Model that extracts features from generated images to be used as inputs for the `safety_checker`.480    """481 482    model_cpu_offload_seq = "text_encoder-->unet->vae"483    _optional_components = ["safety_checker", "feature_extractor"]484    _exclude_from_cpu_offload = ["safety_checker"]485 486    def __init__(487        self,488        vae: AutoencoderKL,489        text_encoder: CLIPTextModel,490        tokenizer: CLIPTokenizer,491        unet: UNet2DConditionModel,492        scheduler: KarrasDiffusionSchedulers,493        safety_checker: StableDiffusionSafetyChecker,494        feature_extractor: CLIPImageProcessor,495        requires_safety_checker: bool = True,496    ):497        super().__init__()498 499        if scheduler is not None and getattr(scheduler.config, "steps_offset", 1) != 1:500            deprecation_message = (501                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"502                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "503                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"504                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"505                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"506                " file"507            )508            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)509            new_config = dict(scheduler.config)510            new_config["steps_offset"] = 1511            scheduler._internal_dict = FrozenDict(new_config)512 513        if scheduler is not None and getattr(scheduler.config, "clip_sample", False) is True:514            deprecation_message = (515                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."516                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"517                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"518                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"519                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"520            )521            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)522            new_config = dict(scheduler.config)523            new_config["clip_sample"] = False524            scheduler._internal_dict = FrozenDict(new_config)525 526        if safety_checker is None and requires_safety_checker:527            logger.warning(528                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"529                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"530                " results in services or applications open to the public. Both the diffusers team and Hugging Face"531                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"532                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"533                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."534            )535 536        if safety_checker is not None and feature_extractor is None:537            raise ValueError(538                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"539                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."540            )541 542        is_unet_version_less_0_9_0 = (543            unet is not None544            and hasattr(unet.config, "_diffusers_version")545            and version.parse(version.parse(unet.config._diffusers_version).base_version) < version.parse("0.9.0.dev0")546        )547        is_unet_sample_size_less_64 = (548            unet is not None and hasattr(unet.config, "sample_size") and unet.config.sample_size < 64549        )550        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:551            deprecation_message = (552                "The configuration file of the unet has set the default `sample_size` to smaller than"553                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"554                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"555                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"556                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"557                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"558                " in the config might lead to incorrect results in future versions. If you have downloaded this"559                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"560                " the `unet/config.json` file"561            )562            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)563            new_config = dict(unet.config)564            new_config["sample_size"] = 64565            unet._internal_dict = FrozenDict(new_config)566        self.register_modules(567            vae=vae,568            text_encoder=text_encoder,569            tokenizer=tokenizer,570            unet=unet,571            scheduler=scheduler,572            safety_checker=safety_checker,573            feature_extractor=feature_extractor,574        )575        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8576 577        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)578        self.register_to_config(579            requires_safety_checker=requires_safety_checker,580        )581 582    def _encode_prompt(583        self,584        prompt,585        device,586        num_images_per_prompt,587        do_classifier_free_guidance,588        negative_prompt=None,589        max_embeddings_multiples=3,590        prompt_embeds: Optional[torch.Tensor] = None,591        negative_prompt_embeds: Optional[torch.Tensor] = None,592        clip_skip: Optional[int] = None,593        lora_scale: Optional[float] = None,594    ):595        r"""596        Encodes the prompt into text encoder hidden states.597 598        Args:599            prompt (`str` or `list(int)`):600                prompt to be encoded601            device: (`torch.device`):602                torch device603            num_images_per_prompt (`int`):604                number of images that should be generated per prompt605            do_classifier_free_guidance (`bool`):606                whether to use classifier free guidance or not607            negative_prompt (`str` or `List[str]`):608                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored609                if `guidance_scale` is less than `1`).610            max_embeddings_multiples (`int`, *optional*, defaults to `3`):611                The max multiple length of prompt embeddings compared to the max output length of text encoder.612        """613        if prompt is not None and isinstance(prompt, str):614            batch_size = 1615        elif prompt is not None and isinstance(prompt, list):616            batch_size = len(prompt)617        else:618            batch_size = prompt_embeds.shape[0]619 620        if negative_prompt_embeds is None:621            if negative_prompt is None:622                negative_prompt = [""] * batch_size623            elif isinstance(negative_prompt, str):624                negative_prompt = [negative_prompt] * batch_size625            if batch_size != len(negative_prompt):626                raise ValueError(627                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"628                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"629                    " the batch size of `prompt`."630                )631        if prompt_embeds is None or negative_prompt_embeds is None:632            if isinstance(self, TextualInversionLoaderMixin):633                prompt = self.maybe_convert_prompt(prompt, self.tokenizer)634                if do_classifier_free_guidance and negative_prompt_embeds is None:635                    negative_prompt = self.maybe_convert_prompt(negative_prompt, self.tokenizer)636 637            prompt_embeds1, negative_prompt_embeds1 = get_weighted_text_embeddings(638                pipe=self,639                prompt=prompt,640                uncond_prompt=negative_prompt if do_classifier_free_guidance else None,641                max_embeddings_multiples=max_embeddings_multiples,642                clip_skip=clip_skip,643                lora_scale=lora_scale,644            )645            if prompt_embeds is None:646                prompt_embeds = prompt_embeds1647            if negative_prompt_embeds is None:648                negative_prompt_embeds = negative_prompt_embeds1649 650        bs_embed, seq_len, _ = prompt_embeds.shape651        # duplicate text embeddings for each generation per prompt, using mps friendly method652        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)653        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)654 655        if do_classifier_free_guidance:656            bs_embed, seq_len, _ = negative_prompt_embeds.shape657            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)658            negative_prompt_embeds = negative_prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)659            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])660 661        return prompt_embeds662 663    def check_inputs(664        self,665        prompt,666        height,667        width,668        strength,669        callback_steps,670        negative_prompt=None,671        prompt_embeds=None,672        negative_prompt_embeds=None,673    ):674        if height % 8 != 0 or width % 8 != 0:675            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")676 677        if strength < 0 or strength > 1:678            raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")679 680        if (callback_steps is None) or (681            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)682        ):683            raise ValueError(684                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"685                f" {type(callback_steps)}."686            )687 688        if prompt is not None and prompt_embeds is not None:689            raise ValueError(690                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"691                " only forward one of the two."692            )693        elif prompt is None and prompt_embeds is None:694            raise ValueError(695                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."696            )697        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):698            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")699 700        if negative_prompt is not None and negative_prompt_embeds is not None:701            raise ValueError(702                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"703                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."704            )705 706        if prompt_embeds is not None and negative_prompt_embeds is not None:707            if prompt_embeds.shape != negative_prompt_embeds.shape:708                raise ValueError(709                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"710                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"711                    f" {negative_prompt_embeds.shape}."712                )713 714    def get_timesteps(self, num_inference_steps, strength, device, is_text2img):715        if is_text2img:716            return self.scheduler.timesteps.to(device), num_inference_steps717        else:718            # get the original timestep using init_timestep719            init_timestep = min(int(num_inference_steps * strength), num_inference_steps)720 721            t_start = max(num_inference_steps - init_timestep, 0)722            timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]723 724            return timesteps, num_inference_steps - t_start725 726    def run_safety_checker(self, image, device, dtype):727        if self.safety_checker is not None:728            safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)729            image, has_nsfw_concept = self.safety_checker(730                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)731            )732        else:733            has_nsfw_concept = None734        return image, has_nsfw_concept735 736    def decode_latents(self, latents):737        latents = 1 / self.vae.config.scaling_factor * latents738        image = self.vae.decode(latents).sample739        image = (image / 2 + 0.5).clamp(0, 1)740        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16741        image = image.cpu().permute(0, 2, 3, 1).float().numpy()742        return image743 744    def prepare_extra_step_kwargs(self, generator, eta):745        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature746        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.747        # eta corresponds to η in DDIM paper: https://huggingface.co/papers/2010.02502748        # and should be between [0, 1]749 750        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())751        extra_step_kwargs = {}752        if accepts_eta:753            extra_step_kwargs["eta"] = eta754 755        # check if the scheduler accepts generator756        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())757        if accepts_generator:758            extra_step_kwargs["generator"] = generator759        return extra_step_kwargs760 761    def prepare_latents(762        self,763        image,764        timestep,765        num_images_per_prompt,766        batch_size,767        num_channels_latents,768        height,769        width,770        dtype,771        device,772        generator,773        latents=None,774    ):775        if image is None:776            batch_size = batch_size * num_images_per_prompt777            shape = (778                batch_size,779                num_channels_latents,780                int(height) // self.vae_scale_factor,781                int(width) // self.vae_scale_factor,782            )783            if isinstance(generator, list) and len(generator) != batch_size:784                raise ValueError(785                    f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"786                    f" size of {batch_size}. Make sure the batch size matches the length of the generators."787                )788 789            if latents is None:790                latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)791            else:792                latents = latents.to(device)793 794            # scale the initial noise by the standard deviation required by the scheduler795            latents = latents * self.scheduler.init_noise_sigma796            return latents, None, None797        else:798            image = image.to(device=self.device, dtype=dtype)799            init_latent_dist = self.vae.encode(image).latent_dist800            init_latents = init_latent_dist.sample(generator=generator)801            init_latents = self.vae.config.scaling_factor * init_latents802 803            # Expand init_latents for batch_size and num_images_per_prompt804            init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0)805            init_latents_orig = init_latents806 807            # add noise to latents using the timesteps808            noise = randn_tensor(init_latents.shape, generator=generator, device=self.device, dtype=dtype)809            init_latents = self.scheduler.add_noise(init_latents, noise, timestep)810            latents = init_latents811            return latents, init_latents_orig, noise812 813    @torch.no_grad()814    def __call__(815        self,816        prompt: Union[str, List[str]],817        negative_prompt: Optional[Union[str, List[str]]] = None,818        image: Union[torch.Tensor, PIL.Image.Image] = None,819        mask_image: Union[torch.Tensor, PIL.Image.Image] = None,820        height: int = 512,821        width: int = 512,822        num_inference_steps: int = 50,823        guidance_scale: float = 7.5,824        strength: float = 0.8,825        num_images_per_prompt: Optional[int] = 1,826        add_predicted_noise: Optional[bool] = False,827        eta: float = 0.0,828        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,829        latents: Optional[torch.Tensor] = None,830        prompt_embeds: Optional[torch.Tensor] = None,831        negative_prompt_embeds: Optional[torch.Tensor] = None,832        max_embeddings_multiples: Optional[int] = 3,833        output_type: Optional[str] = "pil",834        return_dict: bool = True,835        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,836        is_cancelled_callback: Optional[Callable[[], bool]] = None,837        clip_skip: Optional[int] = None,838        callback_steps: int = 1,839        cross_attention_kwargs: Optional[Dict[str, Any]] = None,840    ):841        r"""842        Function invoked when calling the pipeline for generation.843 844        Args:845            prompt (`str` or `List[str]`):846                The prompt or prompts to guide the image generation.847            negative_prompt (`str` or `List[str]`, *optional*):848                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored849                if `guidance_scale` is less than `1`).850            image (`torch.Tensor` or `PIL.Image.Image`):851                `Image`, or tensor representing an image batch, that will be used as the starting point for the852                process.853            mask_image (`torch.Tensor` or `PIL.Image.Image`):854                `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be855                replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a856                PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should857                contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.858            height (`int`, *optional*, defaults to 512):859                The height in pixels of the generated image.860            width (`int`, *optional*, defaults to 512):861                The width in pixels of the generated image.862            num_inference_steps (`int`, *optional*, defaults to 50):863                The number of denoising steps. More denoising steps usually lead to a higher quality image at the864                expense of slower inference.865            guidance_scale (`float`, *optional*, defaults to 7.5):866                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://huggingface.co/papers/2207.12598).867                `guidance_scale` is defined as `w` of equation 2. of [Imagen868                Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale >869                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,870                usually at the expense of lower image quality.871            strength (`float`, *optional*, defaults to 0.8):872                Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1.873                `image` will be used as a starting point, adding more noise to it the larger the `strength`. The874                number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added875                noise will be maximum and the denoising process will run for the full number of iterations specified in876                `num_inference_steps`. A value of 1, therefore, essentially ignores `image`.877            num_images_per_prompt (`int`, *optional*, defaults to 1):878                The number of images to generate per prompt.879            add_predicted_noise (`bool`, *optional*, defaults to True):880                Use predicted noise instead of random noise when constructing noisy versions of the original image in881                the reverse diffusion process882            eta (`float`, *optional*, defaults to 0.0):883                Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to884                [`schedulers.DDIMScheduler`], will be ignored for others.885            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):886                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)887                to make generation deterministic.888            latents (`torch.Tensor`, *optional*):889                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image890                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents891                tensor will ge generated by sampling using the supplied random `generator`.892            prompt_embeds (`torch.Tensor`, *optional*):893                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not894                provided, text embeddings will be generated from `prompt` input argument.895            negative_prompt_embeds (`torch.Tensor`, *optional*):896                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt897                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input898                argument.899            max_embeddings_multiples (`int`, *optional*, defaults to `3`):900                The max multiple length of prompt embeddings compared to the max output length of text encoder.901            output_type (`str`, *optional*, defaults to `"pil"`):902                The output format of the generate image. Choose between903                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.904            return_dict (`bool`, *optional*, defaults to `True`):905                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a906                plain tuple.907            callback (`Callable`, *optional*):908                A function that will be called every `callback_steps` steps during inference. The function will be909                called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.910            is_cancelled_callback (`Callable`, *optional*):911                A function that will be called every `callback_steps` steps during inference. If the function returns912                `True`, the inference will be cancelled.913            clip_skip (`int`, *optional*):914                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that915                the output of the pre-final layer will be used for computing the prompt embeddings.916            callback_steps (`int`, *optional*, defaults to 1):917                The frequency at which the `callback` function will be called. If not specified, the callback will be918                called at every step.919            cross_attention_kwargs (`dict`, *optional*):920                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under921                `self.processor` in922                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).923 924        Returns:925            `None` if cancelled by `is_cancelled_callback`,926            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:927            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.928            When returning a tuple, the first element is a list with the generated images, and the second element is a929            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"930            (nsfw) content, according to the `safety_checker`.931        """932        # 0. Default height and width to unet933        height = height or self.unet.config.sample_size * self.vae_scale_factor934        width = width or self.unet.config.sample_size * self.vae_scale_factor935 936        # 1. Check inputs. Raise error if not correct937        self.check_inputs(938            prompt, height, width, strength, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds939        )940 941        # 2. Define call parameters942        if prompt is not None and isinstance(prompt, str):943            batch_size = 1944        elif prompt is not None and isinstance(prompt, list):945            batch_size = len(prompt)946        else:947            batch_size = prompt_embeds.shape[0]948 949        device = self._execution_device950        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)951        # of the Imagen paper: https://huggingface.co/papers/2205.11487 . `guidance_scale = 1`952        # corresponds to doing no classifier free guidance.953        do_classifier_free_guidance = guidance_scale > 1.0954        lora_scale = cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None955 956        # 3. Encode input prompt957        prompt_embeds = self._encode_prompt(958            prompt,959            device,960            num_images_per_prompt,961            do_classifier_free_guidance,962            negative_prompt,963            max_embeddings_multiples,964            prompt_embeds=prompt_embeds,965            negative_prompt_embeds=negative_prompt_embeds,966            clip_skip=clip_skip,967            lora_scale=lora_scale,968        )969        dtype = prompt_embeds.dtype970 971        # 4. Preprocess image and mask972        if isinstance(image, PIL.Image.Image):973            image = preprocess_image(image, batch_size)974        if image is not None:975            image = image.to(device=self.device, dtype=dtype)976        if isinstance(mask_image, PIL.Image.Image):977            mask_image = preprocess_mask(mask_image, batch_size, self.vae_scale_factor)978        if mask_image is not None:979            mask = mask_image.to(device=self.device, dtype=dtype)980            mask = torch.cat([mask] * num_images_per_prompt)981        else:982            mask = None983 984        # 5. set timesteps985        self.scheduler.set_timesteps(num_inference_steps, device=device)986        timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device, image is None)987        latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)988 989        # 6. Prepare latent variables990        latents, init_latents_orig, noise = self.prepare_latents(991            image,992            latent_timestep,993            num_images_per_prompt,994            batch_size,995            self.unet.config.in_channels,996            height,997            width,998            dtype,999            device,1000            generator,1001            latents,1002        )1003 1004        # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline1005        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)1006 1007        # 8. Denoising loop1008        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order1009        with self.progress_bar(total=num_inference_steps) as progress_bar:1010            for i, t in enumerate(timesteps):1011                # expand the latents if we are doing classifier free guidance1012                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents1013                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)1014 1015                # predict the noise residual1016                noise_pred = self.unet(1017                    latent_model_input,1018                    t,1019                    encoder_hidden_states=prompt_embeds,1020                    cross_attention_kwargs=cross_attention_kwargs,1021                ).sample1022 1023                # perform guidance1024                if do_classifier_free_guidance:1025                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)1026                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)1027 1028                # compute the previous noisy sample x_t -> x_t-11029                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample1030 1031                if mask is not None:1032                    # masking1033                    if add_predicted_noise:1034                        init_latents_proper = self.scheduler.add_noise(1035                            init_latents_orig, noise_pred_uncond, torch.tensor([t])1036                        )1037                    else:1038                        init_latents_proper = self.scheduler.add_noise(init_latents_orig, noise, torch.tensor([t]))1039                    latents = (init_latents_proper * mask) + (latents * (1 - mask))1040 1041                # call the callback, if provided1042                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):1043                    progress_bar.update()1044                    if i % callback_steps == 0:1045                        if callback is not None:1046                            step_idx = i // getattr(self.scheduler, "order", 1)1047                            callback(step_idx, t, latents)1048                        if is_cancelled_callback is not None and is_cancelled_callback():1049                            return None1050 1051        if output_type == "latent":1052            image = latents1053            has_nsfw_concept = None1054        elif output_type == "pil":1055            # 9. Post-processing1056            image = self.decode_latents(latents)1057 1058            # 10. Run safety checker1059            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)1060 1061            # 11. Convert to PIL1062            image = self.numpy_to_pil(image)1063        else:1064            # 9. Post-processing1065            image = self.decode_latents(latents)1066 1067            # 10. Run safety checker1068            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)1069 1070        # Offload last model to CPU1071        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:1072            self.final_offload_hook.offload()1073 1074        if not return_dict:1075            return image, has_nsfw_concept1076 1077        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)1078 1079    def text2img(1080        self,1081        prompt: Union[str, List[str]],1082        negative_prompt: Optional[Union[str, List[str]]] = None,1083        height: int = 512,1084        width: int = 512,1085        num_inference_steps: int = 50,1086        guidance_scale: float = 7.5,1087        num_images_per_prompt: Optional[int] = 1,1088        eta: float = 0.0,1089        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,1090        latents: Optional[torch.Tensor] = None,1091        prompt_embeds: Optional[torch.Tensor] = None,1092        negative_prompt_embeds: Optional[torch.Tensor] = None,1093        max_embeddings_multiples: Optional[int] = 3,1094        output_type: Optional[str] = "pil",1095        return_dict: bool = True,1096        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,1097        is_cancelled_callback: Optional[Callable[[], bool]] = None,1098        clip_skip=None,1099        callback_steps: int = 1,1100        cross_attention_kwargs: Optional[Dict[str, Any]] = None,1101    ):1102        r"""1103        Function for text-to-image generation.1104        Args:1105            prompt (`str` or `List[str]`):1106                The prompt or prompts to guide the image generation.1107            negative_prompt (`str` or `List[str]`, *optional*):1108                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored1109                if `guidance_scale` is less than `1`).1110            height (`int`, *optional*, defaults to 512):1111                The height in pixels of the generated image.1112            width (`int`, *optional*, defaults to 512):1113                The width in pixels of the generated image.1114            num_inference_steps (`int`, *optional*, defaults to 50):1115                The number of denoising steps. More denoising steps usually lead to a higher quality image at the1116                expense of slower inference.1117            guidance_scale (`float`, *optional*, defaults to 7.5):1118                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://huggingface.co/papers/2207.12598).1119                `guidance_scale` is defined as `w` of equation 2. of [Imagen1120                Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale >1121                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,1122                usually at the expense of lower image quality.1123            num_images_per_prompt (`int`, *optional*, defaults to 1):1124                The number of images to generate per prompt.1125            eta (`float`, *optional*, defaults to 0.0):1126                Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to1127                [`schedulers.DDIMScheduler`], will be ignored for others.1128            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):1129                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)1130                to make generation deterministic.1131            latents (`torch.Tensor`, *optional*):1132                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image1133                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents1134                tensor will ge generated by sampling using the supplied random `generator`.1135            prompt_embeds (`torch.Tensor`, *optional*):1136                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not1137                provided, text embeddings will be generated from `prompt` input argument.1138            negative_prompt_embeds (`torch.Tensor`, *optional*):1139                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt1140                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input1141                argument.1142            max_embeddings_multiples (`int`, *optional*, defaults to `3`):1143                The max multiple length of prompt embeddings compared to the max output length of text encoder.1144            output_type (`str`, *optional*, defaults to `"pil"`):1145                The output format of the generate image. Choose between1146                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.1147            return_dict (`bool`, *optional*, defaults to `True`):1148                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a1149                plain tuple.1150            callback (`Callable`, *optional*):1151                A function that will be called every `callback_steps` steps during inference. The function will be1152                called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.1153            is_cancelled_callback (`Callable`, *optional*):1154                A function that will be called every `callback_steps` steps during inference. If the function returns1155                `True`, the inference will be cancelled.1156            clip_skip (`int`, *optional*):1157                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that1158                the output of the pre-final layer will be used for computing the prompt embeddings.1159            callback_steps (`int`, *optional*, defaults to 1):1160                The frequency at which the `callback` function will be called. If not specified, the callback will be1161                called at every step.1162            cross_attention_kwargs (`dict`, *optional*):1163                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under1164                `self.processor` in1165                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).1166 1167        Returns:1168            `None` if cancelled by `is_cancelled_callback`,1169            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:1170            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.1171            When returning a tuple, the first element is a list with the generated images, and the second element is a1172            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"1173            (nsfw) content, according to the `safety_checker`.1174        """1175        return self.__call__(1176            prompt=prompt,1177            negative_prompt=negative_prompt,1178            height=height,1179            width=width,1180            num_inference_steps=num_inference_steps,1181            guidance_scale=guidance_scale,1182            num_images_per_prompt=num_images_per_prompt,1183            eta=eta,1184            generator=generator,1185            latents=latents,1186            prompt_embeds=prompt_embeds,1187            negative_prompt_embeds=negative_prompt_embeds,1188            max_embeddings_multiples=max_embeddings_multiples,1189            output_type=output_type,1190            return_dict=return_dict,1191            callback=callback,1192            is_cancelled_callback=is_cancelled_callback,1193            clip_skip=clip_skip,1194            callback_steps=callback_steps,1195            cross_attention_kwargs=cross_attention_kwargs,1196        )1197 1198    def img2img(1199        self,1200        image: Union[torch.Tensor, PIL.Image.Image],

Showing the first 1,200 of 1432 lines. Download the file for the rest.