diffusers/community-pipelines-mirror

1import inspect2import re3from typing import Any, Callable, Dict, List, Optional, Union4 5import numpy as np6import PIL.Image7import torch8from packaging import version9from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer10 11from diffusers import DiffusionPipeline12from diffusers.configuration_utils import FrozenDict13from diffusers.image_processor import VaeImageProcessor14from diffusers.loaders import FromSingleFileMixin, StableDiffusionLoraLoaderMixin, TextualInversionLoaderMixin15from diffusers.models import AutoencoderKL, UNet2DConditionModel16from diffusers.models.lora import adjust_lora_scale_text_encoder17from diffusers.pipelines.pipeline_utils import StableDiffusionMixin18from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput, StableDiffusionSafetyChecker19from diffusers.schedulers import KarrasDiffusionSchedulers20from diffusers.utils import (21    PIL_INTERPOLATION,22    USE_PEFT_BACKEND,23    deprecate,24    logging,25    scale_lora_layers,26    unscale_lora_layers,27)28from diffusers.utils.torch_utils import randn_tensor29 30 31# ------------------------------------------------------------------------------32 33logger = logging.get_logger(__name__)  # pylint: disable=invalid-name34 35re_attention = re.compile(36    r"""37\\\(|38\\\)|39\\\[|40\\]|41\\\\|42\\|43\(|44\[|45:([+-]?[.\d]+)\)|46\)|47]|48[^\\()\[\]:]+|49:50""",51    re.X,52)53 54 55def parse_prompt_attention(text):56    """57    Parses a string with attention tokens and returns a list of pairs: text and its associated weight.58    Accepted tokens are:59      (abc) - increases attention to abc by a multiplier of 1.160      (abc:3.12) - increases attention to abc by a multiplier of 3.1261      [abc] - decreases attention to abc by a multiplier of 1.162      \\( - literal character '('63      \\[ - literal character '['64      \\) - literal character ')'65      \\] - literal character ']'66      \\ - literal character '\'67      anything else - just text68    >>> parse_prompt_attention('normal text')69    [['normal text', 1.0]]70    >>> parse_prompt_attention('an (important) word')71    [['an ', 1.0], ['important', 1.1], [' word', 1.0]]72    >>> parse_prompt_attention('(unbalanced')73    [['unbalanced', 1.1]]74    >>> parse_prompt_attention('\\(literal\\]')75    [['(literal]', 1.0]]76    >>> parse_prompt_attention('(unnecessary)(parens)')77    [['unnecessaryparens', 1.1]]78    >>> parse_prompt_attention('a (((house:1.3)) [on] a (hill:0.5), sun, (((sky))).')79    [['a ', 1.0],80     ['house', 1.5730000000000004],81     [' ', 1.1],82     ['on', 1.0],83     [' a ', 1.1],84     ['hill', 0.55],85     [', sun, ', 1.1],86     ['sky', 1.4641000000000006],87     ['.', 1.1]]88    """89 90    res = []91    round_brackets = []92    square_brackets = []93 94    round_bracket_multiplier = 1.195    square_bracket_multiplier = 1 / 1.196 97    def multiply_range(start_position, multiplier):98        for p in range(start_position, len(res)):99            res[p][1] *= multiplier100 101    for m in re_attention.finditer(text):102        text = m.group(0)103        weight = m.group(1)104 105        if text.startswith("\\"):106            res.append([text[1:], 1.0])107        elif text == "(":108            round_brackets.append(len(res))109        elif text == "[":110            square_brackets.append(len(res))111        elif weight is not None and len(round_brackets) > 0:112            multiply_range(round_brackets.pop(), float(weight))113        elif text == ")" and len(round_brackets) > 0:114            multiply_range(round_brackets.pop(), round_bracket_multiplier)115        elif text == "]" and len(square_brackets) > 0:116            multiply_range(square_brackets.pop(), square_bracket_multiplier)117        else:118            res.append([text, 1.0])119 120    for pos in round_brackets:121        multiply_range(pos, round_bracket_multiplier)122 123    for pos in square_brackets:124        multiply_range(pos, square_bracket_multiplier)125 126    if len(res) == 0:127        res = [["", 1.0]]128 129    # merge runs of identical weights130    i = 0131    while i + 1 < len(res):132        if res[i][1] == res[i + 1][1]:133            res[i][0] += res[i + 1][0]134            res.pop(i + 1)135        else:136            i += 1137 138    return res139 140 141def get_prompts_with_weights(pipe: DiffusionPipeline, prompt: List[str], max_length: int):142    r"""143    Tokenize a list of prompts and return its tokens with weights of each token.144 145    No padding, starting or ending token is included.146    """147    tokens = []148    weights = []149    truncated = False150    for text in prompt:151        texts_and_weights = parse_prompt_attention(text)152        text_token = []153        text_weight = []154        for word, weight in texts_and_weights:155            # tokenize and discard the starting and the ending token156            token = pipe.tokenizer(word).input_ids[1:-1]157            text_token += token158            # copy the weight by length of token159            text_weight += [weight] * len(token)160            # stop if the text is too long (longer than truncation limit)161            if len(text_token) > max_length:162                truncated = True163                break164        # truncate165        if len(text_token) > max_length:166            truncated = True167            text_token = text_token[:max_length]168            text_weight = text_weight[:max_length]169        tokens.append(text_token)170        weights.append(text_weight)171    if truncated:172        logger.warning("Prompt was truncated. Try to shorten the prompt or increase max_embeddings_multiples")173    return tokens, weights174 175 176def pad_tokens_and_weights(tokens, weights, max_length, bos, eos, pad, no_boseos_middle=True, chunk_length=77):177    r"""178    Pad the tokens (with starting and ending tokens) and weights (with 1.0) to max_length.179    """180    max_embeddings_multiples = (max_length - 2) // (chunk_length - 2)181    weights_length = max_length if no_boseos_middle else max_embeddings_multiples * chunk_length182    for i in range(len(tokens)):183        tokens[i] = [bos] + tokens[i] + [pad] * (max_length - 1 - len(tokens[i]) - 1) + [eos]184        if no_boseos_middle:185            weights[i] = [1.0] + weights[i] + [1.0] * (max_length - 1 - len(weights[i]))186        else:187            w = []188            if len(weights[i]) == 0:189                w = [1.0] * weights_length190            else:191                for j in range(max_embeddings_multiples):192                    w.append(1.0)  # weight for starting token in this chunk193                    w += weights[i][j * (chunk_length - 2) : min(len(weights[i]), (j + 1) * (chunk_length - 2))]194                    w.append(1.0)  # weight for ending token in this chunk195                w += [1.0] * (weights_length - len(w))196            weights[i] = w[:]197 198    return tokens, weights199 200 201def get_unweighted_text_embeddings(202    pipe: DiffusionPipeline,203    text_input: torch.Tensor,204    chunk_length: int,205    no_boseos_middle: Optional[bool] = True,206    clip_skip: Optional[int] = None,207):208    """209    When the length of tokens is a multiple of the capacity of the text encoder,210    it should be split into chunks and sent to the text encoder individually.211    """212    max_embeddings_multiples = (text_input.shape[1] - 2) // (chunk_length - 2)213    if max_embeddings_multiples > 1:214        text_embeddings = []215        for i in range(max_embeddings_multiples):216            # extract the i-th chunk217            text_input_chunk = text_input[:, i * (chunk_length - 2) : (i + 1) * (chunk_length - 2) + 2].clone()218 219            # cover the head and the tail by the starting and the ending tokens220            text_input_chunk[:, 0] = text_input[0, 0]221            text_input_chunk[:, -1] = text_input[0, -1]222            if clip_skip is None:223                prompt_embeds = pipe.text_encoder(text_input_chunk.to(pipe.device))224                text_embedding = prompt_embeds[0]225            else:226                prompt_embeds = pipe.text_encoder(text_input_chunk.to(pipe.device), output_hidden_states=True)227                # Access the `hidden_states` first, that contains a tuple of228                # all the hidden states from the encoder layers. Then index into229                # the tuple to access the hidden states from the desired layer.230                prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)]231                # We also need to apply the final LayerNorm here to not mess with the232                # representations. The `last_hidden_states` that we typically use for233                # obtaining the final prompt representations passes through the LayerNorm234                # layer.235                text_embedding = pipe.text_encoder.text_model.final_layer_norm(prompt_embeds)236 237            if no_boseos_middle:238                if i == 0:239                    # discard the ending token240                    text_embedding = text_embedding[:, :-1]241                elif i == max_embeddings_multiples - 1:242                    # discard the starting token243                    text_embedding = text_embedding[:, 1:]244                else:245                    # discard both starting and ending tokens246                    text_embedding = text_embedding[:, 1:-1]247 248            text_embeddings.append(text_embedding)249        text_embeddings = torch.concat(text_embeddings, axis=1)250    else:251        if clip_skip is None:252            clip_skip = 0253        prompt_embeds = pipe.text_encoder(text_input, output_hidden_states=True)[-1][-(clip_skip + 1)]254        text_embeddings = pipe.text_encoder.text_model.final_layer_norm(prompt_embeds)255    return text_embeddings256 257 258def get_weighted_text_embeddings(259    pipe: DiffusionPipeline,260    prompt: Union[str, List[str]],261    uncond_prompt: Optional[Union[str, List[str]]] = None,262    max_embeddings_multiples: Optional[int] = 3,263    no_boseos_middle: Optional[bool] = False,264    skip_parsing: Optional[bool] = False,265    skip_weighting: Optional[bool] = False,266    clip_skip=None,267    lora_scale=None,268):269    r"""270    Prompts can be assigned with local weights using brackets. For example,271    prompt 'A (very beautiful) masterpiece' highlights the words 'very beautiful',272    and the embedding tokens corresponding to the words get multiplied by a constant, 1.1.273 274    Also, to regularize of the embedding, the weighted embedding would be scaled to preserve the original mean.275 276    Args:277        pipe (`DiffusionPipeline`):278            Pipe to provide access to the tokenizer and the text encoder.279        prompt (`str` or `List[str]`):280            The prompt or prompts to guide the image generation.281        uncond_prompt (`str` or `List[str]`):282            The unconditional prompt or prompts for guide the image generation. If unconditional prompt283            is provided, the embeddings of prompt and uncond_prompt are concatenated.284        max_embeddings_multiples (`int`, *optional*, defaults to `3`):285            The max multiple length of prompt embeddings compared to the max output length of text encoder.286        no_boseos_middle (`bool`, *optional*, defaults to `False`):287            If the length of text token is multiples of the capacity of text encoder, whether reserve the starting and288            ending token in each of the chunk in the middle.289        skip_parsing (`bool`, *optional*, defaults to `False`):290            Skip the parsing of brackets.291        skip_weighting (`bool`, *optional*, defaults to `False`):292            Skip the weighting. When the parsing is skipped, it is forced True.293    """294    # set lora scale so that monkey patched LoRA295    # function of text encoder can correctly access it296    if lora_scale is not None and isinstance(pipe, StableDiffusionLoraLoaderMixin):297        pipe._lora_scale = lora_scale298 299        # dynamically adjust the LoRA scale300        if not USE_PEFT_BACKEND:301            adjust_lora_scale_text_encoder(pipe.text_encoder, lora_scale)302        else:303            scale_lora_layers(pipe.text_encoder, lora_scale)304    max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2305    if isinstance(prompt, str):306        prompt = [prompt]307 308    if not skip_parsing:309        prompt_tokens, prompt_weights = get_prompts_with_weights(pipe, prompt, max_length - 2)310        if uncond_prompt is not None:311            if isinstance(uncond_prompt, str):312                uncond_prompt = [uncond_prompt]313            uncond_tokens, uncond_weights = get_prompts_with_weights(pipe, uncond_prompt, max_length - 2)314    else:315        prompt_tokens = [316            token[1:-1] for token in pipe.tokenizer(prompt, max_length=max_length, truncation=True).input_ids317        ]318        prompt_weights = [[1.0] * len(token) for token in prompt_tokens]319        if uncond_prompt is not None:320            if isinstance(uncond_prompt, str):321                uncond_prompt = [uncond_prompt]322            uncond_tokens = [323                token[1:-1]324                for token in pipe.tokenizer(uncond_prompt, max_length=max_length, truncation=True).input_ids325            ]326            uncond_weights = [[1.0] * len(token) for token in uncond_tokens]327 328    # round up the longest length of tokens to a multiple of (model_max_length - 2)329    max_length = max([len(token) for token in prompt_tokens])330    if uncond_prompt is not None:331        max_length = max(max_length, max([len(token) for token in uncond_tokens]))332 333    max_embeddings_multiples = min(334        max_embeddings_multiples,335        (max_length - 1) // (pipe.tokenizer.model_max_length - 2) + 1,336    )337    max_embeddings_multiples = max(1, max_embeddings_multiples)338    max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2339 340    # pad the length of tokens and weights341    bos = pipe.tokenizer.bos_token_id342    eos = pipe.tokenizer.eos_token_id343    pad = getattr(pipe.tokenizer, "pad_token_id", eos)344    prompt_tokens, prompt_weights = pad_tokens_and_weights(345        prompt_tokens,346        prompt_weights,347        max_length,348        bos,349        eos,350        pad,351        no_boseos_middle=no_boseos_middle,352        chunk_length=pipe.tokenizer.model_max_length,353    )354    prompt_tokens = torch.tensor(prompt_tokens, dtype=torch.long, device=pipe.device)355    if uncond_prompt is not None:356        uncond_tokens, uncond_weights = pad_tokens_and_weights(357            uncond_tokens,358            uncond_weights,359            max_length,360            bos,361            eos,362            pad,363            no_boseos_middle=no_boseos_middle,364            chunk_length=pipe.tokenizer.model_max_length,365        )366        uncond_tokens = torch.tensor(uncond_tokens, dtype=torch.long, device=pipe.device)367 368    # get the embeddings369    text_embeddings = get_unweighted_text_embeddings(370        pipe, prompt_tokens, pipe.tokenizer.model_max_length, no_boseos_middle=no_boseos_middle, clip_skip=clip_skip371    )372    prompt_weights = torch.tensor(prompt_weights, dtype=text_embeddings.dtype, device=text_embeddings.device)373    if uncond_prompt is not None:374        uncond_embeddings = get_unweighted_text_embeddings(375            pipe,376            uncond_tokens,377            pipe.tokenizer.model_max_length,378            no_boseos_middle=no_boseos_middle,379            clip_skip=clip_skip,380        )381        uncond_weights = torch.tensor(uncond_weights, dtype=uncond_embeddings.dtype, device=uncond_embeddings.device)382 383    # assign weights to the prompts and normalize in the sense of mean384    # TODO: should we normalize by chunk or in a whole (current implementation)?385    if (not skip_parsing) and (not skip_weighting):386        previous_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)387        text_embeddings *= prompt_weights.unsqueeze(-1)388        current_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)389        text_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)390        if uncond_prompt is not None:391            previous_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)392            uncond_embeddings *= uncond_weights.unsqueeze(-1)393            current_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)394            uncond_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)395 396    if pipe.text_encoder is not None:397        if isinstance(pipe, StableDiffusionLoraLoaderMixin) and USE_PEFT_BACKEND:398            # Retrieve the original scale by scaling back the LoRA layers399            unscale_lora_layers(pipe.text_encoder, lora_scale)400 401    if uncond_prompt is not None:402        return text_embeddings, uncond_embeddings403    return text_embeddings, None404 405 406def preprocess_image(image, batch_size):407    w, h = image.size408    w, h = (x - x % 8 for x in (w, h))  # resize to integer multiple of 8409    image = image.resize((w, h), resample=PIL_INTERPOLATION["lanczos"])410    image = np.array(image).astype(np.float32) / 255.0411    image = np.vstack([image[None].transpose(0, 3, 1, 2)] * batch_size)412    image = torch.from_numpy(image)413    return 2.0 * image - 1.0414 415 416def preprocess_mask(mask, batch_size, scale_factor=8):417    if not isinstance(mask, torch.Tensor):418        mask = mask.convert("L")419        w, h = mask.size420        w, h = (x - x % 8 for x in (w, h))  # resize to integer multiple of 8421        mask = mask.resize((w // scale_factor, h // scale_factor), resample=PIL_INTERPOLATION["nearest"])422        mask = np.array(mask).astype(np.float32) / 255.0423        mask = np.tile(mask, (4, 1, 1))424        mask = np.vstack([mask[None]] * batch_size)425        mask = 1 - mask  # repaint white, keep black426        mask = torch.from_numpy(mask)427        return mask428 429    else:430        valid_mask_channel_sizes = [1, 3]431        # if mask channel is fourth tensor dimension, permute dimensions to pytorch standard (B, C, H, W)432        if mask.shape[3] in valid_mask_channel_sizes:433            mask = mask.permute(0, 3, 1, 2)434        elif mask.shape[1] not in valid_mask_channel_sizes:435            raise ValueError(436                f"Mask channel dimension of size in {valid_mask_channel_sizes} should be second or fourth dimension,"437                f" but received mask of shape {tuple(mask.shape)}"438            )439        # (potentially) reduce mask channel dimension from 3 to 1 for broadcasting to latent shape440        mask = mask.mean(dim=1, keepdim=True)441        h, w = mask.shape[-2:]442        h, w = (x - x % 8 for x in (h, w))  # resize to integer multiple of 8443        mask = torch.nn.functional.interpolate(mask, (h // scale_factor, w // scale_factor))444        return mask445 446 447class StableDiffusionLongPromptWeightingPipeline(448    DiffusionPipeline,449    StableDiffusionMixin,450    TextualInversionLoaderMixin,451    StableDiffusionLoraLoaderMixin,452    FromSingleFileMixin,453):454    r"""455    Pipeline for text-to-image generation using Stable Diffusion without tokens length limit, and support parsing456    weighting in prompt.457 458    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the459    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)460 461    Args:462        vae ([`AutoencoderKL`]):463            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.464        text_encoder ([`CLIPTextModel`]):465            Frozen text-encoder. Stable Diffusion uses the text portion of466            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically467            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.468        tokenizer (`CLIPTokenizer`):469            Tokenizer of class470            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).471        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.472        scheduler ([`SchedulerMixin`]):473            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of474            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].475        safety_checker ([`StableDiffusionSafetyChecker`]):476            Classification module that estimates whether generated images could be considered offensive or harmful.477            Please, refer to the [model card](https://huggingface.co/CompVis/stable-diffusion-v1-4) for details.478        feature_extractor ([`CLIPImageProcessor`]):479            Model that extracts features from generated images to be used as inputs for the `safety_checker`.480    """481 482    model_cpu_offload_seq = "text_encoder-->unet->vae"483    _optional_components = ["safety_checker", "feature_extractor"]484    _exclude_from_cpu_offload = ["safety_checker"]485 486    def __init__(487        self,488        vae: AutoencoderKL,489        text_encoder: CLIPTextModel,490        tokenizer: CLIPTokenizer,491        unet: UNet2DConditionModel,492        scheduler: KarrasDiffusionSchedulers,493        safety_checker: StableDiffusionSafetyChecker,494        feature_extractor: CLIPImageProcessor,495        requires_safety_checker: bool = True,496    ):497        super().__init__()498 499        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:500            deprecation_message = (501                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"502                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "503                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"504                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"505                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"506                " file"507            )508            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)509            new_config = dict(scheduler.config)510            new_config["steps_offset"] = 1511            scheduler._internal_dict = FrozenDict(new_config)512 513        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:514            deprecation_message = (515                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."516                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"517                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"518                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"519                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"520            )521            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)522            new_config = dict(scheduler.config)523            new_config["clip_sample"] = False524            scheduler._internal_dict = FrozenDict(new_config)525 526        if safety_checker is None and requires_safety_checker:527            logger.warning(528                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"529                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"530                " results in services or applications open to the public. Both the diffusers team and Hugging Face"531                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"532                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"533                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."534            )535 536        if safety_checker is not None and feature_extractor is None:537            raise ValueError(538                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"539                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."540            )541 542        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(543            version.parse(unet.config._diffusers_version).base_version544        ) < version.parse("0.9.0.dev0")545        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64546        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:547            deprecation_message = (548                "The configuration file of the unet has set the default `sample_size` to smaller than"549                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"550                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"551                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"552                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"553                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"554                " in the config might lead to incorrect results in future versions. If you have downloaded this"555                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"556                " the `unet/config.json` file"557            )558            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)559            new_config = dict(unet.config)560            new_config["sample_size"] = 64561            unet._internal_dict = FrozenDict(new_config)562        self.register_modules(563            vae=vae,564            text_encoder=text_encoder,565            tokenizer=tokenizer,566            unet=unet,567            scheduler=scheduler,568            safety_checker=safety_checker,569            feature_extractor=feature_extractor,570        )571        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)572 573        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)574        self.register_to_config(575            requires_safety_checker=requires_safety_checker,576        )577 578    def _encode_prompt(579        self,580        prompt,581        device,582        num_images_per_prompt,583        do_classifier_free_guidance,584        negative_prompt=None,585        max_embeddings_multiples=3,586        prompt_embeds: Optional[torch.Tensor] = None,587        negative_prompt_embeds: Optional[torch.Tensor] = None,588        clip_skip: Optional[int] = None,589        lora_scale: Optional[float] = None,590    ):591        r"""592        Encodes the prompt into text encoder hidden states.593 594        Args:595            prompt (`str` or `list(int)`):596                prompt to be encoded597            device: (`torch.device`):598                torch device599            num_images_per_prompt (`int`):600                number of images that should be generated per prompt601            do_classifier_free_guidance (`bool`):602                whether to use classifier free guidance or not603            negative_prompt (`str` or `List[str]`):604                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored605                if `guidance_scale` is less than `1`).606            max_embeddings_multiples (`int`, *optional*, defaults to `3`):607                The max multiple length of prompt embeddings compared to the max output length of text encoder.608        """609        if prompt is not None and isinstance(prompt, str):610            batch_size = 1611        elif prompt is not None and isinstance(prompt, list):612            batch_size = len(prompt)613        else:614            batch_size = prompt_embeds.shape[0]615 616        if negative_prompt_embeds is None:617            if negative_prompt is None:618                negative_prompt = [""] * batch_size619            elif isinstance(negative_prompt, str):620                negative_prompt = [negative_prompt] * batch_size621            if batch_size != len(negative_prompt):622                raise ValueError(623                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"624                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"625                    " the batch size of `prompt`."626                )627        if prompt_embeds is None or negative_prompt_embeds is None:628            if isinstance(self, TextualInversionLoaderMixin):629                prompt = self.maybe_convert_prompt(prompt, self.tokenizer)630                if do_classifier_free_guidance and negative_prompt_embeds is None:631                    negative_prompt = self.maybe_convert_prompt(negative_prompt, self.tokenizer)632 633            prompt_embeds1, negative_prompt_embeds1 = get_weighted_text_embeddings(634                pipe=self,635                prompt=prompt,636                uncond_prompt=negative_prompt if do_classifier_free_guidance else None,637                max_embeddings_multiples=max_embeddings_multiples,638                clip_skip=clip_skip,639                lora_scale=lora_scale,640            )641            if prompt_embeds is None:642                prompt_embeds = prompt_embeds1643            if negative_prompt_embeds is None:644                negative_prompt_embeds = negative_prompt_embeds1645 646        bs_embed, seq_len, _ = prompt_embeds.shape647        # duplicate text embeddings for each generation per prompt, using mps friendly method648        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)649        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)650 651        if do_classifier_free_guidance:652            bs_embed, seq_len, _ = negative_prompt_embeds.shape653            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)654            negative_prompt_embeds = negative_prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)655            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])656 657        return prompt_embeds658 659    def check_inputs(660        self,661        prompt,662        height,663        width,664        strength,665        callback_steps,666        negative_prompt=None,667        prompt_embeds=None,668        negative_prompt_embeds=None,669    ):670        if height % 8 != 0 or width % 8 != 0:671            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")672 673        if strength < 0 or strength > 1:674            raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")675 676        if (callback_steps is None) or (677            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)678        ):679            raise ValueError(680                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"681                f" {type(callback_steps)}."682            )683 684        if prompt is not None and prompt_embeds is not None:685            raise ValueError(686                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"687                " only forward one of the two."688            )689        elif prompt is None and prompt_embeds is None:690            raise ValueError(691                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."692            )693        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):694            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")695 696        if negative_prompt is not None and negative_prompt_embeds is not None:697            raise ValueError(698                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"699                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."700            )701 702        if prompt_embeds is not None and negative_prompt_embeds is not None:703            if prompt_embeds.shape != negative_prompt_embeds.shape:704                raise ValueError(705                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"706                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"707                    f" {negative_prompt_embeds.shape}."708                )709 710    def get_timesteps(self, num_inference_steps, strength, device, is_text2img):711        if is_text2img:712            return self.scheduler.timesteps.to(device), num_inference_steps713        else:714            # get the original timestep using init_timestep715            init_timestep = min(int(num_inference_steps * strength), num_inference_steps)716 717            t_start = max(num_inference_steps - init_timestep, 0)718            timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]719 720            return timesteps, num_inference_steps - t_start721 722    def run_safety_checker(self, image, device, dtype):723        if self.safety_checker is not None:724            safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)725            image, has_nsfw_concept = self.safety_checker(726                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)727            )728        else:729            has_nsfw_concept = None730        return image, has_nsfw_concept731 732    def decode_latents(self, latents):733        latents = 1 / self.vae.config.scaling_factor * latents734        image = self.vae.decode(latents).sample735        image = (image / 2 + 0.5).clamp(0, 1)736        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16737        image = image.cpu().permute(0, 2, 3, 1).float().numpy()738        return image739 740    def prepare_extra_step_kwargs(self, generator, eta):741        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature742        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.743        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502744        # and should be between [0, 1]745 746        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())747        extra_step_kwargs = {}748        if accepts_eta:749            extra_step_kwargs["eta"] = eta750 751        # check if the scheduler accepts generator752        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())753        if accepts_generator:754            extra_step_kwargs["generator"] = generator755        return extra_step_kwargs756 757    def prepare_latents(758        self,759        image,760        timestep,761        num_images_per_prompt,762        batch_size,763        num_channels_latents,764        height,765        width,766        dtype,767        device,768        generator,769        latents=None,770    ):771        if image is None:772            batch_size = batch_size * num_images_per_prompt773            shape = (774                batch_size,775                num_channels_latents,776                int(height) // self.vae_scale_factor,777                int(width) // self.vae_scale_factor,778            )779            if isinstance(generator, list) and len(generator) != batch_size:780                raise ValueError(781                    f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"782                    f" size of {batch_size}. Make sure the batch size matches the length of the generators."783                )784 785            if latents is None:786                latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)787            else:788                latents = latents.to(device)789 790            # scale the initial noise by the standard deviation required by the scheduler791            latents = latents * self.scheduler.init_noise_sigma792            return latents, None, None793        else:794            image = image.to(device=self.device, dtype=dtype)795            init_latent_dist = self.vae.encode(image).latent_dist796            init_latents = init_latent_dist.sample(generator=generator)797            init_latents = self.vae.config.scaling_factor * init_latents798 799            # Expand init_latents for batch_size and num_images_per_prompt800            init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0)801            init_latents_orig = init_latents802 803            # add noise to latents using the timesteps804            noise = randn_tensor(init_latents.shape, generator=generator, device=self.device, dtype=dtype)805            init_latents = self.scheduler.add_noise(init_latents, noise, timestep)806            latents = init_latents807            return latents, init_latents_orig, noise808 809    @torch.no_grad()810    def __call__(811        self,812        prompt: Union[str, List[str]],813        negative_prompt: Optional[Union[str, List[str]]] = None,814        image: Union[torch.Tensor, PIL.Image.Image] = None,815        mask_image: Union[torch.Tensor, PIL.Image.Image] = None,816        height: int = 512,817        width: int = 512,818        num_inference_steps: int = 50,819        guidance_scale: float = 7.5,820        strength: float = 0.8,821        num_images_per_prompt: Optional[int] = 1,822        add_predicted_noise: Optional[bool] = False,823        eta: float = 0.0,824        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,825        latents: Optional[torch.Tensor] = None,826        prompt_embeds: Optional[torch.Tensor] = None,827        negative_prompt_embeds: Optional[torch.Tensor] = None,828        max_embeddings_multiples: Optional[int] = 3,829        output_type: Optional[str] = "pil",830        return_dict: bool = True,831        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,832        is_cancelled_callback: Optional[Callable[[], bool]] = None,833        clip_skip: Optional[int] = None,834        callback_steps: int = 1,835        cross_attention_kwargs: Optional[Dict[str, Any]] = None,836    ):837        r"""838        Function invoked when calling the pipeline for generation.839 840        Args:841            prompt (`str` or `List[str]`):842                The prompt or prompts to guide the image generation.843            negative_prompt (`str` or `List[str]`, *optional*):844                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored845                if `guidance_scale` is less than `1`).846            image (`torch.Tensor` or `PIL.Image.Image`):847                `Image`, or tensor representing an image batch, that will be used as the starting point for the848                process.849            mask_image (`torch.Tensor` or `PIL.Image.Image`):850                `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be851                replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a852                PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should853                contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.854            height (`int`, *optional*, defaults to 512):855                The height in pixels of the generated image.856            width (`int`, *optional*, defaults to 512):857                The width in pixels of the generated image.858            num_inference_steps (`int`, *optional*, defaults to 50):859                The number of denoising steps. More denoising steps usually lead to a higher quality image at the860                expense of slower inference.861            guidance_scale (`float`, *optional*, defaults to 7.5):862                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).863                `guidance_scale` is defined as `w` of equation 2. of [Imagen864                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >865                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,866                usually at the expense of lower image quality.867            strength (`float`, *optional*, defaults to 0.8):868                Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1.869                `image` will be used as a starting point, adding more noise to it the larger the `strength`. The870                number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added871                noise will be maximum and the denoising process will run for the full number of iterations specified in872                `num_inference_steps`. A value of 1, therefore, essentially ignores `image`.873            num_images_per_prompt (`int`, *optional*, defaults to 1):874                The number of images to generate per prompt.875            add_predicted_noise (`bool`, *optional*, defaults to True):876                Use predicted noise instead of random noise when constructing noisy versions of the original image in877                the reverse diffusion process878            eta (`float`, *optional*, defaults to 0.0):879                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to880                [`schedulers.DDIMScheduler`], will be ignored for others.881            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):882                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)883                to make generation deterministic.884            latents (`torch.Tensor`, *optional*):885                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image886                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents887                tensor will ge generated by sampling using the supplied random `generator`.888            prompt_embeds (`torch.Tensor`, *optional*):889                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not890                provided, text embeddings will be generated from `prompt` input argument.891            negative_prompt_embeds (`torch.Tensor`, *optional*):892                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt893                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input894                argument.895            max_embeddings_multiples (`int`, *optional*, defaults to `3`):896                The max multiple length of prompt embeddings compared to the max output length of text encoder.897            output_type (`str`, *optional*, defaults to `"pil"`):898                The output format of the generate image. Choose between899                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.900            return_dict (`bool`, *optional*, defaults to `True`):901                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a902                plain tuple.903            callback (`Callable`, *optional*):904                A function that will be called every `callback_steps` steps during inference. The function will be905                called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.906            is_cancelled_callback (`Callable`, *optional*):907                A function that will be called every `callback_steps` steps during inference. If the function returns908                `True`, the inference will be cancelled.909            clip_skip (`int`, *optional*):910                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that911                the output of the pre-final layer will be used for computing the prompt embeddings.912            callback_steps (`int`, *optional*, defaults to 1):913                The frequency at which the `callback` function will be called. If not specified, the callback will be914                called at every step.915            cross_attention_kwargs (`dict`, *optional*):916                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under917                `self.processor` in918                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).919 920        Returns:921            `None` if cancelled by `is_cancelled_callback`,922            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:923            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.924            When returning a tuple, the first element is a list with the generated images, and the second element is a925            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"926            (nsfw) content, according to the `safety_checker`.927        """928        # 0. Default height and width to unet929        height = height or self.unet.config.sample_size * self.vae_scale_factor930        width = width or self.unet.config.sample_size * self.vae_scale_factor931 932        # 1. Check inputs. Raise error if not correct933        self.check_inputs(934            prompt, height, width, strength, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds935        )936 937        # 2. Define call parameters938        if prompt is not None and isinstance(prompt, str):939            batch_size = 1940        elif prompt is not None and isinstance(prompt, list):941            batch_size = len(prompt)942        else:943            batch_size = prompt_embeds.shape[0]944 945        device = self._execution_device946        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)947        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`948        # corresponds to doing no classifier free guidance.949        do_classifier_free_guidance = guidance_scale > 1.0950        lora_scale = cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None951 952        # 3. Encode input prompt953        prompt_embeds = self._encode_prompt(954            prompt,955            device,956            num_images_per_prompt,957            do_classifier_free_guidance,958            negative_prompt,959            max_embeddings_multiples,960            prompt_embeds=prompt_embeds,961            negative_prompt_embeds=negative_prompt_embeds,962            clip_skip=clip_skip,963            lora_scale=lora_scale,964        )965        dtype = prompt_embeds.dtype966 967        # 4. Preprocess image and mask968        if isinstance(image, PIL.Image.Image):969            image = preprocess_image(image, batch_size)970        if image is not None:971            image = image.to(device=self.device, dtype=dtype)972        if isinstance(mask_image, PIL.Image.Image):973            mask_image = preprocess_mask(mask_image, batch_size, self.vae_scale_factor)974        if mask_image is not None:975            mask = mask_image.to(device=self.device, dtype=dtype)976            mask = torch.cat([mask] * num_images_per_prompt)977        else:978            mask = None979 980        # 5. set timesteps981        self.scheduler.set_timesteps(num_inference_steps, device=device)982        timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device, image is None)983        latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)984 985        # 6. Prepare latent variables986        latents, init_latents_orig, noise = self.prepare_latents(987            image,988            latent_timestep,989            num_images_per_prompt,990            batch_size,991            self.unet.config.in_channels,992            height,993            width,994            dtype,995            device,996            generator,997            latents,998        )999 1000        # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline1001        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)1002 1003        # 8. Denoising loop1004        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order1005        with self.progress_bar(total=num_inference_steps) as progress_bar:1006            for i, t in enumerate(timesteps):1007                # expand the latents if we are doing classifier free guidance1008                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents1009                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)1010 1011                # predict the noise residual1012                noise_pred = self.unet(1013                    latent_model_input,1014                    t,1015                    encoder_hidden_states=prompt_embeds,1016                    cross_attention_kwargs=cross_attention_kwargs,1017                ).sample1018 1019                # perform guidance1020                if do_classifier_free_guidance:1021                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)1022                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)1023 1024                # compute the previous noisy sample x_t -> x_t-11025                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample1026 1027                if mask is not None:1028                    # masking1029                    if add_predicted_noise:1030                        init_latents_proper = self.scheduler.add_noise(1031                            init_latents_orig, noise_pred_uncond, torch.tensor([t])1032                        )1033                    else:1034                        init_latents_proper = self.scheduler.add_noise(init_latents_orig, noise, torch.tensor([t]))1035                    latents = (init_latents_proper * mask) + (latents * (1 - mask))1036 1037                # call the callback, if provided1038                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):1039                    progress_bar.update()1040                    if i % callback_steps == 0:1041                        if callback is not None:1042                            step_idx = i // getattr(self.scheduler, "order", 1)1043                            callback(step_idx, t, latents)1044                        if is_cancelled_callback is not None and is_cancelled_callback():1045                            return None1046 1047        if output_type == "latent":1048            image = latents1049            has_nsfw_concept = None1050        elif output_type == "pil":1051            # 9. Post-processing1052            image = self.decode_latents(latents)1053 1054            # 10. Run safety checker1055            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)1056 1057            # 11. Convert to PIL1058            image = self.numpy_to_pil(image)1059        else:1060            # 9. Post-processing1061            image = self.decode_latents(latents)1062 1063            # 10. Run safety checker1064            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)1065 1066        # Offload last model to CPU1067        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:1068            self.final_offload_hook.offload()1069 1070        if not return_dict:1071            return image, has_nsfw_concept1072 1073        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)1074 1075    def text2img(1076        self,1077        prompt: Union[str, List[str]],1078        negative_prompt: Optional[Union[str, List[str]]] = None,1079        height: int = 512,1080        width: int = 512,1081        num_inference_steps: int = 50,1082        guidance_scale: float = 7.5,1083        num_images_per_prompt: Optional[int] = 1,1084        eta: float = 0.0,1085        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,1086        latents: Optional[torch.Tensor] = None,1087        prompt_embeds: Optional[torch.Tensor] = None,1088        negative_prompt_embeds: Optional[torch.Tensor] = None,1089        max_embeddings_multiples: Optional[int] = 3,1090        output_type: Optional[str] = "pil",1091        return_dict: bool = True,1092        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,1093        is_cancelled_callback: Optional[Callable[[], bool]] = None,1094        clip_skip=None,1095        callback_steps: int = 1,1096        cross_attention_kwargs: Optional[Dict[str, Any]] = None,1097    ):1098        r"""1099        Function for text-to-image generation.1100        Args:1101            prompt (`str` or `List[str]`):1102                The prompt or prompts to guide the image generation.1103            negative_prompt (`str` or `List[str]`, *optional*):1104                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored1105                if `guidance_scale` is less than `1`).1106            height (`int`, *optional*, defaults to 512):1107                The height in pixels of the generated image.1108            width (`int`, *optional*, defaults to 512):1109                The width in pixels of the generated image.1110            num_inference_steps (`int`, *optional*, defaults to 50):1111                The number of denoising steps. More denoising steps usually lead to a higher quality image at the1112                expense of slower inference.1113            guidance_scale (`float`, *optional*, defaults to 7.5):1114                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).1115                `guidance_scale` is defined as `w` of equation 2. of [Imagen1116                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >1117                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,1118                usually at the expense of lower image quality.1119            num_images_per_prompt (`int`, *optional*, defaults to 1):1120                The number of images to generate per prompt.1121            eta (`float`, *optional*, defaults to 0.0):1122                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to1123                [`schedulers.DDIMScheduler`], will be ignored for others.1124            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):1125                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)1126                to make generation deterministic.1127            latents (`torch.Tensor`, *optional*):1128                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image1129                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents1130                tensor will ge generated by sampling using the supplied random `generator`.1131            prompt_embeds (`torch.Tensor`, *optional*):1132                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not1133                provided, text embeddings will be generated from `prompt` input argument.1134            negative_prompt_embeds (`torch.Tensor`, *optional*):1135                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt1136                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input1137                argument.1138            max_embeddings_multiples (`int`, *optional*, defaults to `3`):1139                The max multiple length of prompt embeddings compared to the max output length of text encoder.1140            output_type (`str`, *optional*, defaults to `"pil"`):1141                The output format of the generate image. Choose between1142                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.1143            return_dict (`bool`, *optional*, defaults to `True`):1144                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a1145                plain tuple.1146            callback (`Callable`, *optional*):1147                A function that will be called every `callback_steps` steps during inference. The function will be1148                called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.1149            is_cancelled_callback (`Callable`, *optional*):1150                A function that will be called every `callback_steps` steps during inference. If the function returns1151                `True`, the inference will be cancelled.1152            clip_skip (`int`, *optional*):1153                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that1154                the output of the pre-final layer will be used for computing the prompt embeddings.1155            callback_steps (`int`, *optional*, defaults to 1):1156                The frequency at which the `callback` function will be called. If not specified, the callback will be1157                called at every step.1158            cross_attention_kwargs (`dict`, *optional*):1159                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under1160                `self.processor` in1161                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).1162 1163        Returns:1164            `None` if cancelled by `is_cancelled_callback`,1165            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:1166            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.1167            When returning a tuple, the first element is a list with the generated images, and the second element is a1168            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"1169            (nsfw) content, according to the `safety_checker`.1170        """1171        return self.__call__(1172            prompt=prompt,1173            negative_prompt=negative_prompt,1174            height=height,1175            width=width,1176            num_inference_steps=num_inference_steps,1177            guidance_scale=guidance_scale,1178            num_images_per_prompt=num_images_per_prompt,1179            eta=eta,1180            generator=generator,1181            latents=latents,1182            prompt_embeds=prompt_embeds,1183            negative_prompt_embeds=negative_prompt_embeds,1184            max_embeddings_multiples=max_embeddings_multiples,1185            output_type=output_type,1186            return_dict=return_dict,1187            callback=callback,1188            is_cancelled_callback=is_cancelled_callback,1189            clip_skip=clip_skip,1190            callback_steps=callback_steps,1191            cross_attention_kwargs=cross_attention_kwargs,1192        )1193 1194    def img2img(1195        self,1196        image: Union[torch.Tensor, PIL.Image.Image],1197        prompt: Union[str, List[str]],1198        negative_prompt: Optional[Union[str, List[str]]] = None,1199        strength: float = 0.8,1200        num_inference_steps: Optional[int] = 50,

Showing the first 1,200 of 1428 lines. Download the file for the rest.