diffusers/community-pipelines-mirror

1import inspect2import re3from typing import Any, Callable, Dict, List, Optional, Union4 5import numpy as np6import PIL.Image7import torch8from packaging import version9from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer10 11from diffusers import DiffusionPipeline12from diffusers.configuration_utils import FrozenDict13from diffusers.image_processor import VaeImageProcessor14from diffusers.loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin15from diffusers.models import AutoencoderKL, UNet2DConditionModel16from diffusers.pipelines.pipeline_utils import StableDiffusionMixin17from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput, StableDiffusionSafetyChecker18from diffusers.schedulers import KarrasDiffusionSchedulers19from diffusers.utils import (20    PIL_INTERPOLATION,21    deprecate,22    logging,23)24from diffusers.utils.torch_utils import randn_tensor25 26 27# ------------------------------------------------------------------------------28 29logger = logging.get_logger(__name__)  # pylint: disable=invalid-name30 31re_attention = re.compile(32    r"""33\\\(|34\\\)|35\\\[|36\\]|37\\\\|38\\|39\(|40\[|41:([+-]?[.\d]+)\)|42\)|43]|44[^\\()\[\]:]+|45:46""",47    re.X,48)49 50 51def parse_prompt_attention(text):52    """53    Parses a string with attention tokens and returns a list of pairs: text and its associated weight.54    Accepted tokens are:55      (abc) - increases attention to abc by a multiplier of 1.156      (abc:3.12) - increases attention to abc by a multiplier of 3.1257      [abc] - decreases attention to abc by a multiplier of 1.158      \\( - literal character '('59      \\[ - literal character '['60      \\) - literal character ')'61      \\] - literal character ']'62      \\ - literal character '\'63      anything else - just text64    >>> parse_prompt_attention('normal text')65    [['normal text', 1.0]]66    >>> parse_prompt_attention('an (important) word')67    [['an ', 1.0], ['important', 1.1], [' word', 1.0]]68    >>> parse_prompt_attention('(unbalanced')69    [['unbalanced', 1.1]]70    >>> parse_prompt_attention('\\(literal\\]')71    [['(literal]', 1.0]]72    >>> parse_prompt_attention('(unnecessary)(parens)')73    [['unnecessaryparens', 1.1]]74    >>> parse_prompt_attention('a (((house:1.3)) [on] a (hill:0.5), sun, (((sky))).')75    [['a ', 1.0],76     ['house', 1.5730000000000004],77     [' ', 1.1],78     ['on', 1.0],79     [' a ', 1.1],80     ['hill', 0.55],81     [', sun, ', 1.1],82     ['sky', 1.4641000000000006],83     ['.', 1.1]]84    """85 86    res = []87    round_brackets = []88    square_brackets = []89 90    round_bracket_multiplier = 1.191    square_bracket_multiplier = 1 / 1.192 93    def multiply_range(start_position, multiplier):94        for p in range(start_position, len(res)):95            res[p][1] *= multiplier96 97    for m in re_attention.finditer(text):98        text = m.group(0)99        weight = m.group(1)100 101        if text.startswith("\\"):102            res.append([text[1:], 1.0])103        elif text == "(":104            round_brackets.append(len(res))105        elif text == "[":106            square_brackets.append(len(res))107        elif weight is not None and len(round_brackets) > 0:108            multiply_range(round_brackets.pop(), float(weight))109        elif text == ")" and len(round_brackets) > 0:110            multiply_range(round_brackets.pop(), round_bracket_multiplier)111        elif text == "]" and len(square_brackets) > 0:112            multiply_range(square_brackets.pop(), square_bracket_multiplier)113        else:114            res.append([text, 1.0])115 116    for pos in round_brackets:117        multiply_range(pos, round_bracket_multiplier)118 119    for pos in square_brackets:120        multiply_range(pos, square_bracket_multiplier)121 122    if len(res) == 0:123        res = [["", 1.0]]124 125    # merge runs of identical weights126    i = 0127    while i + 1 < len(res):128        if res[i][1] == res[i + 1][1]:129            res[i][0] += res[i + 1][0]130            res.pop(i + 1)131        else:132            i += 1133 134    return res135 136 137def get_prompts_with_weights(pipe: DiffusionPipeline, prompt: List[str], max_length: int):138    r"""139    Tokenize a list of prompts and return its tokens with weights of each token.140 141    No padding, starting or ending token is included.142    """143    tokens = []144    weights = []145    truncated = False146    for text in prompt:147        texts_and_weights = parse_prompt_attention(text)148        text_token = []149        text_weight = []150        for word, weight in texts_and_weights:151            # tokenize and discard the starting and the ending token152            token = pipe.tokenizer(word).input_ids[1:-1]153            text_token += token154            # copy the weight by length of token155            text_weight += [weight] * len(token)156            # stop if the text is too long (longer than truncation limit)157            if len(text_token) > max_length:158                truncated = True159                break160        # truncate161        if len(text_token) > max_length:162            truncated = True163            text_token = text_token[:max_length]164            text_weight = text_weight[:max_length]165        tokens.append(text_token)166        weights.append(text_weight)167    if truncated:168        logger.warning("Prompt was truncated. Try to shorten the prompt or increase max_embeddings_multiples")169    return tokens, weights170 171 172def pad_tokens_and_weights(tokens, weights, max_length, bos, eos, pad, no_boseos_middle=True, chunk_length=77):173    r"""174    Pad the tokens (with starting and ending tokens) and weights (with 1.0) to max_length.175    """176    max_embeddings_multiples = (max_length - 2) // (chunk_length - 2)177    weights_length = max_length if no_boseos_middle else max_embeddings_multiples * chunk_length178    for i in range(len(tokens)):179        tokens[i] = [bos] + tokens[i] + [pad] * (max_length - 1 - len(tokens[i]) - 1) + [eos]180        if no_boseos_middle:181            weights[i] = [1.0] + weights[i] + [1.0] * (max_length - 1 - len(weights[i]))182        else:183            w = []184            if len(weights[i]) == 0:185                w = [1.0] * weights_length186            else:187                for j in range(max_embeddings_multiples):188                    w.append(1.0)  # weight for starting token in this chunk189                    w += weights[i][j * (chunk_length - 2) : min(len(weights[i]), (j + 1) * (chunk_length - 2))]190                    w.append(1.0)  # weight for ending token in this chunk191                w += [1.0] * (weights_length - len(w))192            weights[i] = w[:]193 194    return tokens, weights195 196 197def get_unweighted_text_embeddings(198    pipe: DiffusionPipeline,199    text_input: torch.Tensor,200    chunk_length: int,201    no_boseos_middle: Optional[bool] = True,202):203    """204    When the length of tokens is a multiple of the capacity of the text encoder,205    it should be split into chunks and sent to the text encoder individually.206    """207    max_embeddings_multiples = (text_input.shape[1] - 2) // (chunk_length - 2)208    if max_embeddings_multiples > 1:209        text_embeddings = []210        for i in range(max_embeddings_multiples):211            # extract the i-th chunk212            text_input_chunk = text_input[:, i * (chunk_length - 2) : (i + 1) * (chunk_length - 2) + 2].clone()213 214            # cover the head and the tail by the starting and the ending tokens215            text_input_chunk[:, 0] = text_input[0, 0]216            text_input_chunk[:, -1] = text_input[0, -1]217            text_embedding = pipe.text_encoder(text_input_chunk)[0]218 219            if no_boseos_middle:220                if i == 0:221                    # discard the ending token222                    text_embedding = text_embedding[:, :-1]223                elif i == max_embeddings_multiples - 1:224                    # discard the starting token225                    text_embedding = text_embedding[:, 1:]226                else:227                    # discard both starting and ending tokens228                    text_embedding = text_embedding[:, 1:-1]229 230            text_embeddings.append(text_embedding)231        text_embeddings = torch.concat(text_embeddings, axis=1)232    else:233        text_embeddings = pipe.text_encoder(text_input)[0]234    return text_embeddings235 236 237def get_weighted_text_embeddings(238    pipe: DiffusionPipeline,239    prompt: Union[str, List[str]],240    uncond_prompt: Optional[Union[str, List[str]]] = None,241    max_embeddings_multiples: Optional[int] = 3,242    no_boseos_middle: Optional[bool] = False,243    skip_parsing: Optional[bool] = False,244    skip_weighting: Optional[bool] = False,245):246    r"""247    Prompts can be assigned with local weights using brackets. For example,248    prompt 'A (very beautiful) masterpiece' highlights the words 'very beautiful',249    and the embedding tokens corresponding to the words get multiplied by a constant, 1.1.250 251    Also, to regularize of the embedding, the weighted embedding would be scaled to preserve the original mean.252 253    Args:254        pipe (`DiffusionPipeline`):255            Pipe to provide access to the tokenizer and the text encoder.256        prompt (`str` or `List[str]`):257            The prompt or prompts to guide the image generation.258        uncond_prompt (`str` or `List[str]`):259            The unconditional prompt or prompts for guide the image generation. If unconditional prompt260            is provided, the embeddings of prompt and uncond_prompt are concatenated.261        max_embeddings_multiples (`int`, *optional*, defaults to `3`):262            The max multiple length of prompt embeddings compared to the max output length of text encoder.263        no_boseos_middle (`bool`, *optional*, defaults to `False`):264            If the length of text token is multiples of the capacity of text encoder, whether reserve the starting and265            ending token in each of the chunk in the middle.266        skip_parsing (`bool`, *optional*, defaults to `False`):267            Skip the parsing of brackets.268        skip_weighting (`bool`, *optional*, defaults to `False`):269            Skip the weighting. When the parsing is skipped, it is forced True.270    """271    max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2272    if isinstance(prompt, str):273        prompt = [prompt]274 275    if not skip_parsing:276        prompt_tokens, prompt_weights = get_prompts_with_weights(pipe, prompt, max_length - 2)277        if uncond_prompt is not None:278            if isinstance(uncond_prompt, str):279                uncond_prompt = [uncond_prompt]280            uncond_tokens, uncond_weights = get_prompts_with_weights(pipe, uncond_prompt, max_length - 2)281    else:282        prompt_tokens = [283            token[1:-1] for token in pipe.tokenizer(prompt, max_length=max_length, truncation=True).input_ids284        ]285        prompt_weights = [[1.0] * len(token) for token in prompt_tokens]286        if uncond_prompt is not None:287            if isinstance(uncond_prompt, str):288                uncond_prompt = [uncond_prompt]289            uncond_tokens = [290                token[1:-1]291                for token in pipe.tokenizer(uncond_prompt, max_length=max_length, truncation=True).input_ids292            ]293            uncond_weights = [[1.0] * len(token) for token in uncond_tokens]294 295    # round up the longest length of tokens to a multiple of (model_max_length - 2)296    max_length = max([len(token) for token in prompt_tokens])297    if uncond_prompt is not None:298        max_length = max(max_length, max([len(token) for token in uncond_tokens]))299 300    max_embeddings_multiples = min(301        max_embeddings_multiples,302        (max_length - 1) // (pipe.tokenizer.model_max_length - 2) + 1,303    )304    max_embeddings_multiples = max(1, max_embeddings_multiples)305    max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2306 307    # pad the length of tokens and weights308    bos = pipe.tokenizer.bos_token_id309    eos = pipe.tokenizer.eos_token_id310    pad = getattr(pipe.tokenizer, "pad_token_id", eos)311    prompt_tokens, prompt_weights = pad_tokens_and_weights(312        prompt_tokens,313        prompt_weights,314        max_length,315        bos,316        eos,317        pad,318        no_boseos_middle=no_boseos_middle,319        chunk_length=pipe.tokenizer.model_max_length,320    )321    prompt_tokens = torch.tensor(prompt_tokens, dtype=torch.long, device=pipe.device)322    if uncond_prompt is not None:323        uncond_tokens, uncond_weights = pad_tokens_and_weights(324            uncond_tokens,325            uncond_weights,326            max_length,327            bos,328            eos,329            pad,330            no_boseos_middle=no_boseos_middle,331            chunk_length=pipe.tokenizer.model_max_length,332        )333        uncond_tokens = torch.tensor(uncond_tokens, dtype=torch.long, device=pipe.device)334 335    # get the embeddings336    text_embeddings = get_unweighted_text_embeddings(337        pipe,338        prompt_tokens,339        pipe.tokenizer.model_max_length,340        no_boseos_middle=no_boseos_middle,341    )342    prompt_weights = torch.tensor(prompt_weights, dtype=text_embeddings.dtype, device=text_embeddings.device)343    if uncond_prompt is not None:344        uncond_embeddings = get_unweighted_text_embeddings(345            pipe,346            uncond_tokens,347            pipe.tokenizer.model_max_length,348            no_boseos_middle=no_boseos_middle,349        )350        uncond_weights = torch.tensor(uncond_weights, dtype=uncond_embeddings.dtype, device=uncond_embeddings.device)351 352    # assign weights to the prompts and normalize in the sense of mean353    # TODO: should we normalize by chunk or in a whole (current implementation)?354    if (not skip_parsing) and (not skip_weighting):355        previous_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)356        text_embeddings *= prompt_weights.unsqueeze(-1)357        current_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)358        text_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)359        if uncond_prompt is not None:360            previous_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)361            uncond_embeddings *= uncond_weights.unsqueeze(-1)362            current_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)363            uncond_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)364 365    if uncond_prompt is not None:366        return text_embeddings, uncond_embeddings367    return text_embeddings, None368 369 370def preprocess_image(image, batch_size):371    w, h = image.size372    w, h = (x - x % 8 for x in (w, h))  # resize to integer multiple of 8373    image = image.resize((w, h), resample=PIL_INTERPOLATION["lanczos"])374    image = np.array(image).astype(np.float32) / 255.0375    image = np.vstack([image[None].transpose(0, 3, 1, 2)] * batch_size)376    image = torch.from_numpy(image)377    return 2.0 * image - 1.0378 379 380def preprocess_mask(mask, batch_size, scale_factor=8):381    if not isinstance(mask, torch.Tensor):382        mask = mask.convert("L")383        w, h = mask.size384        w, h = (x - x % 8 for x in (w, h))  # resize to integer multiple of 8385        mask = mask.resize((w // scale_factor, h // scale_factor), resample=PIL_INTERPOLATION["nearest"])386        mask = np.array(mask).astype(np.float32) / 255.0387        mask = np.tile(mask, (4, 1, 1))388        mask = np.vstack([mask[None]] * batch_size)389        mask = 1 - mask  # repaint white, keep black390        mask = torch.from_numpy(mask)391        return mask392 393    else:394        valid_mask_channel_sizes = [1, 3]395        # if mask channel is fourth tensor dimension, permute dimensions to pytorch standard (B, C, H, W)396        if mask.shape[3] in valid_mask_channel_sizes:397            mask = mask.permute(0, 3, 1, 2)398        elif mask.shape[1] not in valid_mask_channel_sizes:399            raise ValueError(400                f"Mask channel dimension of size in {valid_mask_channel_sizes} should be second or fourth dimension,"401                f" but received mask of shape {tuple(mask.shape)}"402            )403        # (potentially) reduce mask channel dimension from 3 to 1 for broadcasting to latent shape404        mask = mask.mean(dim=1, keepdim=True)405        h, w = mask.shape[-2:]406        h, w = (x - x % 8 for x in (h, w))  # resize to integer multiple of 8407        mask = torch.nn.functional.interpolate(mask, (h // scale_factor, w // scale_factor))408        return mask409 410 411class StableDiffusionLongPromptWeightingPipeline(412    DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin, FromSingleFileMixin413):414    r"""415    Pipeline for text-to-image generation using Stable Diffusion without tokens length limit, and support parsing416    weighting in prompt.417 418    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the419    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)420 421    Args:422        vae ([`AutoencoderKL`]):423            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.424        text_encoder ([`CLIPTextModel`]):425            Frozen text-encoder. Stable Diffusion uses the text portion of426            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically427            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.428        tokenizer (`CLIPTokenizer`):429            Tokenizer of class430            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).431        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.432        scheduler ([`SchedulerMixin`]):433            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of434            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].435        safety_checker ([`StableDiffusionSafetyChecker`]):436            Classification module that estimates whether generated images could be considered offensive or harmful.437            Please, refer to the [model card](https://huggingface.co/CompVis/stable-diffusion-v1-4) for details.438        feature_extractor ([`CLIPImageProcessor`]):439            Model that extracts features from generated images to be used as inputs for the `safety_checker`.440    """441 442    model_cpu_offload_seq = "text_encoder-->unet->vae"443    _optional_components = ["safety_checker", "feature_extractor"]444    _exclude_from_cpu_offload = ["safety_checker"]445 446    def __init__(447        self,448        vae: AutoencoderKL,449        text_encoder: CLIPTextModel,450        tokenizer: CLIPTokenizer,451        unet: UNet2DConditionModel,452        scheduler: KarrasDiffusionSchedulers,453        safety_checker: StableDiffusionSafetyChecker,454        feature_extractor: CLIPImageProcessor,455        requires_safety_checker: bool = True,456    ):457        super().__init__()458 459        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:460            deprecation_message = (461                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"462                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "463                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"464                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"465                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"466                " file"467            )468            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)469            new_config = dict(scheduler.config)470            new_config["steps_offset"] = 1471            scheduler._internal_dict = FrozenDict(new_config)472 473        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:474            deprecation_message = (475                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."476                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"477                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"478                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"479                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"480            )481            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)482            new_config = dict(scheduler.config)483            new_config["clip_sample"] = False484            scheduler._internal_dict = FrozenDict(new_config)485 486        if safety_checker is None and requires_safety_checker:487            logger.warning(488                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"489                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"490                " results in services or applications open to the public. Both the diffusers team and Hugging Face"491                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"492                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"493                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."494            )495 496        if safety_checker is not None and feature_extractor is None:497            raise ValueError(498                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"499                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."500            )501 502        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(503            version.parse(unet.config._diffusers_version).base_version504        ) < version.parse("0.9.0.dev0")505        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64506        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:507            deprecation_message = (508                "The configuration file of the unet has set the default `sample_size` to smaller than"509                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"510                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"511                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"512                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"513                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"514                " in the config might lead to incorrect results in future versions. If you have downloaded this"515                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"516                " the `unet/config.json` file"517            )518            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)519            new_config = dict(unet.config)520            new_config["sample_size"] = 64521            unet._internal_dict = FrozenDict(new_config)522        self.register_modules(523            vae=vae,524            text_encoder=text_encoder,525            tokenizer=tokenizer,526            unet=unet,527            scheduler=scheduler,528            safety_checker=safety_checker,529            feature_extractor=feature_extractor,530        )531        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)532 533        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)534        self.register_to_config(535            requires_safety_checker=requires_safety_checker,536        )537 538    def _encode_prompt(539        self,540        prompt,541        device,542        num_images_per_prompt,543        do_classifier_free_guidance,544        negative_prompt=None,545        max_embeddings_multiples=3,546        prompt_embeds: Optional[torch.Tensor] = None,547        negative_prompt_embeds: Optional[torch.Tensor] = None,548    ):549        r"""550        Encodes the prompt into text encoder hidden states.551 552        Args:553            prompt (`str` or `list(int)`):554                prompt to be encoded555            device: (`torch.device`):556                torch device557            num_images_per_prompt (`int`):558                number of images that should be generated per prompt559            do_classifier_free_guidance (`bool`):560                whether to use classifier free guidance or not561            negative_prompt (`str` or `List[str]`):562                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored563                if `guidance_scale` is less than `1`).564            max_embeddings_multiples (`int`, *optional*, defaults to `3`):565                The max multiple length of prompt embeddings compared to the max output length of text encoder.566        """567        if prompt is not None and isinstance(prompt, str):568            batch_size = 1569        elif prompt is not None and isinstance(prompt, list):570            batch_size = len(prompt)571        else:572            batch_size = prompt_embeds.shape[0]573 574        if negative_prompt_embeds is None:575            if negative_prompt is None:576                negative_prompt = [""] * batch_size577            elif isinstance(negative_prompt, str):578                negative_prompt = [negative_prompt] * batch_size579            if batch_size != len(negative_prompt):580                raise ValueError(581                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"582                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"583                    " the batch size of `prompt`."584                )585        if prompt_embeds is None or negative_prompt_embeds is None:586            if isinstance(self, TextualInversionLoaderMixin):587                prompt = self.maybe_convert_prompt(prompt, self.tokenizer)588                if do_classifier_free_guidance and negative_prompt_embeds is None:589                    negative_prompt = self.maybe_convert_prompt(negative_prompt, self.tokenizer)590 591            prompt_embeds1, negative_prompt_embeds1 = get_weighted_text_embeddings(592                pipe=self,593                prompt=prompt,594                uncond_prompt=negative_prompt if do_classifier_free_guidance else None,595                max_embeddings_multiples=max_embeddings_multiples,596            )597            if prompt_embeds is None:598                prompt_embeds = prompt_embeds1599            if negative_prompt_embeds is None:600                negative_prompt_embeds = negative_prompt_embeds1601 602        bs_embed, seq_len, _ = prompt_embeds.shape603        # duplicate text embeddings for each generation per prompt, using mps friendly method604        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)605        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)606 607        if do_classifier_free_guidance:608            bs_embed, seq_len, _ = negative_prompt_embeds.shape609            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)610            negative_prompt_embeds = negative_prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)611            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])612 613        return prompt_embeds614 615    def check_inputs(616        self,617        prompt,618        height,619        width,620        strength,621        callback_steps,622        negative_prompt=None,623        prompt_embeds=None,624        negative_prompt_embeds=None,625    ):626        if height % 8 != 0 or width % 8 != 0:627            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")628 629        if strength < 0 or strength > 1:630            raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")631 632        if (callback_steps is None) or (633            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)634        ):635            raise ValueError(636                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"637                f" {type(callback_steps)}."638            )639 640        if prompt is not None and prompt_embeds is not None:641            raise ValueError(642                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"643                " only forward one of the two."644            )645        elif prompt is None and prompt_embeds is None:646            raise ValueError(647                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."648            )649        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):650            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")651 652        if negative_prompt is not None and negative_prompt_embeds is not None:653            raise ValueError(654                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"655                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."656            )657 658        if prompt_embeds is not None and negative_prompt_embeds is not None:659            if prompt_embeds.shape != negative_prompt_embeds.shape:660                raise ValueError(661                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"662                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"663                    f" {negative_prompt_embeds.shape}."664                )665 666    def get_timesteps(self, num_inference_steps, strength, device, is_text2img):667        if is_text2img:668            return self.scheduler.timesteps.to(device), num_inference_steps669        else:670            # get the original timestep using init_timestep671            init_timestep = min(int(num_inference_steps * strength), num_inference_steps)672 673            t_start = max(num_inference_steps - init_timestep, 0)674            timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]675 676            return timesteps, num_inference_steps - t_start677 678    def run_safety_checker(self, image, device, dtype):679        if self.safety_checker is not None:680            safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)681            image, has_nsfw_concept = self.safety_checker(682                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)683            )684        else:685            has_nsfw_concept = None686        return image, has_nsfw_concept687 688    def decode_latents(self, latents):689        latents = 1 / self.vae.config.scaling_factor * latents690        image = self.vae.decode(latents).sample691        image = (image / 2 + 0.5).clamp(0, 1)692        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16693        image = image.cpu().permute(0, 2, 3, 1).float().numpy()694        return image695 696    def prepare_extra_step_kwargs(self, generator, eta):697        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature698        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.699        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502700        # and should be between [0, 1]701 702        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())703        extra_step_kwargs = {}704        if accepts_eta:705            extra_step_kwargs["eta"] = eta706 707        # check if the scheduler accepts generator708        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())709        if accepts_generator:710            extra_step_kwargs["generator"] = generator711        return extra_step_kwargs712 713    def prepare_latents(714        self,715        image,716        timestep,717        num_images_per_prompt,718        batch_size,719        num_channels_latents,720        height,721        width,722        dtype,723        device,724        generator,725        latents=None,726    ):727        if image is None:728            batch_size = batch_size * num_images_per_prompt729            shape = (730                batch_size,731                num_channels_latents,732                int(height) // self.vae_scale_factor,733                int(width) // self.vae_scale_factor,734            )735            if isinstance(generator, list) and len(generator) != batch_size:736                raise ValueError(737                    f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"738                    f" size of {batch_size}. Make sure the batch size matches the length of the generators."739                )740 741            if latents is None:742                latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)743            else:744                latents = latents.to(device)745 746            # scale the initial noise by the standard deviation required by the scheduler747            latents = latents * self.scheduler.init_noise_sigma748            return latents, None, None749        else:750            image = image.to(device=self.device, dtype=dtype)751            init_latent_dist = self.vae.encode(image).latent_dist752            init_latents = init_latent_dist.sample(generator=generator)753            init_latents = self.vae.config.scaling_factor * init_latents754 755            # Expand init_latents for batch_size and num_images_per_prompt756            init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0)757            init_latents_orig = init_latents758 759            # add noise to latents using the timesteps760            noise = randn_tensor(init_latents.shape, generator=generator, device=self.device, dtype=dtype)761            init_latents = self.scheduler.add_noise(init_latents, noise, timestep)762            latents = init_latents763            return latents, init_latents_orig, noise764 765    @torch.no_grad()766    def __call__(767        self,768        prompt: Union[str, List[str]],769        negative_prompt: Optional[Union[str, List[str]]] = None,770        image: Union[torch.Tensor, PIL.Image.Image] = None,771        mask_image: Union[torch.Tensor, PIL.Image.Image] = None,772        height: int = 512,773        width: int = 512,774        num_inference_steps: int = 50,775        guidance_scale: float = 7.5,776        strength: float = 0.8,777        num_images_per_prompt: Optional[int] = 1,778        add_predicted_noise: Optional[bool] = False,779        eta: float = 0.0,780        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,781        latents: Optional[torch.Tensor] = None,782        prompt_embeds: Optional[torch.Tensor] = None,783        negative_prompt_embeds: Optional[torch.Tensor] = None,784        max_embeddings_multiples: Optional[int] = 3,785        output_type: Optional[str] = "pil",786        return_dict: bool = True,787        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,788        is_cancelled_callback: Optional[Callable[[], bool]] = None,789        callback_steps: int = 1,790        cross_attention_kwargs: Optional[Dict[str, Any]] = None,791    ):792        r"""793        Function invoked when calling the pipeline for generation.794 795        Args:796            prompt (`str` or `List[str]`):797                The prompt or prompts to guide the image generation.798            negative_prompt (`str` or `List[str]`, *optional*):799                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored800                if `guidance_scale` is less than `1`).801            image (`torch.Tensor` or `PIL.Image.Image`):802                `Image`, or tensor representing an image batch, that will be used as the starting point for the803                process.804            mask_image (`torch.Tensor` or `PIL.Image.Image`):805                `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be806                replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a807                PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should808                contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.809            height (`int`, *optional*, defaults to 512):810                The height in pixels of the generated image.811            width (`int`, *optional*, defaults to 512):812                The width in pixels of the generated image.813            num_inference_steps (`int`, *optional*, defaults to 50):814                The number of denoising steps. More denoising steps usually lead to a higher quality image at the815                expense of slower inference.816            guidance_scale (`float`, *optional*, defaults to 7.5):817                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).818                `guidance_scale` is defined as `w` of equation 2. of [Imagen819                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >820                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,821                usually at the expense of lower image quality.822            strength (`float`, *optional*, defaults to 0.8):823                Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1.824                `image` will be used as a starting point, adding more noise to it the larger the `strength`. The825                number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added826                noise will be maximum and the denoising process will run for the full number of iterations specified in827                `num_inference_steps`. A value of 1, therefore, essentially ignores `image`.828            num_images_per_prompt (`int`, *optional*, defaults to 1):829                The number of images to generate per prompt.830            add_predicted_noise (`bool`, *optional*, defaults to True):831                Use predicted noise instead of random noise when constructing noisy versions of the original image in832                the reverse diffusion process833            eta (`float`, *optional*, defaults to 0.0):834                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to835                [`schedulers.DDIMScheduler`], will be ignored for others.836            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):837                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)838                to make generation deterministic.839            latents (`torch.Tensor`, *optional*):840                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image841                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents842                tensor will ge generated by sampling using the supplied random `generator`.843            prompt_embeds (`torch.Tensor`, *optional*):844                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not845                provided, text embeddings will be generated from `prompt` input argument.846            negative_prompt_embeds (`torch.Tensor`, *optional*):847                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt848                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input849                argument.850            max_embeddings_multiples (`int`, *optional*, defaults to `3`):851                The max multiple length of prompt embeddings compared to the max output length of text encoder.852            output_type (`str`, *optional*, defaults to `"pil"`):853                The output format of the generate image. Choose between854                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.855            return_dict (`bool`, *optional*, defaults to `True`):856                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a857                plain tuple.858            callback (`Callable`, *optional*):859                A function that will be called every `callback_steps` steps during inference. The function will be860                called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.861            is_cancelled_callback (`Callable`, *optional*):862                A function that will be called every `callback_steps` steps during inference. If the function returns863                `True`, the inference will be cancelled.864            callback_steps (`int`, *optional*, defaults to 1):865                The frequency at which the `callback` function will be called. If not specified, the callback will be866                called at every step.867            cross_attention_kwargs (`dict`, *optional*):868                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under869                `self.processor` in870                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).871 872        Returns:873            `None` if cancelled by `is_cancelled_callback`,874            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:875            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.876            When returning a tuple, the first element is a list with the generated images, and the second element is a877            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"878            (nsfw) content, according to the `safety_checker`.879        """880        # 0. Default height and width to unet881        height = height or self.unet.config.sample_size * self.vae_scale_factor882        width = width or self.unet.config.sample_size * self.vae_scale_factor883 884        # 1. Check inputs. Raise error if not correct885        self.check_inputs(886            prompt, height, width, strength, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds887        )888 889        # 2. Define call parameters890        if prompt is not None and isinstance(prompt, str):891            batch_size = 1892        elif prompt is not None and isinstance(prompt, list):893            batch_size = len(prompt)894        else:895            batch_size = prompt_embeds.shape[0]896 897        device = self._execution_device898        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)899        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`900        # corresponds to doing no classifier free guidance.901        do_classifier_free_guidance = guidance_scale > 1.0902 903        # 3. Encode input prompt904        prompt_embeds = self._encode_prompt(905            prompt,906            device,907            num_images_per_prompt,908            do_classifier_free_guidance,909            negative_prompt,910            max_embeddings_multiples,911            prompt_embeds=prompt_embeds,912            negative_prompt_embeds=negative_prompt_embeds,913        )914        dtype = prompt_embeds.dtype915 916        # 4. Preprocess image and mask917        if isinstance(image, PIL.Image.Image):918            image = preprocess_image(image, batch_size)919        if image is not None:920            image = image.to(device=self.device, dtype=dtype)921        if isinstance(mask_image, PIL.Image.Image):922            mask_image = preprocess_mask(mask_image, batch_size, self.vae_scale_factor)923        if mask_image is not None:924            mask = mask_image.to(device=self.device, dtype=dtype)925            mask = torch.cat([mask] * num_images_per_prompt)926        else:927            mask = None928 929        # 5. set timesteps930        self.scheduler.set_timesteps(num_inference_steps, device=device)931        timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device, image is None)932        latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)933 934        # 6. Prepare latent variables935        latents, init_latents_orig, noise = self.prepare_latents(936            image,937            latent_timestep,938            num_images_per_prompt,939            batch_size,940            self.unet.config.in_channels,941            height,942            width,943            dtype,944            device,945            generator,946            latents,947        )948 949        # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline950        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)951 952        # 8. Denoising loop953        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order954        with self.progress_bar(total=num_inference_steps) as progress_bar:955            for i, t in enumerate(timesteps):956                # expand the latents if we are doing classifier free guidance957                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents958                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)959 960                # predict the noise residual961                noise_pred = self.unet(962                    latent_model_input,963                    t,964                    encoder_hidden_states=prompt_embeds,965                    cross_attention_kwargs=cross_attention_kwargs,966                ).sample967 968                # perform guidance969                if do_classifier_free_guidance:970                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)971                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)972 973                # compute the previous noisy sample x_t -> x_t-1974                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample975 976                if mask is not None:977                    # masking978                    if add_predicted_noise:979                        init_latents_proper = self.scheduler.add_noise(980                            init_latents_orig, noise_pred_uncond, torch.tensor([t])981                        )982                    else:983                        init_latents_proper = self.scheduler.add_noise(init_latents_orig, noise, torch.tensor([t]))984                    latents = (init_latents_proper * mask) + (latents * (1 - mask))985 986                # call the callback, if provided987                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):988                    progress_bar.update()989                    if i % callback_steps == 0:990                        if callback is not None:991                            step_idx = i // getattr(self.scheduler, "order", 1)992                            callback(step_idx, t, latents)993                        if is_cancelled_callback is not None and is_cancelled_callback():994                            return None995 996        if output_type == "latent":997            image = latents998            has_nsfw_concept = None999        elif output_type == "pil":1000            # 9. Post-processing1001            image = self.decode_latents(latents)1002 1003            # 10. Run safety checker1004            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)1005 1006            # 11. Convert to PIL1007            image = self.numpy_to_pil(image)1008        else:1009            # 9. Post-processing1010            image = self.decode_latents(latents)1011 1012            # 10. Run safety checker1013            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)1014 1015        # Offload last model to CPU1016        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:1017            self.final_offload_hook.offload()1018 1019        if not return_dict:1020            return image, has_nsfw_concept1021 1022        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)1023 1024    def text2img(1025        self,1026        prompt: Union[str, List[str]],1027        negative_prompt: Optional[Union[str, List[str]]] = None,1028        height: int = 512,1029        width: int = 512,1030        num_inference_steps: int = 50,1031        guidance_scale: float = 7.5,1032        num_images_per_prompt: Optional[int] = 1,1033        eta: float = 0.0,1034        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,1035        latents: Optional[torch.Tensor] = None,1036        prompt_embeds: Optional[torch.Tensor] = None,1037        negative_prompt_embeds: Optional[torch.Tensor] = None,1038        max_embeddings_multiples: Optional[int] = 3,1039        output_type: Optional[str] = "pil",1040        return_dict: bool = True,1041        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,1042        is_cancelled_callback: Optional[Callable[[], bool]] = None,1043        callback_steps: int = 1,1044        cross_attention_kwargs: Optional[Dict[str, Any]] = None,1045    ):1046        r"""1047        Function for text-to-image generation.1048        Args:1049            prompt (`str` or `List[str]`):1050                The prompt or prompts to guide the image generation.1051            negative_prompt (`str` or `List[str]`, *optional*):1052                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored1053                if `guidance_scale` is less than `1`).1054            height (`int`, *optional*, defaults to 512):1055                The height in pixels of the generated image.1056            width (`int`, *optional*, defaults to 512):1057                The width in pixels of the generated image.1058            num_inference_steps (`int`, *optional*, defaults to 50):1059                The number of denoising steps. More denoising steps usually lead to a higher quality image at the1060                expense of slower inference.1061            guidance_scale (`float`, *optional*, defaults to 7.5):1062                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).1063                `guidance_scale` is defined as `w` of equation 2. of [Imagen1064                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >1065                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,1066                usually at the expense of lower image quality.1067            num_images_per_prompt (`int`, *optional*, defaults to 1):1068                The number of images to generate per prompt.1069            eta (`float`, *optional*, defaults to 0.0):1070                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to1071                [`schedulers.DDIMScheduler`], will be ignored for others.1072            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):1073                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)1074                to make generation deterministic.1075            latents (`torch.Tensor`, *optional*):1076                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image1077                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents1078                tensor will ge generated by sampling using the supplied random `generator`.1079            prompt_embeds (`torch.Tensor`, *optional*):1080                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not1081                provided, text embeddings will be generated from `prompt` input argument.1082            negative_prompt_embeds (`torch.Tensor`, *optional*):1083                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt1084                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input1085                argument.1086            max_embeddings_multiples (`int`, *optional*, defaults to `3`):1087                The max multiple length of prompt embeddings compared to the max output length of text encoder.1088            output_type (`str`, *optional*, defaults to `"pil"`):1089                The output format of the generate image. Choose between1090                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.1091            return_dict (`bool`, *optional*, defaults to `True`):1092                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a1093                plain tuple.1094            callback (`Callable`, *optional*):1095                A function that will be called every `callback_steps` steps during inference. The function will be1096                called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.1097            is_cancelled_callback (`Callable`, *optional*):1098                A function that will be called every `callback_steps` steps during inference. If the function returns1099                `True`, the inference will be cancelled.1100            callback_steps (`int`, *optional*, defaults to 1):1101                The frequency at which the `callback` function will be called. If not specified, the callback will be1102                called at every step.1103            cross_attention_kwargs (`dict`, *optional*):1104                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under1105                `self.processor` in1106                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).1107 1108        Returns:1109            `None` if cancelled by `is_cancelled_callback`,1110            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:1111            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.1112            When returning a tuple, the first element is a list with the generated images, and the second element is a1113            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"1114            (nsfw) content, according to the `safety_checker`.1115        """1116        return self.__call__(1117            prompt=prompt,1118            negative_prompt=negative_prompt,1119            height=height,1120            width=width,1121            num_inference_steps=num_inference_steps,1122            guidance_scale=guidance_scale,1123            num_images_per_prompt=num_images_per_prompt,1124            eta=eta,1125            generator=generator,1126            latents=latents,1127            prompt_embeds=prompt_embeds,1128            negative_prompt_embeds=negative_prompt_embeds,1129            max_embeddings_multiples=max_embeddings_multiples,1130            output_type=output_type,1131            return_dict=return_dict,1132            callback=callback,1133            is_cancelled_callback=is_cancelled_callback,1134            callback_steps=callback_steps,1135            cross_attention_kwargs=cross_attention_kwargs,1136        )1137 1138    def img2img(1139        self,1140        image: Union[torch.Tensor, PIL.Image.Image],1141        prompt: Union[str, List[str]],1142        negative_prompt: Optional[Union[str, List[str]]] = None,1143        strength: float = 0.8,1144        num_inference_steps: Optional[int] = 50,1145        guidance_scale: Optional[float] = 7.5,1146        num_images_per_prompt: Optional[int] = 1,1147        eta: Optional[float] = 0.0,1148        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,1149        prompt_embeds: Optional[torch.Tensor] = None,1150        negative_prompt_embeds: Optional[torch.Tensor] = None,1151        max_embeddings_multiples: Optional[int] = 3,1152        output_type: Optional[str] = "pil",1153        return_dict: bool = True,1154        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,1155        is_cancelled_callback: Optional[Callable[[], bool]] = None,1156        callback_steps: int = 1,1157        cross_attention_kwargs: Optional[Dict[str, Any]] = None,1158    ):1159        r"""1160        Function for image-to-image generation.1161        Args:1162            image (`torch.Tensor` or `PIL.Image.Image`):1163                `Image`, or tensor representing an image batch, that will be used as the starting point for the1164                process.1165            prompt (`str` or `List[str]`):1166                The prompt or prompts to guide the image generation.1167            negative_prompt (`str` or `List[str]`, *optional*):1168                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored1169                if `guidance_scale` is less than `1`).1170            strength (`float`, *optional*, defaults to 0.8):1171                Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1.1172                `image` will be used as a starting point, adding more noise to it the larger the `strength`. The1173                number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added1174                noise will be maximum and the denoising process will run for the full number of iterations specified in1175                `num_inference_steps`. A value of 1, therefore, essentially ignores `image`.1176            num_inference_steps (`int`, *optional*, defaults to 50):1177                The number of denoising steps. More denoising steps usually lead to a higher quality image at the1178                expense of slower inference. This parameter will be modulated by `strength`.1179            guidance_scale (`float`, *optional*, defaults to 7.5):1180                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).1181                `guidance_scale` is defined as `w` of equation 2. of [Imagen1182                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >1183                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,1184                usually at the expense of lower image quality.1185            num_images_per_prompt (`int`, *optional*, defaults to 1):1186                The number of images to generate per prompt.1187            eta (`float`, *optional*, defaults to 0.0):1188                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to1189                [`schedulers.DDIMScheduler`], will be ignored for others.1190            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):1191                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)1192                to make generation deterministic.1193            prompt_embeds (`torch.Tensor`, *optional*):1194                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not1195                provided, text embeddings will be generated from `prompt` input argument.1196            negative_prompt_embeds (`torch.Tensor`, *optional*):1197                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt1198                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input1199                argument.1200            max_embeddings_multiples (`int`, *optional*, defaults to `3`):

Showing the first 1,200 of 1372 lines. Download the file for the rest.