diffusers/community-pipelines-mirror

1# Copyright 2024 Long Lian, the GLIGEN Authors, and The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15# This is a single file implementation of LMD+. See README.md for examples.16 17import ast18import gc19import inspect20import math21import warnings22from collections.abc import Iterable23from typing import Any, Callable, Dict, List, Optional, Union24 25import torch26import torch.nn.functional as F27from packaging import version28from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection29 30from diffusers.configuration_utils import FrozenDict31from diffusers.image_processor import PipelineImageInput, VaeImageProcessor32from diffusers.loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin33from diffusers.models import AutoencoderKL, UNet2DConditionModel34from diffusers.models.attention import Attention, GatedSelfAttentionDense35from diffusers.models.attention_processor import AttnProcessor2_036from diffusers.models.lora import adjust_lora_scale_text_encoder37from diffusers.pipelines import DiffusionPipeline38from diffusers.pipelines.pipeline_utils import StableDiffusionMixin39from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput40from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker41from diffusers.schedulers import KarrasDiffusionSchedulers42from diffusers.utils import (43    USE_PEFT_BACKEND,44    deprecate,45    logging,46    replace_example_docstring,47    scale_lora_layers,48    unscale_lora_layers,49)50from diffusers.utils.torch_utils import randn_tensor51 52 53EXAMPLE_DOC_STRING = """54    Examples:55        ```py56        >>> import torch57        >>> from diffusers import DiffusionPipeline58 59        >>> pipe = DiffusionPipeline.from_pretrained(60        ...     "longlian/lmd_plus",61        ...     custom_pipeline="llm_grounded_diffusion",62        ...     custom_revision="main",63        ...     variant="fp16", torch_dtype=torch.float1664        ... )65        >>> pipe.enable_model_cpu_offload()66 67        >>> # Generate an image described by the prompt and68        >>> # insert objects described by text at the region defined by bounding boxes69        >>> prompt = "a waterfall and a modern high speed train in a beautiful forest with fall foliage"70        >>> boxes = [[0.1387, 0.2051, 0.4277, 0.7090], [0.4980, 0.4355, 0.8516, 0.7266]]71        >>> phrases = ["a waterfall", "a modern high speed train"]72 73        >>> images = pipe(74        ...     prompt=prompt,75        ...     phrases=phrases,76        ...     boxes=boxes,77        ...     gligen_scheduled_sampling_beta=0.4,78        ...     output_type="pil",79        ...     num_inference_steps=50,80        ...     lmd_guidance_kwargs={}81        ... ).images82 83        >>> images[0].save("./lmd_plus_generation.jpg")84 85        >>> # Generate directly from a text prompt and an LLM response86        >>> prompt = "a waterfall and a modern high speed train in a beautiful forest with fall foliage"87        >>> phrases, boxes, bg_prompt, neg_prompt = pipe.parse_llm_response(\"""88        [('a waterfall', [71, 105, 148, 258]), ('a modern high speed train', [255, 223, 181, 149])]89        Background prompt: A beautiful forest with fall foliage90        Negative prompt:91        \""")92 93        >> images = pipe(94        ...     prompt=prompt,95        ...     negative_prompt=neg_prompt,96        ...     phrases=phrases,97        ...     boxes=boxes,98        ...     gligen_scheduled_sampling_beta=0.4,99        ...     output_type="pil",100        ...     num_inference_steps=50,101        ...     lmd_guidance_kwargs={}102        ... ).images103 104        >>> images[0].save("./lmd_plus_generation.jpg")105 106images[0]107 108        ```109"""110 111logger = logging.get_logger(__name__)  # pylint: disable=invalid-name112 113# All keys in Stable Diffusion models: [('down', 0, 0, 0), ('down', 0, 1, 0), ('down', 1, 0, 0), ('down', 1, 1, 0), ('down', 2, 0, 0), ('down', 2, 1, 0), ('mid', 0, 0, 0), ('up', 1, 0, 0), ('up', 1, 1, 0), ('up', 1, 2, 0), ('up', 2, 0, 0), ('up', 2, 1, 0), ('up', 2, 2, 0), ('up', 3, 0, 0), ('up', 3, 1, 0), ('up', 3, 2, 0)]114# Note that the first up block is `UpBlock2D` rather than `CrossAttnUpBlock2D` and does not have attention. The last index is always 0 in our case since we have one `BasicTransformerBlock` in each `Transformer2DModel`.115DEFAULT_GUIDANCE_ATTN_KEYS = [116    ("mid", 0, 0, 0),117    ("up", 1, 0, 0),118    ("up", 1, 1, 0),119    ("up", 1, 2, 0),120]121 122 123def convert_attn_keys(key):124    """Convert the attention key from tuple format to the torch state format"""125 126    if key[0] == "mid":127        assert key[1] == 0, f"mid block only has one block but the index is {key[1]}"128        return f"{key[0]}_block.attentions.{key[2]}.transformer_blocks.{key[3]}.attn2.processor"129 130    return f"{key[0]}_blocks.{key[1]}.attentions.{key[2]}.transformer_blocks.{key[3]}.attn2.processor"131 132 133DEFAULT_GUIDANCE_ATTN_KEYS = [convert_attn_keys(key) for key in DEFAULT_GUIDANCE_ATTN_KEYS]134 135 136def scale_proportion(obj_box, H, W):137    # Separately rounding box_w and box_h to allow shift invariant box sizes. Otherwise box sizes may change when both coordinates being rounded end with ".5".138    x_min, y_min = round(obj_box[0] * W), round(obj_box[1] * H)139    box_w, box_h = round((obj_box[2] - obj_box[0]) * W), round((obj_box[3] - obj_box[1]) * H)140    x_max, y_max = x_min + box_w, y_min + box_h141 142    x_min, y_min = max(x_min, 0), max(y_min, 0)143    x_max, y_max = min(x_max, W), min(y_max, H)144 145    return x_min, y_min, x_max, y_max146 147 148# Adapted from the parent class `AttnProcessor2_0`149class AttnProcessorWithHook(AttnProcessor2_0):150    def __init__(151        self,152        attn_processor_key,153        hidden_size,154        cross_attention_dim,155        hook=None,156        fast_attn=True,157        enabled=True,158    ):159        super().__init__()160        self.attn_processor_key = attn_processor_key161        self.hidden_size = hidden_size162        self.cross_attention_dim = cross_attention_dim163        self.hook = hook164        self.fast_attn = fast_attn165        self.enabled = enabled166 167    def __call__(168        self,169        attn: Attention,170        hidden_states,171        encoder_hidden_states=None,172        attention_mask=None,173        temb=None,174        scale: float = 1.0,175    ):176        residual = hidden_states177 178        if attn.spatial_norm is not None:179            hidden_states = attn.spatial_norm(hidden_states, temb)180 181        input_ndim = hidden_states.ndim182 183        if input_ndim == 4:184            batch_size, channel, height, width = hidden_states.shape185            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)186 187        batch_size, sequence_length, _ = (188            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape189        )190 191        if attention_mask is not None:192            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)193 194        if attn.group_norm is not None:195            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)196 197        args = () if USE_PEFT_BACKEND else (scale,)198        query = attn.to_q(hidden_states, *args)199 200        if encoder_hidden_states is None:201            encoder_hidden_states = hidden_states202        elif attn.norm_cross:203            encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)204 205        key = attn.to_k(encoder_hidden_states, *args)206        value = attn.to_v(encoder_hidden_states, *args)207 208        inner_dim = key.shape[-1]209        head_dim = inner_dim // attn.heads210 211        if (self.hook is not None and self.enabled) or not self.fast_attn:212            query_batch_dim = attn.head_to_batch_dim(query)213            key_batch_dim = attn.head_to_batch_dim(key)214            value_batch_dim = attn.head_to_batch_dim(value)215            attention_probs = attn.get_attention_scores(query_batch_dim, key_batch_dim, attention_mask)216 217        if self.hook is not None and self.enabled:218            # Call the hook with query, key, value, and attention maps219            self.hook(220                self.attn_processor_key,221                query_batch_dim,222                key_batch_dim,223                value_batch_dim,224                attention_probs,225            )226 227        if self.fast_attn:228            query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)229 230            key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)231            value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)232 233            if attention_mask is not None:234                # scaled_dot_product_attention expects attention_mask shape to be235                # (batch, heads, source_length, target_length)236                attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])237 238            # the output of sdp = (batch, num_heads, seq_len, head_dim)239            # TODO: add support for attn.scale when we move to Torch 2.1240            hidden_states = F.scaled_dot_product_attention(241                query,242                key,243                value,244                attn_mask=attention_mask,245                dropout_p=0.0,246                is_causal=False,247            )248            hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)249            hidden_states = hidden_states.to(query.dtype)250        else:251            hidden_states = torch.bmm(attention_probs, value)252            hidden_states = attn.batch_to_head_dim(hidden_states)253 254        # linear proj255        hidden_states = attn.to_out[0](hidden_states, *args)256        # dropout257        hidden_states = attn.to_out[1](hidden_states)258 259        if input_ndim == 4:260            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)261 262        if attn.residual_connection:263            hidden_states = hidden_states + residual264 265        hidden_states = hidden_states / attn.rescale_output_factor266 267        return hidden_states268 269 270class LLMGroundedDiffusionPipeline(271    DiffusionPipeline,272    StableDiffusionMixin,273    TextualInversionLoaderMixin,274    LoraLoaderMixin,275    IPAdapterMixin,276    FromSingleFileMixin,277):278    r"""279    Pipeline for layout-grounded text-to-image generation using LLM-grounded Diffusion (LMD+): https://arxiv.org/pdf/2305.13655.pdf.280 281    This model inherits from [`StableDiffusionPipeline`] and aims at implementing the pipeline with minimal modifications. Check the superclass documentation for the generic methods282    implemented for all pipelines (downloading, saving, running on a particular device, etc.).283 284    This is a simplified implementation that does not perform latent or attention transfer from single object generation to overall generation. The final image is generated directly with attention and adapters control.285 286    Args:287        vae ([`AutoencoderKL`]):288            Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.289        text_encoder ([`~transformers.CLIPTextModel`]):290            Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).291        tokenizer ([`~transformers.CLIPTokenizer`]):292            A `CLIPTokenizer` to tokenize text.293        unet ([`UNet2DConditionModel`]):294            A `UNet2DConditionModel` to denoise the encoded image latents.295        scheduler ([`SchedulerMixin`]):296            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of297            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].298        safety_checker ([`StableDiffusionSafetyChecker`]):299            Classification module that estimates whether generated images could be considered offensive or harmful.300            Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details301            about a model's potential harms.302        feature_extractor ([`~transformers.CLIPImageProcessor`]):303            A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`.304        requires_safety_checker (bool):305            Whether a safety checker is needed for this pipeline.306    """307 308    model_cpu_offload_seq = "text_encoder->unet->vae"309    _optional_components = ["safety_checker", "feature_extractor", "image_encoder"]310    _exclude_from_cpu_offload = ["safety_checker"]311    _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]312 313    objects_text = "Objects: "314    bg_prompt_text = "Background prompt: "315    bg_prompt_text_no_trailing_space = bg_prompt_text.rstrip()316    neg_prompt_text = "Negative prompt: "317    neg_prompt_text_no_trailing_space = neg_prompt_text.rstrip()318 319    def __init__(320        self,321        vae: AutoencoderKL,322        text_encoder: CLIPTextModel,323        tokenizer: CLIPTokenizer,324        unet: UNet2DConditionModel,325        scheduler: KarrasDiffusionSchedulers,326        safety_checker: StableDiffusionSafetyChecker,327        feature_extractor: CLIPImageProcessor,328        image_encoder: CLIPVisionModelWithProjection = None,329        requires_safety_checker: bool = True,330    ):331        # This is copied from StableDiffusionPipeline, with hook initizations for LMD+.332        super().__init__()333 334        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:335            deprecation_message = (336                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"337                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "338                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"339                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"340                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"341                " file"342            )343            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)344            new_config = dict(scheduler.config)345            new_config["steps_offset"] = 1346            scheduler._internal_dict = FrozenDict(new_config)347 348        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:349            deprecation_message = (350                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."351                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"352                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"353                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"354                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"355            )356            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)357            new_config = dict(scheduler.config)358            new_config["clip_sample"] = False359            scheduler._internal_dict = FrozenDict(new_config)360 361        if safety_checker is None and requires_safety_checker:362            logger.warning(363                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"364                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"365                " results in services or applications open to the public. Both the diffusers team and Hugging Face"366                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"367                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"368                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."369            )370 371        if safety_checker is not None and feature_extractor is None:372            raise ValueError(373                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"374                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."375            )376 377        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(378            version.parse(unet.config._diffusers_version).base_version379        ) < version.parse("0.9.0.dev0")380        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64381        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:382            deprecation_message = (383                "The configuration file of the unet has set the default `sample_size` to smaller than"384                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"385                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"386                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"387                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"388                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"389                " in the config might lead to incorrect results in future versions. If you have downloaded this"390                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"391                " the `unet/config.json` file"392            )393            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)394            new_config = dict(unet.config)395            new_config["sample_size"] = 64396            unet._internal_dict = FrozenDict(new_config)397 398        self.register_modules(399            vae=vae,400            text_encoder=text_encoder,401            tokenizer=tokenizer,402            unet=unet,403            scheduler=scheduler,404            safety_checker=safety_checker,405            feature_extractor=feature_extractor,406            image_encoder=image_encoder,407        )408        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)409        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)410        self.register_to_config(requires_safety_checker=requires_safety_checker)411 412        # Initialize the attention hooks for LLM-grounded Diffusion413        self.register_attn_hooks(unet)414        self._saved_attn = None415 416    def attn_hook(self, name, query, key, value, attention_probs):417        if name in DEFAULT_GUIDANCE_ATTN_KEYS:418            self._saved_attn[name] = attention_probs419 420    @classmethod421    def convert_box(cls, box, height, width):422        # box: x, y, w, h (in 512 format) -> x_min, y_min, x_max, y_max423        x_min, y_min = box[0] / width, box[1] / height424        w_box, h_box = box[2] / width, box[3] / height425 426        x_max, y_max = x_min + w_box, y_min + h_box427 428        return x_min, y_min, x_max, y_max429 430    @classmethod431    def _parse_response_with_negative(cls, text):432        if not text:433            raise ValueError("LLM response is empty")434 435        if cls.objects_text in text:436            text = text.split(cls.objects_text)[1]437 438        text_split = text.split(cls.bg_prompt_text_no_trailing_space)439        if len(text_split) == 2:440            gen_boxes, text_rem = text_split441        else:442            raise ValueError(f"LLM response is incomplete: {text}")443 444        text_split = text_rem.split(cls.neg_prompt_text_no_trailing_space)445 446        if len(text_split) == 2:447            bg_prompt, neg_prompt = text_split448        else:449            raise ValueError(f"LLM response is incomplete: {text}")450 451        try:452            gen_boxes = ast.literal_eval(gen_boxes)453        except SyntaxError as e:454            # Sometimes the response is in plain text455            if "No objects" in gen_boxes or gen_boxes.strip() == "":456                gen_boxes = []457            else:458                raise e459        bg_prompt = bg_prompt.strip()460        neg_prompt = neg_prompt.strip()461 462        # LLM may return "None" to mean no negative prompt provided.463        if neg_prompt == "None":464            neg_prompt = ""465 466        return gen_boxes, bg_prompt, neg_prompt467 468    @classmethod469    def parse_llm_response(cls, response, canvas_height=512, canvas_width=512):470        # Infer from spec471        gen_boxes, bg_prompt, neg_prompt = cls._parse_response_with_negative(text=response)472 473        gen_boxes = sorted(gen_boxes, key=lambda gen_box: gen_box[0])474 475        phrases = [name for name, _ in gen_boxes]476        boxes = [cls.convert_box(box, height=canvas_height, width=canvas_width) for _, box in gen_boxes]477 478        return phrases, boxes, bg_prompt, neg_prompt479 480    def check_inputs(481        self,482        prompt,483        height,484        width,485        callback_steps,486        phrases,487        boxes,488        negative_prompt=None,489        prompt_embeds=None,490        negative_prompt_embeds=None,491        phrase_indices=None,492    ):493        if height % 8 != 0 or width % 8 != 0:494            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")495 496        if (callback_steps is None) or (497            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)498        ):499            raise ValueError(500                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"501                f" {type(callback_steps)}."502            )503 504        if prompt is not None and prompt_embeds is not None:505            raise ValueError(506                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"507                " only forward one of the two."508            )509        elif prompt is None and prompt_embeds is None:510            raise ValueError(511                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."512            )513        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):514            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")515        elif prompt is None and phrase_indices is None:516            raise ValueError("If the prompt is None, the phrase_indices cannot be None")517 518        if negative_prompt is not None and negative_prompt_embeds is not None:519            raise ValueError(520                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"521                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."522            )523 524        if prompt_embeds is not None and negative_prompt_embeds is not None:525            if prompt_embeds.shape != negative_prompt_embeds.shape:526                raise ValueError(527                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"528                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"529                    f" {negative_prompt_embeds.shape}."530                )531 532        if len(phrases) != len(boxes):533            raise ValueError(534                "length of `phrases` and `boxes` has to be same, but"535                f" got: `phrases` {len(phrases)} != `boxes` {len(boxes)}"536            )537 538    def register_attn_hooks(self, unet):539        """Registering hooks to obtain the attention maps for guidance"""540 541        attn_procs = {}542 543        for name in unet.attn_processors.keys():544            # Only obtain the queries and keys from cross-attention545            if name.endswith("attn1.processor") or name.endswith("fuser.attn.processor"):546                # Keep the same attn_processors for self-attention (no hooks for self-attention)547                attn_procs[name] = unet.attn_processors[name]548                continue549 550            cross_attention_dim = None if name.endswith("attn1.processor") else unet.config.cross_attention_dim551 552            if name.startswith("mid_block"):553                hidden_size = unet.config.block_out_channels[-1]554            elif name.startswith("up_blocks"):555                block_id = int(name[len("up_blocks.")])556                hidden_size = list(reversed(unet.config.block_out_channels))[block_id]557            elif name.startswith("down_blocks"):558                block_id = int(name[len("down_blocks.")])559                hidden_size = unet.config.block_out_channels[block_id]560 561            attn_procs[name] = AttnProcessorWithHook(562                attn_processor_key=name,563                hidden_size=hidden_size,564                cross_attention_dim=cross_attention_dim,565                hook=self.attn_hook,566                fast_attn=True,567                # Not enabled by default568                enabled=False,569            )570 571        unet.set_attn_processor(attn_procs)572 573    def enable_fuser(self, enabled=True):574        for module in self.unet.modules():575            if isinstance(module, GatedSelfAttentionDense):576                module.enabled = enabled577 578    def enable_attn_hook(self, enabled=True):579        for module in self.unet.attn_processors.values():580            if isinstance(module, AttnProcessorWithHook):581                module.enabled = enabled582 583    def get_token_map(self, prompt, padding="do_not_pad", verbose=False):584        """Get a list of mapping: prompt index to str (prompt in a list of token str)"""585        fg_prompt_tokens = self.tokenizer([prompt], padding=padding, max_length=77, return_tensors="np")586        input_ids = fg_prompt_tokens["input_ids"][0]587 588        token_map = []589        for ind, item in enumerate(input_ids.tolist()):590            token = self.tokenizer._convert_id_to_token(item)591 592            if verbose:593                logger.info(f"{ind}, {token} ({item})")594 595            token_map.append(token)596 597        return token_map598 599    def get_phrase_indices(600        self,601        prompt,602        phrases,603        token_map=None,604        add_suffix_if_not_found=False,605        verbose=False,606    ):607        for obj in phrases:608            # Suffix the prompt with object name for attention guidance if object is not in the prompt, using "|" to separate the prompt and the suffix609            if obj not in prompt:610                prompt += "| " + obj611 612        if token_map is None:613            # We allow using a pre-computed token map.614            token_map = self.get_token_map(prompt=prompt, padding="do_not_pad", verbose=verbose)615        token_map_str = " ".join(token_map)616 617        phrase_indices = []618 619        for obj in phrases:620            phrase_token_map = self.get_token_map(prompt=obj, padding="do_not_pad", verbose=verbose)621            # Remove <bos> and <eos> in substr622            phrase_token_map = phrase_token_map[1:-1]623            phrase_token_map_len = len(phrase_token_map)624            phrase_token_map_str = " ".join(phrase_token_map)625 626            if verbose:627                logger.info(628                    "Full str:",629                    token_map_str,630                    "Substr:",631                    phrase_token_map_str,632                    "Phrase:",633                    phrases,634                )635 636            # Count the number of token before substr637            # The substring comes with a trailing space that needs to be removed by minus one in the index.638            obj_first_index = len(token_map_str[: token_map_str.index(phrase_token_map_str) - 1].split(" "))639 640            obj_position = list(range(obj_first_index, obj_first_index + phrase_token_map_len))641            phrase_indices.append(obj_position)642 643        if add_suffix_if_not_found:644            return phrase_indices, prompt645 646        return phrase_indices647 648    def add_ca_loss_per_attn_map_to_loss(649        self,650        loss,651        attn_map,652        object_number,653        bboxes,654        phrase_indices,655        fg_top_p=0.2,656        bg_top_p=0.2,657        fg_weight=1.0,658        bg_weight=1.0,659    ):660        # b is the number of heads, not batch661        b, i, j = attn_map.shape662        H = W = int(math.sqrt(i))663        for obj_idx in range(object_number):664            obj_loss = 0665            mask = torch.zeros(size=(H, W), device="cuda")666            obj_boxes = bboxes[obj_idx]667 668            # We support two level (one box per phrase) and three level (multiple boxes per phrase)669            if not isinstance(obj_boxes[0], Iterable):670                obj_boxes = [obj_boxes]671 672            for obj_box in obj_boxes:673                # x_min, y_min, x_max, y_max = int(obj_box[0] * W), int(obj_box[1] * H), int(obj_box[2] * W), int(obj_box[3] * H)674                x_min, y_min, x_max, y_max = scale_proportion(obj_box, H=H, W=W)675                mask[y_min:y_max, x_min:x_max] = 1676 677            for obj_position in phrase_indices[obj_idx]:678                # Could potentially optimize to compute this for loop in batch.679                # Could crop the ref cross attention before saving to save memory.680 681                ca_map_obj = attn_map[:, :, obj_position].reshape(b, H, W)682 683                # shape: (b, H * W)684                ca_map_obj = attn_map[:, :, obj_position]  # .reshape(b, H, W)685                k_fg = (mask.sum() * fg_top_p).long().clamp_(min=1)686                k_bg = ((1 - mask).sum() * bg_top_p).long().clamp_(min=1)687 688                mask_1d = mask.view(1, -1)689 690                # Max-based loss function691 692                # Take the topk over spatial dimension, and then take the sum over heads dim693                # The mean is over k_fg and k_bg dimension, so we don't need to sum and divide on our own.694                obj_loss += (1 - (ca_map_obj * mask_1d).topk(k=k_fg).values.mean(dim=1)).sum(dim=0) * fg_weight695                obj_loss += ((ca_map_obj * (1 - mask_1d)).topk(k=k_bg).values.mean(dim=1)).sum(dim=0) * bg_weight696 697            loss += obj_loss / len(phrase_indices[obj_idx])698 699        return loss700 701    def compute_ca_loss(702        self,703        saved_attn,704        bboxes,705        phrase_indices,706        guidance_attn_keys,707        verbose=False,708        **kwargs,709    ):710        """711        The `saved_attn` is supposed to be passed to `save_attn_to_dict` in `cross_attention_kwargs` prior to computing ths loss.712        `AttnProcessor` will put attention maps into the `save_attn_to_dict`.713 714        `index` is the timestep.715        `ref_ca_word_token_only`: This has precedence over `ref_ca_last_token_only` (i.e., if both are enabled, we take the token from word rather than the last token).716        `ref_ca_last_token_only`: `ref_ca_saved_attn` comes from the attention map of the last token of the phrase in single object generation, so we apply it only to the last token of the phrase in overall generation if this is set to True. If set to False, `ref_ca_saved_attn` will be applied to all the text tokens.717        """718        loss = torch.tensor(0).float().cuda()719        object_number = len(bboxes)720        if object_number == 0:721            return loss722 723        for attn_key in guidance_attn_keys:724            # We only have 1 cross attention for mid.725 726            attn_map_integrated = saved_attn[attn_key]727            if not attn_map_integrated.is_cuda:728                attn_map_integrated = attn_map_integrated.cuda()729            # Example dimension: [20, 64, 77]730            attn_map = attn_map_integrated.squeeze(dim=0)731 732            loss = self.add_ca_loss_per_attn_map_to_loss(733                loss, attn_map, object_number, bboxes, phrase_indices, **kwargs734            )735 736        num_attn = len(guidance_attn_keys)737 738        if num_attn > 0:739            loss = loss / (object_number * num_attn)740 741        return loss742 743    @torch.no_grad()744    @replace_example_docstring(EXAMPLE_DOC_STRING)745    def __call__(746        self,747        prompt: Union[str, List[str]] = None,748        height: Optional[int] = None,749        width: Optional[int] = None,750        num_inference_steps: int = 50,751        guidance_scale: float = 7.5,752        gligen_scheduled_sampling_beta: float = 0.3,753        phrases: List[str] = None,754        boxes: List[List[float]] = None,755        negative_prompt: Optional[Union[str, List[str]]] = None,756        num_images_per_prompt: Optional[int] = 1,757        eta: float = 0.0,758        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,759        latents: Optional[torch.Tensor] = None,760        prompt_embeds: Optional[torch.Tensor] = None,761        negative_prompt_embeds: Optional[torch.Tensor] = None,762        ip_adapter_image: Optional[PipelineImageInput] = None,763        output_type: Optional[str] = "pil",764        return_dict: bool = True,765        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,766        callback_steps: int = 1,767        cross_attention_kwargs: Optional[Dict[str, Any]] = None,768        clip_skip: Optional[int] = None,769        lmd_guidance_kwargs: Optional[Dict[str, Any]] = {},770        phrase_indices: Optional[List[int]] = None,771    ):772        r"""773        The call function to the pipeline for generation.774 775        Args:776            prompt (`str` or `List[str]`, *optional*):777                The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.778            height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):779                The height in pixels of the generated image.780            width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):781                The width in pixels of the generated image.782            num_inference_steps (`int`, *optional*, defaults to 50):783                The number of denoising steps. More denoising steps usually lead to a higher quality image at the784                expense of slower inference.785            guidance_scale (`float`, *optional*, defaults to 7.5):786                A higher guidance scale value encourages the model to generate images closely linked to the text787                `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.788            phrases (`List[str]`):789                The phrases to guide what to include in each of the regions defined by the corresponding790                `boxes`. There should only be one phrase per bounding box.791            boxes (`List[List[float]]`):792                The bounding boxes that identify rectangular regions of the image that are going to be filled with the793                content described by the corresponding `phrases`. Each rectangular box is defined as a794                `List[float]` of 4 elements `[xmin, ymin, xmax, ymax]` where each value is between [0,1].795            gligen_scheduled_sampling_beta (`float`, defaults to 0.3):796                Scheduled Sampling factor from [GLIGEN: Open-Set Grounded Text-to-Image797                Generation](https://arxiv.org/pdf/2301.07093.pdf). Scheduled Sampling factor is only varied for798                scheduled sampling during inference for improved quality and controllability.799            negative_prompt (`str` or `List[str]`, *optional*):800                The prompt or prompts to guide what to not include in image generation. If not defined, you need to801                pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).802            num_images_per_prompt (`int`, *optional*, defaults to 1):803                The number of images to generate per prompt.804            eta (`float`, *optional*, defaults to 0.0):805                Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies806                to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.807            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):808                A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make809                generation deterministic.810            latents (`torch.Tensor`, *optional*):811                Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image812                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents813                tensor is generated by sampling using the supplied random `generator`.814            prompt_embeds (`torch.Tensor`, *optional*):815                Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not816                provided, text embeddings are generated from the `prompt` input argument.817            negative_prompt_embeds (`torch.Tensor`, *optional*):818                Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If819                not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.820            ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.821            output_type (`str`, *optional*, defaults to `"pil"`):822                The output format of the generated image. Choose between `PIL.Image` or `np.array`.823            return_dict (`bool`, *optional*, defaults to `True`):824                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a825                plain tuple.826            callback (`Callable`, *optional*):827                A function that calls every `callback_steps` steps during inference. The function is called with the828                following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.829            callback_steps (`int`, *optional*, defaults to 1):830                The frequency at which the `callback` function is called. If not specified, the callback is called at831                every step.832            cross_attention_kwargs (`dict`, *optional*):833                A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in834                [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).835            guidance_rescale (`float`, *optional*, defaults to 0.0):836                Guidance rescale factor from [Common Diffusion Noise Schedules and Sample Steps are837                Flawed](https://arxiv.org/pdf/2305.08891.pdf). Guidance rescale factor should fix overexposure when838                using zero terminal SNR.839            clip_skip (`int`, *optional*):840                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that841                the output of the pre-final layer will be used for computing the prompt embeddings.842            lmd_guidance_kwargs (`dict`, *optional*):843                A kwargs dictionary that if specified is passed along to `latent_lmd_guidance` function. Useful keys include `loss_scale` (the guidance strength), `loss_threshold` (when loss is lower than this value, the guidance is not applied anymore), `max_iter` (the number of iterations of guidance for each step), and `guidance_timesteps` (the number of diffusion timesteps to apply guidance on). See `latent_lmd_guidance` for implementation details.844            phrase_indices (`list` of `list`, *optional*): The indices of the tokens of each phrase in the overall prompt. If omitted, the pipeline will match the first token subsequence. The pipeline will append the missing phrases to the end of the prompt by default.845        Examples:846 847        Returns:848            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:849                If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned,850                otherwise a `tuple` is returned where the first element is a list with the generated images and the851                second element is a list of `bool`s indicating whether the corresponding generated image contains852                "not-safe-for-work" (nsfw) content.853        """854        # 0. Default height and width to unet855        height = height or self.unet.config.sample_size * self.vae_scale_factor856        width = width or self.unet.config.sample_size * self.vae_scale_factor857 858        # 1. Check inputs. Raise error if not correct859        self.check_inputs(860            prompt,861            height,862            width,863            callback_steps,864            phrases,865            boxes,866            negative_prompt,867            prompt_embeds,868            negative_prompt_embeds,869            phrase_indices,870        )871 872        # 2. Define call parameters873        if prompt is not None and isinstance(prompt, str):874            batch_size = 1875            if phrase_indices is None:876                phrase_indices, prompt = self.get_phrase_indices(prompt, phrases, add_suffix_if_not_found=True)877        elif prompt is not None and isinstance(prompt, list):878            batch_size = len(prompt)879            if phrase_indices is None:880                phrase_indices = []881                prompt_parsed = []882                for prompt_item in prompt:883                    (884                        phrase_indices_parsed_item,885                        prompt_parsed_item,886                    ) = self.get_phrase_indices(prompt_item, add_suffix_if_not_found=True)887                    phrase_indices.append(phrase_indices_parsed_item)888                    prompt_parsed.append(prompt_parsed_item)889                prompt = prompt_parsed890        else:891            batch_size = prompt_embeds.shape[0]892 893        device = self._execution_device894        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)895        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`896        # corresponds to doing no classifier free guidance.897        do_classifier_free_guidance = guidance_scale > 1.0898 899        # 3. Encode input prompt900        prompt_embeds, negative_prompt_embeds = self.encode_prompt(901            prompt,902            device,903            num_images_per_prompt,904            do_classifier_free_guidance,905            negative_prompt,906            prompt_embeds=prompt_embeds,907            negative_prompt_embeds=negative_prompt_embeds,908            clip_skip=clip_skip,909        )910 911        cond_prompt_embeds = prompt_embeds912 913        # For classifier free guidance, we need to do two forward passes.914        # Here we concatenate the unconditional and text embeddings into a single batch915        # to avoid doing two forward passes916        if do_classifier_free_guidance:917            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])918 919        if ip_adapter_image is not None:920            image_embeds, negative_image_embeds = self.encode_image(ip_adapter_image, device, num_images_per_prompt)921            if self.do_classifier_free_guidance:922                image_embeds = torch.cat([negative_image_embeds, image_embeds])923 924        # 4. Prepare timesteps925        self.scheduler.set_timesteps(num_inference_steps, device=device)926        timesteps = self.scheduler.timesteps927 928        # 5. Prepare latent variables929        num_channels_latents = self.unet.config.in_channels930        latents = self.prepare_latents(931            batch_size * num_images_per_prompt,932            num_channels_latents,933            height,934            width,935            prompt_embeds.dtype,936            device,937            generator,938            latents,939        )940 941        # 5.1 Prepare GLIGEN variables942        max_objs = 30943        if len(boxes) > max_objs:944            warnings.warn(945                f"More that {max_objs} objects found. Only first {max_objs} objects will be processed.",946                FutureWarning,947            )948            phrases = phrases[:max_objs]949            boxes = boxes[:max_objs]950 951        n_objs = len(boxes)952        if n_objs:953            # prepare batched input to the PositionNet (boxes, phrases, mask)954            # Get tokens for phrases from pre-trained CLIPTokenizer955            tokenizer_inputs = self.tokenizer(phrases, padding=True, return_tensors="pt").to(device)956            # For the token, we use the same pre-trained text encoder957            # to obtain its text feature958            _text_embeddings = self.text_encoder(**tokenizer_inputs).pooler_output959 960        # For each entity, described in phrases, is denoted with a bounding box,961        # we represent the location information as (xmin,ymin,xmax,ymax)962        cond_boxes = torch.zeros(max_objs, 4, device=device, dtype=self.text_encoder.dtype)963        if n_objs:964            cond_boxes[:n_objs] = torch.tensor(boxes)965        text_embeddings = torch.zeros(966            max_objs,967            self.unet.config.cross_attention_dim,968            device=device,969            dtype=self.text_encoder.dtype,970        )971        if n_objs:972            text_embeddings[:n_objs] = _text_embeddings973        # Generate a mask for each object that is entity described by phrases974        masks = torch.zeros(max_objs, device=device, dtype=self.text_encoder.dtype)975        masks[:n_objs] = 1976 977        repeat_batch = batch_size * num_images_per_prompt978        cond_boxes = cond_boxes.unsqueeze(0).expand(repeat_batch, -1, -1).clone()979        text_embeddings = text_embeddings.unsqueeze(0).expand(repeat_batch, -1, -1).clone()980        masks = masks.unsqueeze(0).expand(repeat_batch, -1).clone()981        if do_classifier_free_guidance:982            repeat_batch = repeat_batch * 2983            cond_boxes = torch.cat([cond_boxes] * 2)984            text_embeddings = torch.cat([text_embeddings] * 2)985            masks = torch.cat([masks] * 2)986            masks[: repeat_batch // 2] = 0987        if cross_attention_kwargs is None:988            cross_attention_kwargs = {}989        cross_attention_kwargs["gligen"] = {990            "boxes": cond_boxes,991            "positive_embeddings": text_embeddings,992            "masks": masks,993        }994 995        num_grounding_steps = int(gligen_scheduled_sampling_beta * len(timesteps))996        self.enable_fuser(True)997 998        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline999        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)1000 1001        # 6.1 Add image embeds for IP-Adapter1002        added_cond_kwargs = {"image_embeds": image_embeds} if ip_adapter_image is not None else None1003 1004        loss_attn = torch.tensor(10000.0)1005 1006        # 7. Denoising loop1007        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order1008        with self.progress_bar(total=num_inference_steps) as progress_bar:1009            for i, t in enumerate(timesteps):1010                # Scheduled sampling1011                if i == num_grounding_steps:1012                    self.enable_fuser(False)1013 1014                if latents.shape[1] != 4:1015                    latents = torch.randn_like(latents[:, :4])1016 1017                # 7.1 Perform LMD guidance1018                if boxes:1019                    latents, loss_attn = self.latent_lmd_guidance(1020                        cond_prompt_embeds,1021                        index=i,1022                        boxes=boxes,1023                        phrase_indices=phrase_indices,1024                        t=t,1025                        latents=latents,1026                        loss=loss_attn,1027                        **lmd_guidance_kwargs,1028                    )1029 1030                # expand the latents if we are doing classifier free guidance1031                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents1032                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)1033 1034                # predict the noise residual1035                noise_pred = self.unet(1036                    latent_model_input,1037                    t,1038                    encoder_hidden_states=prompt_embeds,1039                    cross_attention_kwargs=cross_attention_kwargs,1040                    added_cond_kwargs=added_cond_kwargs,1041                ).sample1042 1043                # perform guidance1044                if do_classifier_free_guidance:1045                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)1046                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)1047 1048                # compute the previous noisy sample x_t -> x_t-11049                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample1050 1051                # call the callback, if provided1052                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):1053                    progress_bar.update()1054                    if callback is not None and i % callback_steps == 0:1055                        step_idx = i // getattr(self.scheduler, "order", 1)1056                        callback(step_idx, t, latents)1057 1058        if not output_type == "latent":1059            image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]1060            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)1061        else:1062            image = latents1063            has_nsfw_concept = None1064 1065        if has_nsfw_concept is None:1066            do_denormalize = [True] * image.shape[0]1067        else:1068            do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept]1069 1070        image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)1071 1072        # Offload last model to CPU1073        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:1074            self.final_offload_hook.offload()1075 1076        if not return_dict:1077            return (image, has_nsfw_concept)1078 1079        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)1080 1081    @torch.set_grad_enabled(True)1082    def latent_lmd_guidance(1083        self,1084        cond_embeddings,1085        index,1086        boxes,1087        phrase_indices,1088        t,1089        latents,1090        loss,1091        *,1092        loss_scale=20,1093        loss_threshold=5.0,1094        max_iter=[3] * 5 + [2] * 5 + [1] * 5,1095        guidance_timesteps=15,1096        cross_attention_kwargs=None,1097        guidance_attn_keys=DEFAULT_GUIDANCE_ATTN_KEYS,1098        verbose=False,1099        clear_cache=False,1100        unet_additional_kwargs={},1101        guidance_callback=None,1102        **kwargs,1103    ):1104        scheduler, unet = self.scheduler, self.unet1105 1106        iteration = 01107 1108        if index < guidance_timesteps:1109            if isinstance(max_iter, list):1110                max_iter = max_iter[index]1111 1112            if verbose:1113                logger.info(1114                    f"time index {index}, loss: {loss.item()/loss_scale:.3f} (de-scaled with scale {loss_scale:.1f}), loss threshold: {loss_threshold:.3f}"1115                )1116 1117            try:1118                self.enable_attn_hook(enabled=True)1119 1120                while (1121                    loss.item() / loss_scale > loss_threshold and iteration < max_iter and index < guidance_timesteps1122                ):1123                    self._saved_attn = {}1124 1125                    latents.requires_grad_(True)1126                    latent_model_input = latents1127                    latent_model_input = scheduler.scale_model_input(latent_model_input, t)1128 1129                    unet(1130                        latent_model_input,1131                        t,1132                        encoder_hidden_states=cond_embeddings,1133                        cross_attention_kwargs=cross_attention_kwargs,1134                        **unet_additional_kwargs,1135                    )1136 1137                    # update latents with guidance1138                    loss = (1139                        self.compute_ca_loss(1140                            saved_attn=self._saved_attn,1141                            bboxes=boxes,1142                            phrase_indices=phrase_indices,1143                            guidance_attn_keys=guidance_attn_keys,1144                            verbose=verbose,1145                            **kwargs,1146                        )1147                        * loss_scale1148                    )1149 1150                    if torch.isnan(loss):1151                        raise RuntimeError("**Loss is NaN**")1152 1153                    # This callback allows visualizations.1154                    if guidance_callback is not None:1155                        guidance_callback(self, latents, loss, iteration, index)1156 1157                    self._saved_attn = None1158 1159                    grad_cond = torch.autograd.grad(loss.requires_grad_(True), [latents])[0]1160 1161                    latents.requires_grad_(False)1162 1163                    # Scaling with classifier guidance1164                    alpha_prod_t = scheduler.alphas_cumprod[t]1165                    # Classifier guidance: https://arxiv.org/pdf/2105.05233.pdf1166                    # DDIM: https://arxiv.org/pdf/2010.02502.pdf1167                    scale = (1 - alpha_prod_t) ** (0.5)1168                    latents = latents - scale * grad_cond1169 1170                    iteration += 11171 1172                    if clear_cache:1173                        gc.collect()1174                        torch.cuda.empty_cache()1175 1176                    if verbose:1177                        logger.info(1178                            f"time index {index}, loss: {loss.item()/loss_scale:.3f}, loss threshold: {loss_threshold:.3f}, iteration: {iteration}"1179                        )1180 1181            finally:1182                self.enable_attn_hook(enabled=False)1183 1184        return latents, loss1185 1186    # Below are methods copied from StableDiffusionPipeline1187    # The design choice of not inheriting from StableDiffusionPipeline is discussed here: https://github.com/huggingface/diffusers/pull/5993#issuecomment-18342585171188 1189    # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt1190    def _encode_prompt(1191        self,1192        prompt,1193        device,1194        num_images_per_prompt,1195        do_classifier_free_guidance,1196        negative_prompt=None,1197        prompt_embeds: Optional[torch.Tensor] = None,1198        negative_prompt_embeds: Optional[torch.Tensor] = None,1199        lora_scale: Optional[float] = None,1200        **kwargs,

Showing the first 1,200 of 1559 lines. Download the file for the rest.