diffusers/community-pipelines-mirror

1# Copyright 2025 Long Lian, the GLIGEN Authors, and The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15# This is a single file implementation of LMD+. See README.md for examples.16 17import ast18import gc19import inspect20import math21import warnings22from collections.abc import Iterable23from typing import Any, Callable, Dict, List, Optional, Union24 25import torch26import torch.nn.functional as F27from packaging import version28from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection29 30from diffusers.configuration_utils import FrozenDict31from diffusers.image_processor import PipelineImageInput, VaeImageProcessor32from diffusers.loaders import (33    FromSingleFileMixin,34    IPAdapterMixin,35    StableDiffusionLoraLoaderMixin,36    TextualInversionLoaderMixin,37)38from diffusers.models import AutoencoderKL, UNet2DConditionModel39from diffusers.models.attention import Attention, GatedSelfAttentionDense40from diffusers.models.attention_processor import AttnProcessor2_041from diffusers.models.lora import adjust_lora_scale_text_encoder42from diffusers.pipelines import DiffusionPipeline43from diffusers.pipelines.pipeline_utils import StableDiffusionMixin44from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput45from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker46from diffusers.schedulers import KarrasDiffusionSchedulers47from diffusers.utils import (48    USE_PEFT_BACKEND,49    deprecate,50    logging,51    replace_example_docstring,52    scale_lora_layers,53    unscale_lora_layers,54)55from diffusers.utils.torch_utils import randn_tensor56 57 58EXAMPLE_DOC_STRING = """59    Examples:60        ```py61        >>> import torch62        >>> from diffusers import DiffusionPipeline63 64        >>> pipe = DiffusionPipeline.from_pretrained(65        ...     "longlian/lmd_plus",66        ...     custom_pipeline="llm_grounded_diffusion",67        ...     custom_revision="main",68        ...     variant="fp16", torch_dtype=torch.float1669        ... )70        >>> pipe.enable_model_cpu_offload()71 72        >>> # Generate an image described by the prompt and73        >>> # insert objects described by text at the region defined by bounding boxes74        >>> prompt = "a waterfall and a modern high speed train in a beautiful forest with fall foliage"75        >>> boxes = [[0.1387, 0.2051, 0.4277, 0.7090], [0.4980, 0.4355, 0.8516, 0.7266]]76        >>> phrases = ["a waterfall", "a modern high speed train"]77 78        >>> images = pipe(79        ...     prompt=prompt,80        ...     phrases=phrases,81        ...     boxes=boxes,82        ...     gligen_scheduled_sampling_beta=0.4,83        ...     output_type="pil",84        ...     num_inference_steps=50,85        ...     lmd_guidance_kwargs={}86        ... ).images87 88        >>> images[0].save("./lmd_plus_generation.jpg")89 90        >>> # Generate directly from a text prompt and an LLM response91        >>> prompt = "a waterfall and a modern high speed train in a beautiful forest with fall foliage"92        >>> phrases, boxes, bg_prompt, neg_prompt = pipe.parse_llm_response(\"""93        [('a waterfall', [71, 105, 148, 258]), ('a modern high speed train', [255, 223, 181, 149])]94        Background prompt: A beautiful forest with fall foliage95        Negative prompt:96        \""")97 98        >> images = pipe(99        ...     prompt=prompt,100        ...     negative_prompt=neg_prompt,101        ...     phrases=phrases,102        ...     boxes=boxes,103        ...     gligen_scheduled_sampling_beta=0.4,104        ...     output_type="pil",105        ...     num_inference_steps=50,106        ...     lmd_guidance_kwargs={}107        ... ).images108 109        >>> images[0].save("./lmd_plus_generation.jpg")110 111images[0]112 113        ```114"""115 116logger = logging.get_logger(__name__)  # pylint: disable=invalid-name117 118# All keys in Stable Diffusion models: [('down', 0, 0, 0), ('down', 0, 1, 0), ('down', 1, 0, 0), ('down', 1, 1, 0), ('down', 2, 0, 0), ('down', 2, 1, 0), ('mid', 0, 0, 0), ('up', 1, 0, 0), ('up', 1, 1, 0), ('up', 1, 2, 0), ('up', 2, 0, 0), ('up', 2, 1, 0), ('up', 2, 2, 0), ('up', 3, 0, 0), ('up', 3, 1, 0), ('up', 3, 2, 0)]119# Note that the first up block is `UpBlock2D` rather than `CrossAttnUpBlock2D` and does not have attention. The last index is always 0 in our case since we have one `BasicTransformerBlock` in each `Transformer2DModel`.120DEFAULT_GUIDANCE_ATTN_KEYS = [121    ("mid", 0, 0, 0),122    ("up", 1, 0, 0),123    ("up", 1, 1, 0),124    ("up", 1, 2, 0),125]126 127 128def convert_attn_keys(key):129    """Convert the attention key from tuple format to the torch state format"""130 131    if key[0] == "mid":132        assert key[1] == 0, f"mid block only has one block but the index is {key[1]}"133        return f"{key[0]}_block.attentions.{key[2]}.transformer_blocks.{key[3]}.attn2.processor"134 135    return f"{key[0]}_blocks.{key[1]}.attentions.{key[2]}.transformer_blocks.{key[3]}.attn2.processor"136 137 138DEFAULT_GUIDANCE_ATTN_KEYS = [convert_attn_keys(key) for key in DEFAULT_GUIDANCE_ATTN_KEYS]139 140 141def scale_proportion(obj_box, H, W):142    # Separately rounding box_w and box_h to allow shift invariant box sizes. Otherwise box sizes may change when both coordinates being rounded end with ".5".143    x_min, y_min = round(obj_box[0] * W), round(obj_box[1] * H)144    box_w, box_h = round((obj_box[2] - obj_box[0]) * W), round((obj_box[3] - obj_box[1]) * H)145    x_max, y_max = x_min + box_w, y_min + box_h146 147    x_min, y_min = max(x_min, 0), max(y_min, 0)148    x_max, y_max = min(x_max, W), min(y_max, H)149 150    return x_min, y_min, x_max, y_max151 152 153# Adapted from the parent class `AttnProcessor2_0`154class AttnProcessorWithHook(AttnProcessor2_0):155    def __init__(156        self,157        attn_processor_key,158        hidden_size,159        cross_attention_dim,160        hook=None,161        fast_attn=True,162        enabled=True,163    ):164        super().__init__()165        self.attn_processor_key = attn_processor_key166        self.hidden_size = hidden_size167        self.cross_attention_dim = cross_attention_dim168        self.hook = hook169        self.fast_attn = fast_attn170        self.enabled = enabled171 172    def __call__(173        self,174        attn: Attention,175        hidden_states,176        encoder_hidden_states=None,177        attention_mask=None,178        temb=None,179        scale: float = 1.0,180    ):181        residual = hidden_states182 183        if attn.spatial_norm is not None:184            hidden_states = attn.spatial_norm(hidden_states, temb)185 186        input_ndim = hidden_states.ndim187 188        if input_ndim == 4:189            batch_size, channel, height, width = hidden_states.shape190            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)191 192        batch_size, sequence_length, _ = (193            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape194        )195 196        if attention_mask is not None:197            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)198 199        if attn.group_norm is not None:200            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)201 202        args = () if USE_PEFT_BACKEND else (scale,)203        query = attn.to_q(hidden_states, *args)204 205        if encoder_hidden_states is None:206            encoder_hidden_states = hidden_states207        elif attn.norm_cross:208            encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)209 210        key = attn.to_k(encoder_hidden_states, *args)211        value = attn.to_v(encoder_hidden_states, *args)212 213        inner_dim = key.shape[-1]214        head_dim = inner_dim // attn.heads215 216        if (self.hook is not None and self.enabled) or not self.fast_attn:217            query_batch_dim = attn.head_to_batch_dim(query)218            key_batch_dim = attn.head_to_batch_dim(key)219            value_batch_dim = attn.head_to_batch_dim(value)220            attention_probs = attn.get_attention_scores(query_batch_dim, key_batch_dim, attention_mask)221 222        if self.hook is not None and self.enabled:223            # Call the hook with query, key, value, and attention maps224            self.hook(225                self.attn_processor_key,226                query_batch_dim,227                key_batch_dim,228                value_batch_dim,229                attention_probs,230            )231 232        if self.fast_attn:233            query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)234 235            key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)236            value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)237 238            if attention_mask is not None:239                # scaled_dot_product_attention expects attention_mask shape to be240                # (batch, heads, source_length, target_length)241                attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])242 243            # the output of sdp = (batch, num_heads, seq_len, head_dim)244            # TODO: add support for attn.scale when we move to Torch 2.1245            hidden_states = F.scaled_dot_product_attention(246                query,247                key,248                value,249                attn_mask=attention_mask,250                dropout_p=0.0,251                is_causal=False,252            )253            hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)254            hidden_states = hidden_states.to(query.dtype)255        else:256            hidden_states = torch.bmm(attention_probs, value)257            hidden_states = attn.batch_to_head_dim(hidden_states)258 259        # linear proj260        hidden_states = attn.to_out[0](hidden_states, *args)261        # dropout262        hidden_states = attn.to_out[1](hidden_states)263 264        if input_ndim == 4:265            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)266 267        if attn.residual_connection:268            hidden_states = hidden_states + residual269 270        hidden_states = hidden_states / attn.rescale_output_factor271 272        return hidden_states273 274 275class LLMGroundedDiffusionPipeline(276    DiffusionPipeline,277    StableDiffusionMixin,278    TextualInversionLoaderMixin,279    StableDiffusionLoraLoaderMixin,280    IPAdapterMixin,281    FromSingleFileMixin,282):283    r"""284    Pipeline for layout-grounded text-to-image generation using LLM-grounded Diffusion (LMD+): https://huggingface.co/papers/2305.13655.285 286    This model inherits from [`StableDiffusionPipeline`] and aims at implementing the pipeline with minimal modifications. Check the superclass documentation for the generic methods287    implemented for all pipelines (downloading, saving, running on a particular device, etc.).288 289    This is a simplified implementation that does not perform latent or attention transfer from single object generation to overall generation. The final image is generated directly with attention and adapters control.290 291    Args:292        vae ([`AutoencoderKL`]):293            Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.294        text_encoder ([`~transformers.CLIPTextModel`]):295            Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).296        tokenizer ([`~transformers.CLIPTokenizer`]):297            A `CLIPTokenizer` to tokenize text.298        unet ([`UNet2DConditionModel`]):299            A `UNet2DConditionModel` to denoise the encoded image latents.300        scheduler ([`SchedulerMixin`]):301            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of302            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].303        safety_checker ([`StableDiffusionSafetyChecker`]):304            Classification module that estimates whether generated images could be considered offensive or harmful.305            Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details306            about a model's potential harms.307        feature_extractor ([`~transformers.CLIPImageProcessor`]):308            A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`.309        requires_safety_checker (bool):310            Whether a safety checker is needed for this pipeline.311    """312 313    model_cpu_offload_seq = "text_encoder->unet->vae"314    _optional_components = ["safety_checker", "feature_extractor", "image_encoder"]315    _exclude_from_cpu_offload = ["safety_checker"]316    _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]317 318    objects_text = "Objects: "319    bg_prompt_text = "Background prompt: "320    bg_prompt_text_no_trailing_space = bg_prompt_text.rstrip()321    neg_prompt_text = "Negative prompt: "322    neg_prompt_text_no_trailing_space = neg_prompt_text.rstrip()323 324    def __init__(325        self,326        vae: AutoencoderKL,327        text_encoder: CLIPTextModel,328        tokenizer: CLIPTokenizer,329        unet: UNet2DConditionModel,330        scheduler: KarrasDiffusionSchedulers,331        safety_checker: StableDiffusionSafetyChecker,332        feature_extractor: CLIPImageProcessor,333        image_encoder: CLIPVisionModelWithProjection = None,334        requires_safety_checker: bool = True,335    ):336        # This is copied from StableDiffusionPipeline, with hook initizations for LMD+.337        super().__init__()338 339        if scheduler is not None and getattr(scheduler.config, "steps_offset", 1) != 1:340            deprecation_message = (341                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"342                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "343                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"344                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"345                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"346                " file"347            )348            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)349            new_config = dict(scheduler.config)350            new_config["steps_offset"] = 1351            scheduler._internal_dict = FrozenDict(new_config)352 353        if scheduler is not None and getattr(scheduler.config, "clip_sample", False) is True:354            deprecation_message = (355                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."356                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"357                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"358                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"359                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"360            )361            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)362            new_config = dict(scheduler.config)363            new_config["clip_sample"] = False364            scheduler._internal_dict = FrozenDict(new_config)365 366        if safety_checker is None and requires_safety_checker:367            logger.warning(368                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"369                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"370                " results in services or applications open to the public. Both the diffusers team and Hugging Face"371                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"372                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"373                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."374            )375 376        if safety_checker is not None and feature_extractor is None:377            raise ValueError(378                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"379                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."380            )381 382        is_unet_version_less_0_9_0 = (383            unet is not None384            and hasattr(unet.config, "_diffusers_version")385            and version.parse(version.parse(unet.config._diffusers_version).base_version) < version.parse("0.9.0.dev0")386        )387        is_unet_sample_size_less_64 = (388            unet is not None and hasattr(unet.config, "sample_size") and unet.config.sample_size < 64389        )390        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:391            deprecation_message = (392                "The configuration file of the unet has set the default `sample_size` to smaller than"393                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"394                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"395                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"396                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"397                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"398                " in the config might lead to incorrect results in future versions. If you have downloaded this"399                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"400                " the `unet/config.json` file"401            )402            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)403            new_config = dict(unet.config)404            new_config["sample_size"] = 64405            unet._internal_dict = FrozenDict(new_config)406 407        self.register_modules(408            vae=vae,409            text_encoder=text_encoder,410            tokenizer=tokenizer,411            unet=unet,412            scheduler=scheduler,413            safety_checker=safety_checker,414            feature_extractor=feature_extractor,415            image_encoder=image_encoder,416        )417        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8418        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)419        self.register_to_config(requires_safety_checker=requires_safety_checker)420 421        # Initialize the attention hooks for LLM-grounded Diffusion422        self.register_attn_hooks(unet)423        self._saved_attn = None424 425    def attn_hook(self, name, query, key, value, attention_probs):426        if name in DEFAULT_GUIDANCE_ATTN_KEYS:427            self._saved_attn[name] = attention_probs428 429    @classmethod430    def convert_box(cls, box, height, width):431        # box: x, y, w, h (in 512 format) -> x_min, y_min, x_max, y_max432        x_min, y_min = box[0] / width, box[1] / height433        w_box, h_box = box[2] / width, box[3] / height434 435        x_max, y_max = x_min + w_box, y_min + h_box436 437        return x_min, y_min, x_max, y_max438 439    @classmethod440    def _parse_response_with_negative(cls, text):441        if not text:442            raise ValueError("LLM response is empty")443 444        if cls.objects_text in text:445            text = text.split(cls.objects_text)[1]446 447        text_split = text.split(cls.bg_prompt_text_no_trailing_space)448        if len(text_split) == 2:449            gen_boxes, text_rem = text_split450        else:451            raise ValueError(f"LLM response is incomplete: {text}")452 453        text_split = text_rem.split(cls.neg_prompt_text_no_trailing_space)454 455        if len(text_split) == 2:456            bg_prompt, neg_prompt = text_split457        else:458            raise ValueError(f"LLM response is incomplete: {text}")459 460        try:461            gen_boxes = ast.literal_eval(gen_boxes)462        except SyntaxError as e:463            # Sometimes the response is in plain text464            if "No objects" in gen_boxes or gen_boxes.strip() == "":465                gen_boxes = []466            else:467                raise e468        bg_prompt = bg_prompt.strip()469        neg_prompt = neg_prompt.strip()470 471        # LLM may return "None" to mean no negative prompt provided.472        if neg_prompt == "None":473            neg_prompt = ""474 475        return gen_boxes, bg_prompt, neg_prompt476 477    @classmethod478    def parse_llm_response(cls, response, canvas_height=512, canvas_width=512):479        # Infer from spec480        gen_boxes, bg_prompt, neg_prompt = cls._parse_response_with_negative(text=response)481 482        gen_boxes = sorted(gen_boxes, key=lambda gen_box: gen_box[0])483 484        phrases = [name for name, _ in gen_boxes]485        boxes = [cls.convert_box(box, height=canvas_height, width=canvas_width) for _, box in gen_boxes]486 487        return phrases, boxes, bg_prompt, neg_prompt488 489    def check_inputs(490        self,491        prompt,492        height,493        width,494        callback_steps,495        phrases,496        boxes,497        negative_prompt=None,498        prompt_embeds=None,499        negative_prompt_embeds=None,500        phrase_indices=None,501    ):502        if height % 8 != 0 or width % 8 != 0:503            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")504 505        if (callback_steps is None) or (506            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)507        ):508            raise ValueError(509                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"510                f" {type(callback_steps)}."511            )512 513        if prompt is not None and prompt_embeds is not None:514            raise ValueError(515                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"516                " only forward one of the two."517            )518        elif prompt is None and prompt_embeds is None:519            raise ValueError(520                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."521            )522        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):523            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")524        elif prompt is None and phrase_indices is None:525            raise ValueError("If the prompt is None, the phrase_indices cannot be None")526 527        if negative_prompt is not None and negative_prompt_embeds is not None:528            raise ValueError(529                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"530                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."531            )532 533        if prompt_embeds is not None and negative_prompt_embeds is not None:534            if prompt_embeds.shape != negative_prompt_embeds.shape:535                raise ValueError(536                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"537                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"538                    f" {negative_prompt_embeds.shape}."539                )540 541        if len(phrases) != len(boxes):542            raise ValueError(543                "length of `phrases` and `boxes` has to be same, but"544                f" got: `phrases` {len(phrases)} != `boxes` {len(boxes)}"545            )546 547    def register_attn_hooks(self, unet):548        """Registering hooks to obtain the attention maps for guidance"""549 550        attn_procs = {}551 552        for name in unet.attn_processors.keys():553            # Only obtain the queries and keys from cross-attention554            if name.endswith("attn1.processor") or name.endswith("fuser.attn.processor"):555                # Keep the same attn_processors for self-attention (no hooks for self-attention)556                attn_procs[name] = unet.attn_processors[name]557                continue558 559            cross_attention_dim = None if name.endswith("attn1.processor") else unet.config.cross_attention_dim560 561            if name.startswith("mid_block"):562                hidden_size = unet.config.block_out_channels[-1]563            elif name.startswith("up_blocks"):564                block_id = int(name[len("up_blocks.")])565                hidden_size = list(reversed(unet.config.block_out_channels))[block_id]566            elif name.startswith("down_blocks"):567                block_id = int(name[len("down_blocks.")])568                hidden_size = unet.config.block_out_channels[block_id]569 570            attn_procs[name] = AttnProcessorWithHook(571                attn_processor_key=name,572                hidden_size=hidden_size,573                cross_attention_dim=cross_attention_dim,574                hook=self.attn_hook,575                fast_attn=True,576                # Not enabled by default577                enabled=False,578            )579 580        unet.set_attn_processor(attn_procs)581 582    def enable_fuser(self, enabled=True):583        for module in self.unet.modules():584            if isinstance(module, GatedSelfAttentionDense):585                module.enabled = enabled586 587    def enable_attn_hook(self, enabled=True):588        for module in self.unet.attn_processors.values():589            if isinstance(module, AttnProcessorWithHook):590                module.enabled = enabled591 592    def get_token_map(self, prompt, padding="do_not_pad", verbose=False):593        """Get a list of mapping: prompt index to str (prompt in a list of token str)"""594        fg_prompt_tokens = self.tokenizer([prompt], padding=padding, max_length=77, return_tensors="np")595        input_ids = fg_prompt_tokens["input_ids"][0]596 597        token_map = []598        for ind, item in enumerate(input_ids.tolist()):599            token = self.tokenizer._convert_id_to_token(item)600 601            if verbose:602                logger.info(f"{ind}, {token} ({item})")603 604            token_map.append(token)605 606        return token_map607 608    def get_phrase_indices(609        self,610        prompt,611        phrases,612        token_map=None,613        add_suffix_if_not_found=False,614        verbose=False,615    ):616        for obj in phrases:617            # Suffix the prompt with object name for attention guidance if object is not in the prompt, using "|" to separate the prompt and the suffix618            if obj not in prompt:619                prompt += "| " + obj620 621        if token_map is None:622            # We allow using a pre-computed token map.623            token_map = self.get_token_map(prompt=prompt, padding="do_not_pad", verbose=verbose)624        token_map_str = " ".join(token_map)625 626        phrase_indices = []627 628        for obj in phrases:629            phrase_token_map = self.get_token_map(prompt=obj, padding="do_not_pad", verbose=verbose)630            # Remove <bos> and <eos> in substr631            phrase_token_map = phrase_token_map[1:-1]632            phrase_token_map_len = len(phrase_token_map)633            phrase_token_map_str = " ".join(phrase_token_map)634 635            if verbose:636                logger.info(637                    "Full str:",638                    token_map_str,639                    "Substr:",640                    phrase_token_map_str,641                    "Phrase:",642                    phrases,643                )644 645            # Count the number of token before substr646            # The substring comes with a trailing space that needs to be removed by minus one in the index.647            obj_first_index = len(token_map_str[: token_map_str.index(phrase_token_map_str) - 1].split(" "))648 649            obj_position = list(range(obj_first_index, obj_first_index + phrase_token_map_len))650            phrase_indices.append(obj_position)651 652        if add_suffix_if_not_found:653            return phrase_indices, prompt654 655        return phrase_indices656 657    def add_ca_loss_per_attn_map_to_loss(658        self,659        loss,660        attn_map,661        object_number,662        bboxes,663        phrase_indices,664        fg_top_p=0.2,665        bg_top_p=0.2,666        fg_weight=1.0,667        bg_weight=1.0,668    ):669        # b is the number of heads, not batch670        b, i, j = attn_map.shape671        H = W = int(math.sqrt(i))672        for obj_idx in range(object_number):673            obj_loss = 0674            mask = torch.zeros(size=(H, W), device="cuda")675            obj_boxes = bboxes[obj_idx]676 677            # We support two level (one box per phrase) and three level (multiple boxes per phrase)678            if not isinstance(obj_boxes[0], Iterable):679                obj_boxes = [obj_boxes]680 681            for obj_box in obj_boxes:682                # x_min, y_min, x_max, y_max = int(obj_box[0] * W), int(obj_box[1] * H), int(obj_box[2] * W), int(obj_box[3] * H)683                x_min, y_min, x_max, y_max = scale_proportion(obj_box, H=H, W=W)684                mask[y_min:y_max, x_min:x_max] = 1685 686            for obj_position in phrase_indices[obj_idx]:687                # Could potentially optimize to compute this for loop in batch.688                # Could crop the ref cross attention before saving to save memory.689 690                ca_map_obj = attn_map[:, :, obj_position].reshape(b, H, W)691 692                # shape: (b, H * W)693                ca_map_obj = attn_map[:, :, obj_position]  # .reshape(b, H, W)694                k_fg = (mask.sum() * fg_top_p).long().clamp_(min=1)695                k_bg = ((1 - mask).sum() * bg_top_p).long().clamp_(min=1)696 697                mask_1d = mask.view(1, -1)698 699                # Max-based loss function700 701                # Take the topk over spatial dimension, and then take the sum over heads dim702                # The mean is over k_fg and k_bg dimension, so we don't need to sum and divide on our own.703                obj_loss += (1 - (ca_map_obj * mask_1d).topk(k=k_fg).values.mean(dim=1)).sum(dim=0) * fg_weight704                obj_loss += ((ca_map_obj * (1 - mask_1d)).topk(k=k_bg).values.mean(dim=1)).sum(dim=0) * bg_weight705 706            loss += obj_loss / len(phrase_indices[obj_idx])707 708        return loss709 710    def compute_ca_loss(711        self,712        saved_attn,713        bboxes,714        phrase_indices,715        guidance_attn_keys,716        verbose=False,717        **kwargs,718    ):719        """720        The `saved_attn` is supposed to be passed to `save_attn_to_dict` in `cross_attention_kwargs` prior to computing ths loss.721        `AttnProcessor` will put attention maps into the `save_attn_to_dict`.722 723        `index` is the timestep.724        `ref_ca_word_token_only`: This has precedence over `ref_ca_last_token_only` (i.e., if both are enabled, we take the token from word rather than the last token).725        `ref_ca_last_token_only`: `ref_ca_saved_attn` comes from the attention map of the last token of the phrase in single object generation, so we apply it only to the last token of the phrase in overall generation if this is set to True. If set to False, `ref_ca_saved_attn` will be applied to all the text tokens.726        """727        loss = torch.tensor(0).float().cuda()728        object_number = len(bboxes)729        if object_number == 0:730            return loss731 732        for attn_key in guidance_attn_keys:733            # We only have 1 cross attention for mid.734 735            attn_map_integrated = saved_attn[attn_key]736            if not attn_map_integrated.is_cuda:737                attn_map_integrated = attn_map_integrated.cuda()738            # Example dimension: [20, 64, 77]739            attn_map = attn_map_integrated.squeeze(dim=0)740 741            loss = self.add_ca_loss_per_attn_map_to_loss(742                loss, attn_map, object_number, bboxes, phrase_indices, **kwargs743            )744 745        num_attn = len(guidance_attn_keys)746 747        if num_attn > 0:748            loss = loss / (object_number * num_attn)749 750        return loss751 752    @torch.no_grad()753    @replace_example_docstring(EXAMPLE_DOC_STRING)754    def __call__(755        self,756        prompt: Union[str, List[str]] = None,757        height: Optional[int] = None,758        width: Optional[int] = None,759        num_inference_steps: int = 50,760        guidance_scale: float = 7.5,761        gligen_scheduled_sampling_beta: float = 0.3,762        phrases: List[str] = None,763        boxes: List[List[float]] = None,764        negative_prompt: Optional[Union[str, List[str]]] = None,765        num_images_per_prompt: Optional[int] = 1,766        eta: float = 0.0,767        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,768        latents: Optional[torch.Tensor] = None,769        prompt_embeds: Optional[torch.Tensor] = None,770        negative_prompt_embeds: Optional[torch.Tensor] = None,771        ip_adapter_image: Optional[PipelineImageInput] = None,772        output_type: Optional[str] = "pil",773        return_dict: bool = True,774        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,775        callback_steps: int = 1,776        cross_attention_kwargs: Optional[Dict[str, Any]] = None,777        clip_skip: Optional[int] = None,778        lmd_guidance_kwargs: Optional[Dict[str, Any]] = {},779        phrase_indices: Optional[List[int]] = None,780    ):781        r"""782        The call function to the pipeline for generation.783 784        Args:785            prompt (`str` or `List[str]`, *optional*):786                The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.787            height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):788                The height in pixels of the generated image.789            width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):790                The width in pixels of the generated image.791            num_inference_steps (`int`, *optional*, defaults to 50):792                The number of denoising steps. More denoising steps usually lead to a higher quality image at the793                expense of slower inference.794            guidance_scale (`float`, *optional*, defaults to 7.5):795                A higher guidance scale value encourages the model to generate images closely linked to the text796                `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.797            phrases (`List[str]`):798                The phrases to guide what to include in each of the regions defined by the corresponding799                `boxes`. There should only be one phrase per bounding box.800            boxes (`List[List[float]]`):801                The bounding boxes that identify rectangular regions of the image that are going to be filled with the802                content described by the corresponding `phrases`. Each rectangular box is defined as a803                `List[float]` of 4 elements `[xmin, ymin, xmax, ymax]` where each value is between [0,1].804            gligen_scheduled_sampling_beta (`float`, defaults to 0.3):805                Scheduled Sampling factor from [GLIGEN: Open-Set Grounded Text-to-Image806                Generation](https://huggingface.co/papers/2301.07093). Scheduled Sampling factor is only varied for807                scheduled sampling during inference for improved quality and controllability.808            negative_prompt (`str` or `List[str]`, *optional*):809                The prompt or prompts to guide what to not include in image generation. If not defined, you need to810                pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).811            num_images_per_prompt (`int`, *optional*, defaults to 1):812                The number of images to generate per prompt.813            eta (`float`, *optional*, defaults to 0.0):814                Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies815                to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.816            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):817                A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make818                generation deterministic.819            latents (`torch.Tensor`, *optional*):820                Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image821                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents822                tensor is generated by sampling using the supplied random `generator`.823            prompt_embeds (`torch.Tensor`, *optional*):824                Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not825                provided, text embeddings are generated from the `prompt` input argument.826            negative_prompt_embeds (`torch.Tensor`, *optional*):827                Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If828                not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.829            ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.830            output_type (`str`, *optional*, defaults to `"pil"`):831                The output format of the generated image. Choose between `PIL.Image` or `np.array`.832            return_dict (`bool`, *optional*, defaults to `True`):833                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a834                plain tuple.835            callback (`Callable`, *optional*):836                A function that calls every `callback_steps` steps during inference. The function is called with the837                following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.838            callback_steps (`int`, *optional*, defaults to 1):839                The frequency at which the `callback` function is called. If not specified, the callback is called at840                every step.841            cross_attention_kwargs (`dict`, *optional*):842                A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in843                [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).844            guidance_rescale (`float`, *optional*, defaults to 0.0):845                Guidance rescale factor from [Common Diffusion Noise Schedules and Sample Steps are846                Flawed](https://huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when847                using zero terminal SNR.848            clip_skip (`int`, *optional*):849                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that850                the output of the pre-final layer will be used for computing the prompt embeddings.851            lmd_guidance_kwargs (`dict`, *optional*):852                A kwargs dictionary that if specified is passed along to `latent_lmd_guidance` function. Useful keys include `loss_scale` (the guidance strength), `loss_threshold` (when loss is lower than this value, the guidance is not applied anymore), `max_iter` (the number of iterations of guidance for each step), and `guidance_timesteps` (the number of diffusion timesteps to apply guidance on). See `latent_lmd_guidance` for implementation details.853            phrase_indices (`list` of `list`, *optional*): The indices of the tokens of each phrase in the overall prompt. If omitted, the pipeline will match the first token subsequence. The pipeline will append the missing phrases to the end of the prompt by default.854        Examples:855 856        Returns:857            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:858                If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned,859                otherwise a `tuple` is returned where the first element is a list with the generated images and the860                second element is a list of `bool`s indicating whether the corresponding generated image contains861                "not-safe-for-work" (nsfw) content.862        """863        # 0. Default height and width to unet864        height = height or self.unet.config.sample_size * self.vae_scale_factor865        width = width or self.unet.config.sample_size * self.vae_scale_factor866 867        # 1. Check inputs. Raise error if not correct868        self.check_inputs(869            prompt,870            height,871            width,872            callback_steps,873            phrases,874            boxes,875            negative_prompt,876            prompt_embeds,877            negative_prompt_embeds,878            phrase_indices,879        )880 881        # 2. Define call parameters882        if prompt is not None and isinstance(prompt, str):883            batch_size = 1884            if phrase_indices is None:885                phrase_indices, prompt = self.get_phrase_indices(prompt, phrases, add_suffix_if_not_found=True)886        elif prompt is not None and isinstance(prompt, list):887            batch_size = len(prompt)888            if phrase_indices is None:889                phrase_indices = []890                prompt_parsed = []891                for prompt_item in prompt:892                    (893                        phrase_indices_parsed_item,894                        prompt_parsed_item,895                    ) = self.get_phrase_indices(prompt_item, add_suffix_if_not_found=True)896                    phrase_indices.append(phrase_indices_parsed_item)897                    prompt_parsed.append(prompt_parsed_item)898                prompt = prompt_parsed899        else:900            batch_size = prompt_embeds.shape[0]901 902        device = self._execution_device903        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)904        # of the Imagen paper: https://huggingface.co/papers/2205.11487 . `guidance_scale = 1`905        # corresponds to doing no classifier free guidance.906        do_classifier_free_guidance = guidance_scale > 1.0907 908        # 3. Encode input prompt909        prompt_embeds, negative_prompt_embeds = self.encode_prompt(910            prompt,911            device,912            num_images_per_prompt,913            do_classifier_free_guidance,914            negative_prompt,915            prompt_embeds=prompt_embeds,916            negative_prompt_embeds=negative_prompt_embeds,917            clip_skip=clip_skip,918        )919 920        cond_prompt_embeds = prompt_embeds921 922        # For classifier free guidance, we need to do two forward passes.923        # Here we concatenate the unconditional and text embeddings into a single batch924        # to avoid doing two forward passes925        if do_classifier_free_guidance:926            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])927 928        if ip_adapter_image is not None:929            image_embeds, negative_image_embeds = self.encode_image(ip_adapter_image, device, num_images_per_prompt)930            if self.do_classifier_free_guidance:931                image_embeds = torch.cat([negative_image_embeds, image_embeds])932 933        # 4. Prepare timesteps934        self.scheduler.set_timesteps(num_inference_steps, device=device)935        timesteps = self.scheduler.timesteps936 937        # 5. Prepare latent variables938        num_channels_latents = self.unet.config.in_channels939        latents = self.prepare_latents(940            batch_size * num_images_per_prompt,941            num_channels_latents,942            height,943            width,944            prompt_embeds.dtype,945            device,946            generator,947            latents,948        )949 950        # 5.1 Prepare GLIGEN variables951        max_objs = 30952        if len(boxes) > max_objs:953            warnings.warn(954                f"More that {max_objs} objects found. Only first {max_objs} objects will be processed.",955                FutureWarning,956            )957            phrases = phrases[:max_objs]958            boxes = boxes[:max_objs]959 960        n_objs = len(boxes)961        if n_objs:962            # prepare batched input to the PositionNet (boxes, phrases, mask)963            # Get tokens for phrases from pre-trained CLIPTokenizer964            tokenizer_inputs = self.tokenizer(phrases, padding=True, return_tensors="pt").to(device)965            # For the token, we use the same pre-trained text encoder966            # to obtain its text feature967            _text_embeddings = self.text_encoder(**tokenizer_inputs).pooler_output968 969        # For each entity, described in phrases, is denoted with a bounding box,970        # we represent the location information as (xmin,ymin,xmax,ymax)971        cond_boxes = torch.zeros(max_objs, 4, device=device, dtype=self.text_encoder.dtype)972        if n_objs:973            cond_boxes[:n_objs] = torch.tensor(boxes)974        text_embeddings = torch.zeros(975            max_objs,976            self.unet.config.cross_attention_dim,977            device=device,978            dtype=self.text_encoder.dtype,979        )980        if n_objs:981            text_embeddings[:n_objs] = _text_embeddings982        # Generate a mask for each object that is entity described by phrases983        masks = torch.zeros(max_objs, device=device, dtype=self.text_encoder.dtype)984        masks[:n_objs] = 1985 986        repeat_batch = batch_size * num_images_per_prompt987        cond_boxes = cond_boxes.unsqueeze(0).expand(repeat_batch, -1, -1).clone()988        text_embeddings = text_embeddings.unsqueeze(0).expand(repeat_batch, -1, -1).clone()989        masks = masks.unsqueeze(0).expand(repeat_batch, -1).clone()990        if do_classifier_free_guidance:991            repeat_batch = repeat_batch * 2992            cond_boxes = torch.cat([cond_boxes] * 2)993            text_embeddings = torch.cat([text_embeddings] * 2)994            masks = torch.cat([masks] * 2)995            masks[: repeat_batch // 2] = 0996        if cross_attention_kwargs is None:997            cross_attention_kwargs = {}998        cross_attention_kwargs["gligen"] = {999            "boxes": cond_boxes,1000            "positive_embeddings": text_embeddings,1001            "masks": masks,1002        }1003 1004        num_grounding_steps = int(gligen_scheduled_sampling_beta * len(timesteps))1005        self.enable_fuser(True)1006 1007        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline1008        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)1009 1010        # 6.1 Add image embeds for IP-Adapter1011        added_cond_kwargs = {"image_embeds": image_embeds} if ip_adapter_image is not None else None1012 1013        loss_attn = torch.tensor(10000.0)1014 1015        # 7. Denoising loop1016        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order1017        with self.progress_bar(total=num_inference_steps) as progress_bar:1018            for i, t in enumerate(timesteps):1019                # Scheduled sampling1020                if i == num_grounding_steps:1021                    self.enable_fuser(False)1022 1023                if latents.shape[1] != 4:1024                    latents = torch.randn_like(latents[:, :4])1025 1026                # 7.1 Perform LMD guidance1027                if boxes:1028                    latents, loss_attn = self.latent_lmd_guidance(1029                        cond_prompt_embeds,1030                        index=i,1031                        boxes=boxes,1032                        phrase_indices=phrase_indices,1033                        t=t,1034                        latents=latents,1035                        loss=loss_attn,1036                        **lmd_guidance_kwargs,1037                    )1038 1039                # expand the latents if we are doing classifier free guidance1040                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents1041                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)1042 1043                # predict the noise residual1044                noise_pred = self.unet(1045                    latent_model_input,1046                    t,1047                    encoder_hidden_states=prompt_embeds,1048                    cross_attention_kwargs=cross_attention_kwargs,1049                    added_cond_kwargs=added_cond_kwargs,1050                ).sample1051 1052                # perform guidance1053                if do_classifier_free_guidance:1054                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)1055                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)1056 1057                # compute the previous noisy sample x_t -> x_t-11058                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample1059 1060                # call the callback, if provided1061                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):1062                    progress_bar.update()1063                    if callback is not None and i % callback_steps == 0:1064                        step_idx = i // getattr(self.scheduler, "order", 1)1065                        callback(step_idx, t, latents)1066 1067        if not output_type == "latent":1068            image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]1069            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)1070        else:1071            image = latents1072            has_nsfw_concept = None1073 1074        if has_nsfw_concept is None:1075            do_denormalize = [True] * image.shape[0]1076        else:1077            do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept]1078 1079        image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)1080 1081        # Offload last model to CPU1082        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:1083            self.final_offload_hook.offload()1084 1085        if not return_dict:1086            return (image, has_nsfw_concept)1087 1088        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)1089 1090    @torch.set_grad_enabled(True)1091    def latent_lmd_guidance(1092        self,1093        cond_embeddings,1094        index,1095        boxes,1096        phrase_indices,1097        t,1098        latents,1099        loss,1100        *,1101        loss_scale=20,1102        loss_threshold=5.0,1103        max_iter=[3] * 5 + [2] * 5 + [1] * 5,1104        guidance_timesteps=15,1105        cross_attention_kwargs=None,1106        guidance_attn_keys=DEFAULT_GUIDANCE_ATTN_KEYS,1107        verbose=False,1108        clear_cache=False,1109        unet_additional_kwargs={},1110        guidance_callback=None,1111        **kwargs,1112    ):1113        scheduler, unet = self.scheduler, self.unet1114 1115        iteration = 01116 1117        if index < guidance_timesteps:1118            if isinstance(max_iter, list):1119                max_iter = max_iter[index]1120 1121            if verbose:1122                logger.info(1123                    f"time index {index}, loss: {loss.item() / loss_scale:.3f} (de-scaled with scale {loss_scale:.1f}), loss threshold: {loss_threshold:.3f}"1124                )1125 1126            try:1127                self.enable_attn_hook(enabled=True)1128 1129                while (1130                    loss.item() / loss_scale > loss_threshold and iteration < max_iter and index < guidance_timesteps1131                ):1132                    self._saved_attn = {}1133 1134                    latents.requires_grad_(True)1135                    latent_model_input = latents1136                    latent_model_input = scheduler.scale_model_input(latent_model_input, t)1137 1138                    unet(1139                        latent_model_input,1140                        t,1141                        encoder_hidden_states=cond_embeddings,1142                        cross_attention_kwargs=cross_attention_kwargs,1143                        **unet_additional_kwargs,1144                    )1145 1146                    # update latents with guidance1147                    loss = (1148                        self.compute_ca_loss(1149                            saved_attn=self._saved_attn,1150                            bboxes=boxes,1151                            phrase_indices=phrase_indices,1152                            guidance_attn_keys=guidance_attn_keys,1153                            verbose=verbose,1154                            **kwargs,1155                        )1156                        * loss_scale1157                    )1158 1159                    if torch.isnan(loss):1160                        raise RuntimeError("**Loss is NaN**")1161 1162                    # This callback allows visualizations.1163                    if guidance_callback is not None:1164                        guidance_callback(self, latents, loss, iteration, index)1165 1166                    self._saved_attn = None1167 1168                    grad_cond = torch.autograd.grad(loss.requires_grad_(True), [latents])[0]1169 1170                    latents.requires_grad_(False)1171 1172                    # Scaling with classifier guidance1173                    alpha_prod_t = scheduler.alphas_cumprod[t]1174                    # Classifier guidance: https://huggingface.co/papers/2105.052331175                    # DDIM: https://huggingface.co/papers/2010.025021176                    scale = (1 - alpha_prod_t) ** (0.5)1177                    latents = latents - scale * grad_cond1178 1179                    iteration += 11180 1181                    if clear_cache:1182                        gc.collect()1183                        torch.cuda.empty_cache()1184 1185                    if verbose:1186                        logger.info(1187                            f"time index {index}, loss: {loss.item() / loss_scale:.3f}, loss threshold: {loss_threshold:.3f}, iteration: {iteration}"1188                        )1189 1190            finally:1191                self.enable_attn_hook(enabled=False)1192 1193        return latents, loss1194 1195    # Below are methods copied from StableDiffusionPipeline1196    # The design choice of not inheriting from StableDiffusionPipeline is discussed here: https://github.com/huggingface/diffusers/pull/5993#issuecomment-18342585171197 1198    # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt1199    def _encode_prompt(1200        self,

Showing the first 1,200 of 1568 lines. Download the file for the rest.