diffusers/community-pipelines-mirror

1# Copyright 2023 Long Lian, the GLIGEN Authors, and The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15# This is a single file implementation of LMD+. See README.md for examples.16 17import ast18import gc19import inspect20import math21import warnings22from collections.abc import Iterable23from typing import Any, Callable, Dict, List, Optional, Union24 25import torch26import torch.nn.functional as F27from packaging import version28from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection29 30from diffusers.configuration_utils import FrozenDict31from diffusers.image_processor import PipelineImageInput, VaeImageProcessor32from diffusers.loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin33from diffusers.models import AutoencoderKL, UNet2DConditionModel34from diffusers.models.attention import Attention, GatedSelfAttentionDense35from diffusers.models.attention_processor import AttnProcessor2_036from diffusers.models.lora import adjust_lora_scale_text_encoder37from diffusers.pipelines import DiffusionPipeline38from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput39from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker40from diffusers.schedulers import KarrasDiffusionSchedulers41from diffusers.utils import (42    USE_PEFT_BACKEND,43    deprecate,44    logging,45    replace_example_docstring,46    scale_lora_layers,47    unscale_lora_layers,48)49from diffusers.utils.torch_utils import randn_tensor50 51 52EXAMPLE_DOC_STRING = """53    Examples:54        ```py55        >>> import torch56        >>> from diffusers import DiffusionPipeline57 58        >>> pipe = DiffusionPipeline.from_pretrained(59        ...     "longlian/lmd_plus",60        ...     custom_pipeline="llm_grounded_diffusion",61        ...     custom_revision="main",62        ...     variant="fp16", torch_dtype=torch.float1663        ... )64        >>> pipe.enable_model_cpu_offload()65 66        >>> # Generate an image described by the prompt and67        >>> # insert objects described by text at the region defined by bounding boxes68        >>> prompt = "a waterfall and a modern high speed train in a beautiful forest with fall foliage"69        >>> boxes = [[0.1387, 0.2051, 0.4277, 0.7090], [0.4980, 0.4355, 0.8516, 0.7266]]70        >>> phrases = ["a waterfall", "a modern high speed train"]71 72        >>> images = pipe(73        ...     prompt=prompt,74        ...     phrases=phrases,75        ...     boxes=boxes,76        ...     gligen_scheduled_sampling_beta=0.4,77        ...     output_type="pil",78        ...     num_inference_steps=50,79        ...     lmd_guidance_kwargs={}80        ... ).images81 82        >>> images[0].save("./lmd_plus_generation.jpg")83 84        >>> # Generate directly from a text prompt and an LLM response85        >>> prompt = "a waterfall and a modern high speed train in a beautiful forest with fall foliage"86        >>> phrases, boxes, bg_prompt, neg_prompt = pipe.parse_llm_response(\"""87        [('a waterfall', [71, 105, 148, 258]), ('a modern high speed train', [255, 223, 181, 149])]88        Background prompt: A beautiful forest with fall foliage89        Negative prompt:90        \""")91 92        >> images = pipe(93        ...     prompt=prompt,94        ...     negative_prompt=neg_prompt,95        ...     phrases=phrases,96        ...     boxes=boxes,97        ...     gligen_scheduled_sampling_beta=0.4,98        ...     output_type="pil",99        ...     num_inference_steps=50,100        ...     lmd_guidance_kwargs={}101        ... ).images102 103        >>> images[0].save("./lmd_plus_generation.jpg")104 105images[0]106 107        ```108"""109 110logger = logging.get_logger(__name__)  # pylint: disable=invalid-name111 112# All keys in Stable Diffusion models: [('down', 0, 0, 0), ('down', 0, 1, 0), ('down', 1, 0, 0), ('down', 1, 1, 0), ('down', 2, 0, 0), ('down', 2, 1, 0), ('mid', 0, 0, 0), ('up', 1, 0, 0), ('up', 1, 1, 0), ('up', 1, 2, 0), ('up', 2, 0, 0), ('up', 2, 1, 0), ('up', 2, 2, 0), ('up', 3, 0, 0), ('up', 3, 1, 0), ('up', 3, 2, 0)]113# Note that the first up block is `UpBlock2D` rather than `CrossAttnUpBlock2D` and does not have attention. The last index is always 0 in our case since we have one `BasicTransformerBlock` in each `Transformer2DModel`.114DEFAULT_GUIDANCE_ATTN_KEYS = [115    ("mid", 0, 0, 0),116    ("up", 1, 0, 0),117    ("up", 1, 1, 0),118    ("up", 1, 2, 0),119]120 121 122def convert_attn_keys(key):123    """Convert the attention key from tuple format to the torch state format"""124 125    if key[0] == "mid":126        assert key[1] == 0, f"mid block only has one block but the index is {key[1]}"127        return f"{key[0]}_block.attentions.{key[2]}.transformer_blocks.{key[3]}.attn2.processor"128 129    return f"{key[0]}_blocks.{key[1]}.attentions.{key[2]}.transformer_blocks.{key[3]}.attn2.processor"130 131 132DEFAULT_GUIDANCE_ATTN_KEYS = [convert_attn_keys(key) for key in DEFAULT_GUIDANCE_ATTN_KEYS]133 134 135def scale_proportion(obj_box, H, W):136    # Separately rounding box_w and box_h to allow shift invariant box sizes. Otherwise box sizes may change when both coordinates being rounded end with ".5".137    x_min, y_min = round(obj_box[0] * W), round(obj_box[1] * H)138    box_w, box_h = round((obj_box[2] - obj_box[0]) * W), round((obj_box[3] - obj_box[1]) * H)139    x_max, y_max = x_min + box_w, y_min + box_h140 141    x_min, y_min = max(x_min, 0), max(y_min, 0)142    x_max, y_max = min(x_max, W), min(y_max, H)143 144    return x_min, y_min, x_max, y_max145 146 147# Adapted from the parent class `AttnProcessor2_0`148class AttnProcessorWithHook(AttnProcessor2_0):149    def __init__(150        self,151        attn_processor_key,152        hidden_size,153        cross_attention_dim,154        hook=None,155        fast_attn=True,156        enabled=True,157    ):158        super().__init__()159        self.attn_processor_key = attn_processor_key160        self.hidden_size = hidden_size161        self.cross_attention_dim = cross_attention_dim162        self.hook = hook163        self.fast_attn = fast_attn164        self.enabled = enabled165 166    def __call__(167        self,168        attn: Attention,169        hidden_states,170        encoder_hidden_states=None,171        attention_mask=None,172        temb=None,173        scale: float = 1.0,174    ):175        residual = hidden_states176 177        if attn.spatial_norm is not None:178            hidden_states = attn.spatial_norm(hidden_states, temb)179 180        input_ndim = hidden_states.ndim181 182        if input_ndim == 4:183            batch_size, channel, height, width = hidden_states.shape184            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)185 186        batch_size, sequence_length, _ = (187            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape188        )189 190        if attention_mask is not None:191            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)192 193        if attn.group_norm is not None:194            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)195 196        args = () if USE_PEFT_BACKEND else (scale,)197        query = attn.to_q(hidden_states, *args)198 199        if encoder_hidden_states is None:200            encoder_hidden_states = hidden_states201        elif attn.norm_cross:202            encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)203 204        key = attn.to_k(encoder_hidden_states, *args)205        value = attn.to_v(encoder_hidden_states, *args)206 207        inner_dim = key.shape[-1]208        head_dim = inner_dim // attn.heads209 210        if (self.hook is not None and self.enabled) or not self.fast_attn:211            query_batch_dim = attn.head_to_batch_dim(query)212            key_batch_dim = attn.head_to_batch_dim(key)213            value_batch_dim = attn.head_to_batch_dim(value)214            attention_probs = attn.get_attention_scores(query_batch_dim, key_batch_dim, attention_mask)215 216        if self.hook is not None and self.enabled:217            # Call the hook with query, key, value, and attention maps218            self.hook(219                self.attn_processor_key,220                query_batch_dim,221                key_batch_dim,222                value_batch_dim,223                attention_probs,224            )225 226        if self.fast_attn:227            query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)228 229            key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)230            value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)231 232            if attention_mask is not None:233                # scaled_dot_product_attention expects attention_mask shape to be234                # (batch, heads, source_length, target_length)235                attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])236 237            # the output of sdp = (batch, num_heads, seq_len, head_dim)238            # TODO: add support for attn.scale when we move to Torch 2.1239            hidden_states = F.scaled_dot_product_attention(240                query,241                key,242                value,243                attn_mask=attention_mask,244                dropout_p=0.0,245                is_causal=False,246            )247            hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)248            hidden_states = hidden_states.to(query.dtype)249        else:250            hidden_states = torch.bmm(attention_probs, value)251            hidden_states = attn.batch_to_head_dim(hidden_states)252 253        # linear proj254        hidden_states = attn.to_out[0](hidden_states, *args)255        # dropout256        hidden_states = attn.to_out[1](hidden_states)257 258        if input_ndim == 4:259            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)260 261        if attn.residual_connection:262            hidden_states = hidden_states + residual263 264        hidden_states = hidden_states / attn.rescale_output_factor265 266        return hidden_states267 268 269class LLMGroundedDiffusionPipeline(270    DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin, IPAdapterMixin, FromSingleFileMixin271):272    r"""273    Pipeline for layout-grounded text-to-image generation using LLM-grounded Diffusion (LMD+): https://arxiv.org/pdf/2305.13655.pdf.274 275    This model inherits from [`StableDiffusionPipeline`] and aims at implementing the pipeline with minimal modifications. Check the superclass documentation for the generic methods276    implemented for all pipelines (downloading, saving, running on a particular device, etc.).277 278    This is a simplified implementation that does not perform latent or attention transfer from single object generation to overall generation. The final image is generated directly with attention and adapters control.279 280    Args:281        vae ([`AutoencoderKL`]):282            Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.283        text_encoder ([`~transformers.CLIPTextModel`]):284            Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).285        tokenizer ([`~transformers.CLIPTokenizer`]):286            A `CLIPTokenizer` to tokenize text.287        unet ([`UNet2DConditionModel`]):288            A `UNet2DConditionModel` to denoise the encoded image latents.289        scheduler ([`SchedulerMixin`]):290            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of291            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].292        safety_checker ([`StableDiffusionSafetyChecker`]):293            Classification module that estimates whether generated images could be considered offensive or harmful.294            Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details295            about a model's potential harms.296        feature_extractor ([`~transformers.CLIPImageProcessor`]):297            A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`.298        requires_safety_checker (bool):299            Whether a safety checker is needed for this pipeline.300    """301 302    model_cpu_offload_seq = "text_encoder->unet->vae"303    _optional_components = ["safety_checker", "feature_extractor", "image_encoder"]304    _exclude_from_cpu_offload = ["safety_checker"]305    _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]306 307    objects_text = "Objects: "308    bg_prompt_text = "Background prompt: "309    bg_prompt_text_no_trailing_space = bg_prompt_text.rstrip()310    neg_prompt_text = "Negative prompt: "311    neg_prompt_text_no_trailing_space = neg_prompt_text.rstrip()312 313    def __init__(314        self,315        vae: AutoencoderKL,316        text_encoder: CLIPTextModel,317        tokenizer: CLIPTokenizer,318        unet: UNet2DConditionModel,319        scheduler: KarrasDiffusionSchedulers,320        safety_checker: StableDiffusionSafetyChecker,321        feature_extractor: CLIPImageProcessor,322        image_encoder: CLIPVisionModelWithProjection = None,323        requires_safety_checker: bool = True,324    ):325        # This is copied from StableDiffusionPipeline, with hook initizations for LMD+.326        super().__init__()327 328        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:329            deprecation_message = (330                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"331                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "332                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"333                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"334                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"335                " file"336            )337            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)338            new_config = dict(scheduler.config)339            new_config["steps_offset"] = 1340            scheduler._internal_dict = FrozenDict(new_config)341 342        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:343            deprecation_message = (344                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."345                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"346                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"347                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"348                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"349            )350            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)351            new_config = dict(scheduler.config)352            new_config["clip_sample"] = False353            scheduler._internal_dict = FrozenDict(new_config)354 355        if safety_checker is None and requires_safety_checker:356            logger.warning(357                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"358                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"359                " results in services or applications open to the public. Both the diffusers team and Hugging Face"360                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"361                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"362                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."363            )364 365        if safety_checker is not None and feature_extractor is None:366            raise ValueError(367                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"368                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."369            )370 371        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(372            version.parse(unet.config._diffusers_version).base_version373        ) < version.parse("0.9.0.dev0")374        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64375        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:376            deprecation_message = (377                "The configuration file of the unet has set the default `sample_size` to smaller than"378                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"379                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"380                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"381                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"382                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"383                " in the config might lead to incorrect results in future versions. If you have downloaded this"384                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"385                " the `unet/config.json` file"386            )387            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)388            new_config = dict(unet.config)389            new_config["sample_size"] = 64390            unet._internal_dict = FrozenDict(new_config)391 392        self.register_modules(393            vae=vae,394            text_encoder=text_encoder,395            tokenizer=tokenizer,396            unet=unet,397            scheduler=scheduler,398            safety_checker=safety_checker,399            feature_extractor=feature_extractor,400            image_encoder=image_encoder,401        )402        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)403        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)404        self.register_to_config(requires_safety_checker=requires_safety_checker)405 406        # Initialize the attention hooks for LLM-grounded Diffusion407        self.register_attn_hooks(unet)408        self._saved_attn = None409 410    def attn_hook(self, name, query, key, value, attention_probs):411        if name in DEFAULT_GUIDANCE_ATTN_KEYS:412            self._saved_attn[name] = attention_probs413 414    @classmethod415    def convert_box(cls, box, height, width):416        # box: x, y, w, h (in 512 format) -> x_min, y_min, x_max, y_max417        x_min, y_min = box[0] / width, box[1] / height418        w_box, h_box = box[2] / width, box[3] / height419 420        x_max, y_max = x_min + w_box, y_min + h_box421 422        return x_min, y_min, x_max, y_max423 424    @classmethod425    def _parse_response_with_negative(cls, text):426        if not text:427            raise ValueError("LLM response is empty")428 429        if cls.objects_text in text:430            text = text.split(cls.objects_text)[1]431 432        text_split = text.split(cls.bg_prompt_text_no_trailing_space)433        if len(text_split) == 2:434            gen_boxes, text_rem = text_split435        else:436            raise ValueError(f"LLM response is incomplete: {text}")437 438        text_split = text_rem.split(cls.neg_prompt_text_no_trailing_space)439 440        if len(text_split) == 2:441            bg_prompt, neg_prompt = text_split442        else:443            raise ValueError(f"LLM response is incomplete: {text}")444 445        try:446            gen_boxes = ast.literal_eval(gen_boxes)447        except SyntaxError as e:448            # Sometimes the response is in plain text449            if "No objects" in gen_boxes or gen_boxes.strip() == "":450                gen_boxes = []451            else:452                raise e453        bg_prompt = bg_prompt.strip()454        neg_prompt = neg_prompt.strip()455 456        # LLM may return "None" to mean no negative prompt provided.457        if neg_prompt == "None":458            neg_prompt = ""459 460        return gen_boxes, bg_prompt, neg_prompt461 462    @classmethod463    def parse_llm_response(cls, response, canvas_height=512, canvas_width=512):464        # Infer from spec465        gen_boxes, bg_prompt, neg_prompt = cls._parse_response_with_negative(text=response)466 467        gen_boxes = sorted(gen_boxes, key=lambda gen_box: gen_box[0])468 469        phrases = [name for name, _ in gen_boxes]470        boxes = [cls.convert_box(box, height=canvas_height, width=canvas_width) for _, box in gen_boxes]471 472        return phrases, boxes, bg_prompt, neg_prompt473 474    def check_inputs(475        self,476        prompt,477        height,478        width,479        callback_steps,480        phrases,481        boxes,482        negative_prompt=None,483        prompt_embeds=None,484        negative_prompt_embeds=None,485        phrase_indices=None,486    ):487        if height % 8 != 0 or width % 8 != 0:488            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")489 490        if (callback_steps is None) or (491            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)492        ):493            raise ValueError(494                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"495                f" {type(callback_steps)}."496            )497 498        if prompt is not None and prompt_embeds is not None:499            raise ValueError(500                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"501                " only forward one of the two."502            )503        elif prompt is None and prompt_embeds is None:504            raise ValueError(505                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."506            )507        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):508            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")509        elif prompt is None and phrase_indices is None:510            raise ValueError("If the prompt is None, the phrase_indices cannot be None")511 512        if negative_prompt is not None and negative_prompt_embeds is not None:513            raise ValueError(514                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"515                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."516            )517 518        if prompt_embeds is not None and negative_prompt_embeds is not None:519            if prompt_embeds.shape != negative_prompt_embeds.shape:520                raise ValueError(521                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"522                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"523                    f" {negative_prompt_embeds.shape}."524                )525 526        if len(phrases) != len(boxes):527            ValueError(528                "length of `phrases` and `boxes` has to be same, but"529                f" got: `phrases` {len(phrases)} != `boxes` {len(boxes)}"530            )531 532    def register_attn_hooks(self, unet):533        """Registering hooks to obtain the attention maps for guidance"""534 535        attn_procs = {}536 537        for name in unet.attn_processors.keys():538            # Only obtain the queries and keys from cross-attention539            if name.endswith("attn1.processor") or name.endswith("fuser.attn.processor"):540                # Keep the same attn_processors for self-attention (no hooks for self-attention)541                attn_procs[name] = unet.attn_processors[name]542                continue543 544            cross_attention_dim = None if name.endswith("attn1.processor") else unet.config.cross_attention_dim545 546            if name.startswith("mid_block"):547                hidden_size = unet.config.block_out_channels[-1]548            elif name.startswith("up_blocks"):549                block_id = int(name[len("up_blocks.")])550                hidden_size = list(reversed(unet.config.block_out_channels))[block_id]551            elif name.startswith("down_blocks"):552                block_id = int(name[len("down_blocks.")])553                hidden_size = unet.config.block_out_channels[block_id]554 555            attn_procs[name] = AttnProcessorWithHook(556                attn_processor_key=name,557                hidden_size=hidden_size,558                cross_attention_dim=cross_attention_dim,559                hook=self.attn_hook,560                fast_attn=True,561                # Not enabled by default562                enabled=False,563            )564 565        unet.set_attn_processor(attn_procs)566 567    def enable_fuser(self, enabled=True):568        for module in self.unet.modules():569            if isinstance(module, GatedSelfAttentionDense):570                module.enabled = enabled571 572    def enable_attn_hook(self, enabled=True):573        for module in self.unet.attn_processors.values():574            if isinstance(module, AttnProcessorWithHook):575                module.enabled = enabled576 577    def get_token_map(self, prompt, padding="do_not_pad", verbose=False):578        """Get a list of mapping: prompt index to str (prompt in a list of token str)"""579        fg_prompt_tokens = self.tokenizer([prompt], padding=padding, max_length=77, return_tensors="np")580        input_ids = fg_prompt_tokens["input_ids"][0]581 582        token_map = []583        for ind, item in enumerate(input_ids.tolist()):584            token = self.tokenizer._convert_id_to_token(item)585 586            if verbose:587                logger.info(f"{ind}, {token} ({item})")588 589            token_map.append(token)590 591        return token_map592 593    def get_phrase_indices(594        self,595        prompt,596        phrases,597        token_map=None,598        add_suffix_if_not_found=False,599        verbose=False,600    ):601        for obj in phrases:602            # Suffix the prompt with object name for attention guidance if object is not in the prompt, using "|" to separate the prompt and the suffix603            if obj not in prompt:604                prompt += "| " + obj605 606        if token_map is None:607            # We allow using a pre-computed token map.608            token_map = self.get_token_map(prompt=prompt, padding="do_not_pad", verbose=verbose)609        token_map_str = " ".join(token_map)610 611        phrase_indices = []612 613        for obj in phrases:614            phrase_token_map = self.get_token_map(prompt=obj, padding="do_not_pad", verbose=verbose)615            # Remove <bos> and <eos> in substr616            phrase_token_map = phrase_token_map[1:-1]617            phrase_token_map_len = len(phrase_token_map)618            phrase_token_map_str = " ".join(phrase_token_map)619 620            if verbose:621                logger.info(622                    "Full str:",623                    token_map_str,624                    "Substr:",625                    phrase_token_map_str,626                    "Phrase:",627                    phrases,628                )629 630            # Count the number of token before substr631            # The substring comes with a trailing space that needs to be removed by minus one in the index.632            obj_first_index = len(token_map_str[: token_map_str.index(phrase_token_map_str) - 1].split(" "))633 634            obj_position = list(range(obj_first_index, obj_first_index + phrase_token_map_len))635            phrase_indices.append(obj_position)636 637        if add_suffix_if_not_found:638            return phrase_indices, prompt639 640        return phrase_indices641 642    def add_ca_loss_per_attn_map_to_loss(643        self,644        loss,645        attn_map,646        object_number,647        bboxes,648        phrase_indices,649        fg_top_p=0.2,650        bg_top_p=0.2,651        fg_weight=1.0,652        bg_weight=1.0,653    ):654        # b is the number of heads, not batch655        b, i, j = attn_map.shape656        H = W = int(math.sqrt(i))657        for obj_idx in range(object_number):658            obj_loss = 0659            mask = torch.zeros(size=(H, W), device="cuda")660            obj_boxes = bboxes[obj_idx]661 662            # We support two level (one box per phrase) and three level (multiple boxes per phrase)663            if not isinstance(obj_boxes[0], Iterable):664                obj_boxes = [obj_boxes]665 666            for obj_box in obj_boxes:667                # x_min, y_min, x_max, y_max = int(obj_box[0] * W), int(obj_box[1] * H), int(obj_box[2] * W), int(obj_box[3] * H)668                x_min, y_min, x_max, y_max = scale_proportion(obj_box, H=H, W=W)669                mask[y_min:y_max, x_min:x_max] = 1670 671            for obj_position in phrase_indices[obj_idx]:672                # Could potentially optimize to compute this for loop in batch.673                # Could crop the ref cross attention before saving to save memory.674 675                ca_map_obj = attn_map[:, :, obj_position].reshape(b, H, W)676 677                # shape: (b, H * W)678                ca_map_obj = attn_map[:, :, obj_position]  # .reshape(b, H, W)679                k_fg = (mask.sum() * fg_top_p).long().clamp_(min=1)680                k_bg = ((1 - mask).sum() * bg_top_p).long().clamp_(min=1)681 682                mask_1d = mask.view(1, -1)683 684                # Max-based loss function685 686                # Take the topk over spatial dimension, and then take the sum over heads dim687                # The mean is over k_fg and k_bg dimension, so we don't need to sum and divide on our own.688                obj_loss += (1 - (ca_map_obj * mask_1d).topk(k=k_fg).values.mean(dim=1)).sum(dim=0) * fg_weight689                obj_loss += ((ca_map_obj * (1 - mask_1d)).topk(k=k_bg).values.mean(dim=1)).sum(dim=0) * bg_weight690 691            loss += obj_loss / len(phrase_indices[obj_idx])692 693        return loss694 695    def compute_ca_loss(696        self,697        saved_attn,698        bboxes,699        phrase_indices,700        guidance_attn_keys,701        verbose=False,702        **kwargs,703    ):704        """705        The `saved_attn` is supposed to be passed to `save_attn_to_dict` in `cross_attention_kwargs` prior to computing ths loss.706        `AttnProcessor` will put attention maps into the `save_attn_to_dict`.707 708        `index` is the timestep.709        `ref_ca_word_token_only`: This has precedence over `ref_ca_last_token_only` (i.e., if both are enabled, we take the token from word rather than the last token).710        `ref_ca_last_token_only`: `ref_ca_saved_attn` comes from the attention map of the last token of the phrase in single object generation, so we apply it only to the last token of the phrase in overall generation if this is set to True. If set to False, `ref_ca_saved_attn` will be applied to all the text tokens.711        """712        loss = torch.tensor(0).float().cuda()713        object_number = len(bboxes)714        if object_number == 0:715            return loss716 717        for attn_key in guidance_attn_keys:718            # We only have 1 cross attention for mid.719 720            attn_map_integrated = saved_attn[attn_key]721            if not attn_map_integrated.is_cuda:722                attn_map_integrated = attn_map_integrated.cuda()723            # Example dimension: [20, 64, 77]724            attn_map = attn_map_integrated.squeeze(dim=0)725 726            loss = self.add_ca_loss_per_attn_map_to_loss(727                loss, attn_map, object_number, bboxes, phrase_indices, **kwargs728            )729 730        num_attn = len(guidance_attn_keys)731 732        if num_attn > 0:733            loss = loss / (object_number * num_attn)734 735        return loss736 737    @torch.no_grad()738    @replace_example_docstring(EXAMPLE_DOC_STRING)739    def __call__(740        self,741        prompt: Union[str, List[str]] = None,742        height: Optional[int] = None,743        width: Optional[int] = None,744        num_inference_steps: int = 50,745        guidance_scale: float = 7.5,746        gligen_scheduled_sampling_beta: float = 0.3,747        phrases: List[str] = None,748        boxes: List[List[float]] = None,749        negative_prompt: Optional[Union[str, List[str]]] = None,750        num_images_per_prompt: Optional[int] = 1,751        eta: float = 0.0,752        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,753        latents: Optional[torch.FloatTensor] = None,754        prompt_embeds: Optional[torch.FloatTensor] = None,755        negative_prompt_embeds: Optional[torch.FloatTensor] = None,756        ip_adapter_image: Optional[PipelineImageInput] = None,757        output_type: Optional[str] = "pil",758        return_dict: bool = True,759        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,760        callback_steps: int = 1,761        cross_attention_kwargs: Optional[Dict[str, Any]] = None,762        clip_skip: Optional[int] = None,763        lmd_guidance_kwargs: Optional[Dict[str, Any]] = {},764        phrase_indices: Optional[List[int]] = None,765    ):766        r"""767        The call function to the pipeline for generation.768 769        Args:770            prompt (`str` or `List[str]`, *optional*):771                The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.772            height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):773                The height in pixels of the generated image.774            width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):775                The width in pixels of the generated image.776            num_inference_steps (`int`, *optional*, defaults to 50):777                The number of denoising steps. More denoising steps usually lead to a higher quality image at the778                expense of slower inference.779            guidance_scale (`float`, *optional*, defaults to 7.5):780                A higher guidance scale value encourages the model to generate images closely linked to the text781                `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.782            phrases (`List[str]`):783                The phrases to guide what to include in each of the regions defined by the corresponding784                `boxes`. There should only be one phrase per bounding box.785            boxes (`List[List[float]]`):786                The bounding boxes that identify rectangular regions of the image that are going to be filled with the787                content described by the corresponding `phrases`. Each rectangular box is defined as a788                `List[float]` of 4 elements `[xmin, ymin, xmax, ymax]` where each value is between [0,1].789            gligen_scheduled_sampling_beta (`float`, defaults to 0.3):790                Scheduled Sampling factor from [GLIGEN: Open-Set Grounded Text-to-Image791                Generation](https://arxiv.org/pdf/2301.07093.pdf). Scheduled Sampling factor is only varied for792                scheduled sampling during inference for improved quality and controllability.793            negative_prompt (`str` or `List[str]`, *optional*):794                The prompt or prompts to guide what to not include in image generation. If not defined, you need to795                pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).796            num_images_per_prompt (`int`, *optional*, defaults to 1):797                The number of images to generate per prompt.798            eta (`float`, *optional*, defaults to 0.0):799                Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies800                to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.801            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):802                A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make803                generation deterministic.804            latents (`torch.FloatTensor`, *optional*):805                Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image806                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents807                tensor is generated by sampling using the supplied random `generator`.808            prompt_embeds (`torch.FloatTensor`, *optional*):809                Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not810                provided, text embeddings are generated from the `prompt` input argument.811            negative_prompt_embeds (`torch.FloatTensor`, *optional*):812                Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If813                not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.814            ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.815            output_type (`str`, *optional*, defaults to `"pil"`):816                The output format of the generated image. Choose between `PIL.Image` or `np.array`.817            return_dict (`bool`, *optional*, defaults to `True`):818                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a819                plain tuple.820            callback (`Callable`, *optional*):821                A function that calls every `callback_steps` steps during inference. The function is called with the822                following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.823            callback_steps (`int`, *optional*, defaults to 1):824                The frequency at which the `callback` function is called. If not specified, the callback is called at825                every step.826            cross_attention_kwargs (`dict`, *optional*):827                A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in828                [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).829            guidance_rescale (`float`, *optional*, defaults to 0.0):830                Guidance rescale factor from [Common Diffusion Noise Schedules and Sample Steps are831                Flawed](https://arxiv.org/pdf/2305.08891.pdf). Guidance rescale factor should fix overexposure when832                using zero terminal SNR.833            clip_skip (`int`, *optional*):834                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that835                the output of the pre-final layer will be used for computing the prompt embeddings.836            lmd_guidance_kwargs (`dict`, *optional*):837                A kwargs dictionary that if specified is passed along to `latent_lmd_guidance` function. Useful keys include `loss_scale` (the guidance strength), `loss_threshold` (when loss is lower than this value, the guidance is not applied anymore), `max_iter` (the number of iterations of guidance for each step), and `guidance_timesteps` (the number of diffusion timesteps to apply guidance on). See `latent_lmd_guidance` for implementation details.838            phrase_indices (`list` of `list`, *optional*): The indices of the tokens of each phrase in the overall prompt. If omitted, the pipeline will match the first token subsequence. The pipeline will append the missing phrases to the end of the prompt by default.839        Examples:840 841        Returns:842            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:843                If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned,844                otherwise a `tuple` is returned where the first element is a list with the generated images and the845                second element is a list of `bool`s indicating whether the corresponding generated image contains846                "not-safe-for-work" (nsfw) content.847        """848        # 0. Default height and width to unet849        height = height or self.unet.config.sample_size * self.vae_scale_factor850        width = width or self.unet.config.sample_size * self.vae_scale_factor851 852        # 1. Check inputs. Raise error if not correct853        self.check_inputs(854            prompt,855            height,856            width,857            callback_steps,858            phrases,859            boxes,860            negative_prompt,861            prompt_embeds,862            negative_prompt_embeds,863            phrase_indices,864        )865 866        # 2. Define call parameters867        if prompt is not None and isinstance(prompt, str):868            batch_size = 1869            if phrase_indices is None:870                phrase_indices, prompt = self.get_phrase_indices(prompt, phrases, add_suffix_if_not_found=True)871        elif prompt is not None and isinstance(prompt, list):872            batch_size = len(prompt)873            if phrase_indices is None:874                phrase_indices = []875                prompt_parsed = []876                for prompt_item in prompt:877                    (878                        phrase_indices_parsed_item,879                        prompt_parsed_item,880                    ) = self.get_phrase_indices(prompt_item, add_suffix_if_not_found=True)881                    phrase_indices.append(phrase_indices_parsed_item)882                    prompt_parsed.append(prompt_parsed_item)883                prompt = prompt_parsed884        else:885            batch_size = prompt_embeds.shape[0]886 887        device = self._execution_device888        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)889        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`890        # corresponds to doing no classifier free guidance.891        do_classifier_free_guidance = guidance_scale > 1.0892 893        # 3. Encode input prompt894        prompt_embeds, negative_prompt_embeds = self.encode_prompt(895            prompt,896            device,897            num_images_per_prompt,898            do_classifier_free_guidance,899            negative_prompt,900            prompt_embeds=prompt_embeds,901            negative_prompt_embeds=negative_prompt_embeds,902            clip_skip=clip_skip,903        )904 905        cond_prompt_embeds = prompt_embeds906 907        # For classifier free guidance, we need to do two forward passes.908        # Here we concatenate the unconditional and text embeddings into a single batch909        # to avoid doing two forward passes910        if do_classifier_free_guidance:911            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])912 913        if ip_adapter_image is not None:914            image_embeds, negative_image_embeds = self.encode_image(ip_adapter_image, device, num_images_per_prompt)915            if self.do_classifier_free_guidance:916                image_embeds = torch.cat([negative_image_embeds, image_embeds])917 918        # 4. Prepare timesteps919        self.scheduler.set_timesteps(num_inference_steps, device=device)920        timesteps = self.scheduler.timesteps921 922        # 5. Prepare latent variables923        num_channels_latents = self.unet.config.in_channels924        latents = self.prepare_latents(925            batch_size * num_images_per_prompt,926            num_channels_latents,927            height,928            width,929            prompt_embeds.dtype,930            device,931            generator,932            latents,933        )934 935        # 5.1 Prepare GLIGEN variables936        max_objs = 30937        if len(boxes) > max_objs:938            warnings.warn(939                f"More that {max_objs} objects found. Only first {max_objs} objects will be processed.",940                FutureWarning,941            )942            phrases = phrases[:max_objs]943            boxes = boxes[:max_objs]944 945        n_objs = len(boxes)946        if n_objs:947            # prepare batched input to the PositionNet (boxes, phrases, mask)948            # Get tokens for phrases from pre-trained CLIPTokenizer949            tokenizer_inputs = self.tokenizer(phrases, padding=True, return_tensors="pt").to(device)950            # For the token, we use the same pre-trained text encoder951            # to obtain its text feature952            _text_embeddings = self.text_encoder(**tokenizer_inputs).pooler_output953 954        # For each entity, described in phrases, is denoted with a bounding box,955        # we represent the location information as (xmin,ymin,xmax,ymax)956        cond_boxes = torch.zeros(max_objs, 4, device=device, dtype=self.text_encoder.dtype)957        if n_objs:958            cond_boxes[:n_objs] = torch.tensor(boxes)959        text_embeddings = torch.zeros(960            max_objs,961            self.unet.config.cross_attention_dim,962            device=device,963            dtype=self.text_encoder.dtype,964        )965        if n_objs:966            text_embeddings[:n_objs] = _text_embeddings967        # Generate a mask for each object that is entity described by phrases968        masks = torch.zeros(max_objs, device=device, dtype=self.text_encoder.dtype)969        masks[:n_objs] = 1970 971        repeat_batch = batch_size * num_images_per_prompt972        cond_boxes = cond_boxes.unsqueeze(0).expand(repeat_batch, -1, -1).clone()973        text_embeddings = text_embeddings.unsqueeze(0).expand(repeat_batch, -1, -1).clone()974        masks = masks.unsqueeze(0).expand(repeat_batch, -1).clone()975        if do_classifier_free_guidance:976            repeat_batch = repeat_batch * 2977            cond_boxes = torch.cat([cond_boxes] * 2)978            text_embeddings = torch.cat([text_embeddings] * 2)979            masks = torch.cat([masks] * 2)980            masks[: repeat_batch // 2] = 0981        if cross_attention_kwargs is None:982            cross_attention_kwargs = {}983        cross_attention_kwargs["gligen"] = {984            "boxes": cond_boxes,985            "positive_embeddings": text_embeddings,986            "masks": masks,987        }988 989        num_grounding_steps = int(gligen_scheduled_sampling_beta * len(timesteps))990        self.enable_fuser(True)991 992        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline993        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)994 995        # 6.1 Add image embeds for IP-Adapter996        added_cond_kwargs = {"image_embeds": image_embeds} if ip_adapter_image is not None else None997 998        loss_attn = torch.tensor(10000.0)999 1000        # 7. Denoising loop1001        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order1002        with self.progress_bar(total=num_inference_steps) as progress_bar:1003            for i, t in enumerate(timesteps):1004                # Scheduled sampling1005                if i == num_grounding_steps:1006                    self.enable_fuser(False)1007 1008                if latents.shape[1] != 4:1009                    latents = torch.randn_like(latents[:, :4])1010 1011                # 7.1 Perform LMD guidance1012                if boxes:1013                    latents, loss_attn = self.latent_lmd_guidance(1014                        cond_prompt_embeds,1015                        index=i,1016                        boxes=boxes,1017                        phrase_indices=phrase_indices,1018                        t=t,1019                        latents=latents,1020                        loss=loss_attn,1021                        **lmd_guidance_kwargs,1022                    )1023 1024                # expand the latents if we are doing classifier free guidance1025                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents1026                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)1027 1028                # predict the noise residual1029                noise_pred = self.unet(1030                    latent_model_input,1031                    t,1032                    encoder_hidden_states=prompt_embeds,1033                    cross_attention_kwargs=cross_attention_kwargs,1034                    added_cond_kwargs=added_cond_kwargs,1035                ).sample1036 1037                # perform guidance1038                if do_classifier_free_guidance:1039                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)1040                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)1041 1042                # compute the previous noisy sample x_t -> x_t-11043                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample1044 1045                # call the callback, if provided1046                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):1047                    progress_bar.update()1048                    if callback is not None and i % callback_steps == 0:1049                        step_idx = i // getattr(self.scheduler, "order", 1)1050                        callback(step_idx, t, latents)1051 1052        if not output_type == "latent":1053            image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]1054            image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)1055        else:1056            image = latents1057            has_nsfw_concept = None1058 1059        if has_nsfw_concept is None:1060            do_denormalize = [True] * image.shape[0]1061        else:1062            do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept]1063 1064        image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)1065 1066        # Offload last model to CPU1067        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:1068            self.final_offload_hook.offload()1069 1070        if not return_dict:1071            return (image, has_nsfw_concept)1072 1073        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)1074 1075    @torch.set_grad_enabled(True)1076    def latent_lmd_guidance(1077        self,1078        cond_embeddings,1079        index,1080        boxes,1081        phrase_indices,1082        t,1083        latents,1084        loss,1085        *,1086        loss_scale=20,1087        loss_threshold=5.0,1088        max_iter=[3] * 5 + [2] * 5 + [1] * 5,1089        guidance_timesteps=15,1090        cross_attention_kwargs=None,1091        guidance_attn_keys=DEFAULT_GUIDANCE_ATTN_KEYS,1092        verbose=False,1093        clear_cache=False,1094        unet_additional_kwargs={},1095        guidance_callback=None,1096        **kwargs,1097    ):1098        scheduler, unet = self.scheduler, self.unet1099 1100        iteration = 01101 1102        if index < guidance_timesteps:1103            if isinstance(max_iter, list):1104                max_iter = max_iter[index]1105 1106            if verbose:1107                logger.info(1108                    f"time index {index}, loss: {loss.item()/loss_scale:.3f} (de-scaled with scale {loss_scale:.1f}), loss threshold: {loss_threshold:.3f}"1109                )1110 1111            try:1112                self.enable_attn_hook(enabled=True)1113 1114                while (1115                    loss.item() / loss_scale > loss_threshold and iteration < max_iter and index < guidance_timesteps1116                ):1117                    self._saved_attn = {}1118 1119                    latents.requires_grad_(True)1120                    latent_model_input = latents1121                    latent_model_input = scheduler.scale_model_input(latent_model_input, t)1122 1123                    unet(1124                        latent_model_input,1125                        t,1126                        encoder_hidden_states=cond_embeddings,1127                        cross_attention_kwargs=cross_attention_kwargs,1128                        **unet_additional_kwargs,1129                    )1130 1131                    # update latents with guidance1132                    loss = (1133                        self.compute_ca_loss(1134                            saved_attn=self._saved_attn,1135                            bboxes=boxes,1136                            phrase_indices=phrase_indices,1137                            guidance_attn_keys=guidance_attn_keys,1138                            verbose=verbose,1139                            **kwargs,1140                        )1141                        * loss_scale1142                    )1143 1144                    if torch.isnan(loss):1145                        raise RuntimeError("**Loss is NaN**")1146 1147                    # This callback allows visualizations.1148                    if guidance_callback is not None:1149                        guidance_callback(self, latents, loss, iteration, index)1150 1151                    self._saved_attn = None1152 1153                    grad_cond = torch.autograd.grad(loss.requires_grad_(True), [latents])[0]1154 1155                    latents.requires_grad_(False)1156 1157                    # Scaling with classifier guidance1158                    alpha_prod_t = scheduler.alphas_cumprod[t]1159                    # Classifier guidance: https://arxiv.org/pdf/2105.05233.pdf1160                    # DDIM: https://arxiv.org/pdf/2010.02502.pdf1161                    scale = (1 - alpha_prod_t) ** (0.5)1162                    latents = latents - scale * grad_cond1163 1164                    iteration += 11165 1166                    if clear_cache:1167                        gc.collect()1168                        torch.cuda.empty_cache()1169 1170                    if verbose:1171                        logger.info(1172                            f"time index {index}, loss: {loss.item()/loss_scale:.3f}, loss threshold: {loss_threshold:.3f}, iteration: {iteration}"1173                        )1174 1175            finally:1176                self.enable_attn_hook(enabled=False)1177 1178        return latents, loss1179 1180    # Below are methods copied from StableDiffusionPipeline1181    # The design choice of not inheriting from StableDiffusionPipeline is discussed here: https://github.com/huggingface/diffusers/pull/5993#issuecomment-18342585171182 1183    # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_slicing1184    def enable_vae_slicing(self):1185        r"""1186        Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to1187        compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.1188        """1189        self.vae.enable_slicing()1190 1191    # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_slicing1192    def disable_vae_slicing(self):1193        r"""1194        Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to1195        computing decoding in one step.1196        """1197        self.vae.disable_slicing()1198 1199    # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_tiling1200    def enable_vae_tiling(self):

Showing the first 1,200 of 1614 lines. Download the file for the rest.