diffusers/community-pipelines-mirror

1# Copyright 2024 Jingyang Zhang and The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import abc16import inspect17import math18import numbers19from typing import Any, Callable, Dict, List, Optional, Union20 21import numpy as np22import torch23import torch.nn as nn24import torch.nn.functional as F25from packaging import version26from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection27 28from diffusers.configuration_utils import FrozenDict29from diffusers.image_processor import PipelineImageInput, VaeImageProcessor30from diffusers.loaders import (31    FromSingleFileMixin,32    IPAdapterMixin,33    StableDiffusionLoraLoaderMixin,34    TextualInversionLoaderMixin,35)36from diffusers.models import AutoencoderKL, ImageProjection, UNet2DConditionModel37from diffusers.models.attention_processor import Attention, FusedAttnProcessor2_038from diffusers.models.lora import adjust_lora_scale_text_encoder39from diffusers.pipelines.pipeline_utils import DiffusionPipeline40from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput41from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker42from diffusers.schedulers import KarrasDiffusionSchedulers43from diffusers.utils import (44    USE_PEFT_BACKEND,45    deprecate,46    logging,47    replace_example_docstring,48    scale_lora_layers,49    unscale_lora_layers,50)51from diffusers.utils.torch_utils import randn_tensor52 53 54logger = logging.get_logger(__name__)  # pylint: disable=invalid-name55 56EXAMPLE_DOC_STRING = """57    Examples:58        ```py59        >>> import torch60        >>> from diffusers import StableDiffusionPipeline61 62        >>> pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16)63        >>> pipe = pipe.to("cuda")64 65        >>> prompt = "a photo of an astronaut riding a horse on mars"66        >>> image = pipe(prompt).images[0]67        ```68"""69 70 71class GaussianSmoothing(nn.Module):72    """73    Copied from official repo: https://github.com/showlab/BoxDiff/blob/master/utils/gaussian_smoothing.py74    Apply gaussian smoothing on a75    1d, 2d or 3d tensor. Filtering is performed seperately for each channel76    in the input using a depthwise convolution.77    Arguments:78        channels (int, sequence): Number of channels of the input tensors. Output will79            have this number of channels as well.80        kernel_size (int, sequence): Size of the gaussian kernel.81        sigma (float, sequence): Standard deviation of the gaussian kernel.82        dim (int, optional): The number of dimensions of the data.83            Default value is 2 (spatial).84    """85 86    def __init__(self, channels, kernel_size, sigma, dim=2):87        super(GaussianSmoothing, self).__init__()88        if isinstance(kernel_size, numbers.Number):89            kernel_size = [kernel_size] * dim90        if isinstance(sigma, numbers.Number):91            sigma = [sigma] * dim92 93        # The gaussian kernel is the product of the94        # gaussian function of each dimension.95        kernel = 196        meshgrids = torch.meshgrid([torch.arange(size, dtype=torch.float32) for size in kernel_size])97        for size, std, mgrid in zip(kernel_size, sigma, meshgrids):98            mean = (size - 1) / 299            kernel *= 1 / (std * math.sqrt(2 * math.pi)) * torch.exp(-(((mgrid - mean) / (2 * std)) ** 2))100 101        # Make sure sum of values in gaussian kernel equals 1.102        kernel = kernel / torch.sum(kernel)103 104        # Reshape to depthwise convolutional weight105        kernel = kernel.view(1, 1, *kernel.size())106        kernel = kernel.repeat(channels, *[1] * (kernel.dim() - 1))107 108        self.register_buffer("weight", kernel)109        self.groups = channels110 111        if dim == 1:112            self.conv = F.conv1d113        elif dim == 2:114            self.conv = F.conv2d115        elif dim == 3:116            self.conv = F.conv3d117        else:118            raise RuntimeError("Only 1, 2 and 3 dimensions are supported. Received {}.".format(dim))119 120    def forward(self, input):121        """122        Apply gaussian filter to input.123        Arguments:124            input (torch.Tensor): Input to apply gaussian filter on.125        Returns:126            filtered (torch.Tensor): Filtered output.127        """128        return self.conv(input, weight=self.weight.to(input.dtype), groups=self.groups)129 130 131class AttendExciteCrossAttnProcessor:132    def __init__(self, attnstore, place_in_unet):133        super().__init__()134        self.attnstore = attnstore135        self.place_in_unet = place_in_unet136 137    def __call__(138        self,139        attn: Attention,140        hidden_states: torch.FloatTensor,141        encoder_hidden_states: Optional[torch.FloatTensor] = None,142        attention_mask: Optional[torch.FloatTensor] = None,143    ) -> torch.Tensor:144        batch_size, sequence_length, _ = hidden_states.shape145        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size=1)146        query = attn.to_q(hidden_states)147 148        is_cross = encoder_hidden_states is not None149        encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states150        key = attn.to_k(encoder_hidden_states)151        value = attn.to_v(encoder_hidden_states)152 153        query = attn.head_to_batch_dim(query)154        key = attn.head_to_batch_dim(key)155        value = attn.head_to_batch_dim(value)156 157        attention_probs = attn.get_attention_scores(query, key, attention_mask)158        self.attnstore(attention_probs, is_cross, self.place_in_unet)159 160        hidden_states = torch.bmm(attention_probs, value)161        hidden_states = attn.batch_to_head_dim(hidden_states)162 163        # linear proj164        hidden_states = attn.to_out[0](hidden_states)165        # dropout166        hidden_states = attn.to_out[1](hidden_states)167 168        return hidden_states169 170 171class AttentionControl(abc.ABC):172    def step_callback(self, x_t):173        return x_t174 175    def between_steps(self):176        return177 178    # @property179    # def num_uncond_att_layers(self):180    #     return 0181 182    @abc.abstractmethod183    def forward(self, attn, is_cross: bool, place_in_unet: str):184        raise NotImplementedError185 186    def __call__(self, attn, is_cross: bool, place_in_unet: str):187        if self.cur_att_layer >= self.num_uncond_att_layers:188            self.forward(attn, is_cross, place_in_unet)189        self.cur_att_layer += 1190        if self.cur_att_layer == self.num_att_layers + self.num_uncond_att_layers:191            self.cur_att_layer = 0192            self.cur_step += 1193            self.between_steps()194 195    def reset(self):196        self.cur_step = 0197        self.cur_att_layer = 0198 199    def __init__(self):200        self.cur_step = 0201        self.num_att_layers = -1202        self.cur_att_layer = 0203 204 205class AttentionStore(AttentionControl):206    @staticmethod207    def get_empty_store():208        return {"down_cross": [], "mid_cross": [], "up_cross": [], "down_self": [], "mid_self": [], "up_self": []}209 210    def forward(self, attn, is_cross: bool, place_in_unet: str):211        key = f"{place_in_unet}_{'cross' if is_cross else 'self'}"212        if attn.shape[1] <= 32**2:  # avoid memory overhead213            self.step_store[key].append(attn)214        return attn215 216    def between_steps(self):217        self.attention_store = self.step_store218        if self.save_global_store:219            with torch.no_grad():220                if len(self.global_store) == 0:221                    self.global_store = self.step_store222                else:223                    for key in self.global_store:224                        for i in range(len(self.global_store[key])):225                            self.global_store[key][i] += self.step_store[key][i].detach()226        self.step_store = self.get_empty_store()227        self.step_store = self.get_empty_store()228 229    def get_average_attention(self):230        average_attention = self.attention_store231        return average_attention232 233    def get_average_global_attention(self):234        average_attention = {235            key: [item / self.cur_step for item in self.global_store[key]] for key in self.attention_store236        }237        return average_attention238 239    def reset(self):240        super(AttentionStore, self).reset()241        self.step_store = self.get_empty_store()242        self.attention_store = {}243        self.global_store = {}244 245    def __init__(self, save_global_store=False):246        """247        Initialize an empty AttentionStore248        :param step_index: used to visualize only a specific step in the diffusion process249        """250        super(AttentionStore, self).__init__()251        self.save_global_store = save_global_store252        self.step_store = self.get_empty_store()253        self.attention_store = {}254        self.global_store = {}255        self.curr_step_index = 0256        self.num_uncond_att_layers = 0257 258 259def aggregate_attention(260    attention_store: AttentionStore, res: int, from_where: List[str], is_cross: bool, select: int261) -> torch.Tensor:262    """Aggregates the attention across the different layers and heads at the specified resolution."""263    out = []264    attention_maps = attention_store.get_average_attention()265 266    # for k, v in attention_maps.items():267    #     for vv in v:268    #         print(vv.shape)269    # exit()270 271    num_pixels = res**2272    for location in from_where:273        for item in attention_maps[f"{location}_{'cross' if is_cross else 'self'}"]:274            if item.shape[1] == num_pixels:275                cross_maps = item.reshape(1, -1, res, res, item.shape[-1])[select]276                out.append(cross_maps)277    out = torch.cat(out, dim=0)278    out = out.sum(0) / out.shape[0]279    return out280 281 282def register_attention_control(model, controller):283    attn_procs = {}284    cross_att_count = 0285    for name in model.unet.attn_processors.keys():286        # cross_attention_dim = None if name.endswith("attn1.processor") else model.unet.config.cross_attention_dim287        if name.startswith("mid_block"):288            # hidden_size = model.unet.config.block_out_channels[-1]289            place_in_unet = "mid"290        elif name.startswith("up_blocks"):291            # block_id = int(name[len("up_blocks.")])292            # hidden_size = list(reversed(model.unet.config.block_out_channels))[block_id]293            place_in_unet = "up"294        elif name.startswith("down_blocks"):295            # block_id = int(name[len("down_blocks.")])296            # hidden_size = model.unet.config.block_out_channels[block_id]297            place_in_unet = "down"298        else:299            continue300 301        cross_att_count += 1302        attn_procs[name] = AttendExciteCrossAttnProcessor(attnstore=controller, place_in_unet=place_in_unet)303    model.unet.set_attn_processor(attn_procs)304    controller.num_att_layers = cross_att_count305 306 307def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):308    """309    Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and310    Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4311    """312    std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)313    std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)314    # rescale the results from guidance (fixes overexposure)315    noise_pred_rescaled = noise_cfg * (std_text / std_cfg)316    # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images317    noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg318    return noise_cfg319 320 321def retrieve_timesteps(322    scheduler,323    num_inference_steps: Optional[int] = None,324    device: Optional[Union[str, torch.device]] = None,325    timesteps: Optional[List[int]] = None,326    **kwargs,327):328    """329    Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles330    custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.331 332    Args:333        scheduler (`SchedulerMixin`):334            The scheduler to get timesteps from.335        num_inference_steps (`int`):336            The number of diffusion steps used when generating samples with a pre-trained model. If used,337            `timesteps` must be `None`.338        device (`str` or `torch.device`, *optional*):339            The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.340        timesteps (`List[int]`, *optional*):341                Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default342                timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps`343                must be `None`.344 345    Returns:346        `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the347        second element is the number of inference steps.348    """349    if timesteps is not None:350        accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())351        if not accepts_timesteps:352            raise ValueError(353                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"354                f" timestep schedules. Please check whether you are using the correct scheduler."355            )356        scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)357        timesteps = scheduler.timesteps358        num_inference_steps = len(timesteps)359    else:360        scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)361        timesteps = scheduler.timesteps362    return timesteps, num_inference_steps363 364 365class StableDiffusionBoxDiffPipeline(366    DiffusionPipeline, TextualInversionLoaderMixin, StableDiffusionLoraLoaderMixin, IPAdapterMixin, FromSingleFileMixin367):368    r"""369    Pipeline for text-to-image generation using Stable Diffusion with BoxDiff.370 371    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods372    implemented for all pipelines (downloading, saving, running on a particular device, etc.).373 374    The pipeline also inherits the following loading methods:375        - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings376        - [`~loaders.StableDiffusionLoraLoaderMixin.load_lora_weights`] for loading LoRA weights377        - [`~loaders.StableDiffusionLoraLoaderMixin.save_lora_weights`] for saving LoRA weights378        - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files379        - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters380 381    Args:382        vae ([`AutoencoderKL`]):383            Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.384        text_encoder ([`~transformers.CLIPTextModel`]):385            Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).386        tokenizer ([`~transformers.CLIPTokenizer`]):387            A `CLIPTokenizer` to tokenize text.388        unet ([`UNet2DConditionModel`]):389            A `UNet2DConditionModel` to denoise the encoded image latents.390        scheduler ([`SchedulerMixin`]):391            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of392            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].393        safety_checker ([`StableDiffusionSafetyChecker`]):394            Classification module that estimates whether generated images could be considered offensive or harmful.395            Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details396            about a model's potential harms.397        feature_extractor ([`~transformers.CLIPImageProcessor`]):398            A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`.399    """400 401    model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae"402    _optional_components = ["safety_checker", "feature_extractor", "image_encoder"]403    _exclude_from_cpu_offload = ["safety_checker"]404    _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]405 406    def __init__(407        self,408        vae: AutoencoderKL,409        text_encoder: CLIPTextModel,410        tokenizer: CLIPTokenizer,411        unet: UNet2DConditionModel,412        scheduler: KarrasDiffusionSchedulers,413        safety_checker: StableDiffusionSafetyChecker,414        feature_extractor: CLIPImageProcessor,415        image_encoder: CLIPVisionModelWithProjection = None,416        requires_safety_checker: bool = True,417    ):418        super().__init__()419 420        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:421            deprecation_message = (422                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"423                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "424                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"425                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"426                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"427                " file"428            )429            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)430            new_config = dict(scheduler.config)431            new_config["steps_offset"] = 1432            scheduler._internal_dict = FrozenDict(new_config)433 434        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:435            deprecation_message = (436                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."437                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"438                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"439                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"440                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"441            )442            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)443            new_config = dict(scheduler.config)444            new_config["clip_sample"] = False445            scheduler._internal_dict = FrozenDict(new_config)446 447        if safety_checker is None and requires_safety_checker:448            logger.warning(449                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"450                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"451                " results in services or applications open to the public. Both the diffusers team and Hugging Face"452                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"453                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"454                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."455            )456 457        if safety_checker is not None and feature_extractor is None:458            raise ValueError(459                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"460                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."461            )462 463        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(464            version.parse(unet.config._diffusers_version).base_version465        ) < version.parse("0.9.0.dev0")466        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64467        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:468            deprecation_message = (469                "The configuration file of the unet has set the default `sample_size` to smaller than"470                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"471                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"472                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"473                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"474                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"475                " in the config might lead to incorrect results in future versions. If you have downloaded this"476                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"477                " the `unet/config.json` file"478            )479            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)480            new_config = dict(unet.config)481            new_config["sample_size"] = 64482            unet._internal_dict = FrozenDict(new_config)483 484        self.register_modules(485            vae=vae,486            text_encoder=text_encoder,487            tokenizer=tokenizer,488            unet=unet,489            scheduler=scheduler,490            safety_checker=safety_checker,491            feature_extractor=feature_extractor,492            image_encoder=image_encoder,493        )494        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)495        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)496        self.register_to_config(requires_safety_checker=requires_safety_checker)497 498    def enable_vae_slicing(self):499        r"""500        Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to501        compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.502        """503        self.vae.enable_slicing()504 505    def disable_vae_slicing(self):506        r"""507        Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to508        computing decoding in one step.509        """510        self.vae.disable_slicing()511 512    def enable_vae_tiling(self):513        r"""514        Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to515        compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow516        processing larger images.517        """518        self.vae.enable_tiling()519 520    def disable_vae_tiling(self):521        r"""522        Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to523        computing decoding in one step.524        """525        self.vae.disable_tiling()526 527    def _encode_prompt(528        self,529        prompt,530        device,531        num_images_per_prompt,532        do_classifier_free_guidance,533        negative_prompt=None,534        prompt_embeds: Optional[torch.FloatTensor] = None,535        negative_prompt_embeds: Optional[torch.FloatTensor] = None,536        lora_scale: Optional[float] = None,537        **kwargs,538    ):539        deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple."540        deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False)541 542        prompt_embeds_tuple = self.encode_prompt(543            prompt=prompt,544            device=device,545            num_images_per_prompt=num_images_per_prompt,546            do_classifier_free_guidance=do_classifier_free_guidance,547            negative_prompt=negative_prompt,548            prompt_embeds=prompt_embeds,549            negative_prompt_embeds=negative_prompt_embeds,550            lora_scale=lora_scale,551            **kwargs,552        )553 554        # concatenate for backwards comp555        prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]])556 557        return prompt_embeds558 559    def encode_prompt(560        self,561        prompt,562        device,563        num_images_per_prompt,564        do_classifier_free_guidance,565        negative_prompt=None,566        prompt_embeds: Optional[torch.FloatTensor] = None,567        negative_prompt_embeds: Optional[torch.FloatTensor] = None,568        lora_scale: Optional[float] = None,569        clip_skip: Optional[int] = None,570    ):571        r"""572        Encodes the prompt into text encoder hidden states.573 574        Args:575            prompt (`str` or `List[str]`, *optional*):576                prompt to be encoded577            device: (`torch.device`):578                torch device579            num_images_per_prompt (`int`):580                number of images that should be generated per prompt581            do_classifier_free_guidance (`bool`):582                whether to use classifier free guidance or not583            negative_prompt (`str` or `List[str]`, *optional*):584                The prompt or prompts not to guide the image generation. If not defined, one has to pass585                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is586                less than `1`).587            prompt_embeds (`torch.FloatTensor`, *optional*):588                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not589                provided, text embeddings will be generated from `prompt` input argument.590            negative_prompt_embeds (`torch.FloatTensor`, *optional*):591                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt592                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input593                argument.594            lora_scale (`float`, *optional*):595                A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.596            clip_skip (`int`, *optional*):597                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that598                the output of the pre-final layer will be used for computing the prompt embeddings.599        """600        # set lora scale so that monkey patched LoRA601        # function of text encoder can correctly access it602        if lora_scale is not None and isinstance(self, StableDiffusionLoraLoaderMixin):603            self._lora_scale = lora_scale604 605            # dynamically adjust the LoRA scale606            if not USE_PEFT_BACKEND:607                adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)608            else:609                scale_lora_layers(self.text_encoder, lora_scale)610 611        if prompt is not None and isinstance(prompt, str):612            batch_size = 1613        elif prompt is not None and isinstance(prompt, list):614            batch_size = len(prompt)615        else:616            batch_size = prompt_embeds.shape[0]617 618        if prompt_embeds is None:619            # textual inversion: procecss multi-vector tokens if necessary620            if isinstance(self, TextualInversionLoaderMixin):621                prompt = self.maybe_convert_prompt(prompt, self.tokenizer)622 623            text_inputs = self.tokenizer(624                prompt,625                padding="max_length",626                max_length=self.tokenizer.model_max_length,627                truncation=True,628                return_tensors="pt",629            )630            text_input_ids = text_inputs.input_ids631            untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids632 633            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(634                text_input_ids, untruncated_ids635            ):636                removed_text = self.tokenizer.batch_decode(637                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]638                )639                logger.warning(640                    "The following part of your input was truncated because CLIP can only handle sequences up to"641                    f" {self.tokenizer.model_max_length} tokens: {removed_text}"642                )643 644            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:645                attention_mask = text_inputs.attention_mask.to(device)646            else:647                attention_mask = None648 649            if clip_skip is None:650                prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask)651                prompt_embeds = prompt_embeds[0]652            else:653                prompt_embeds = self.text_encoder(654                    text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True655                )656                # Access the `hidden_states` first, that contains a tuple of657                # all the hidden states from the encoder layers. Then index into658                # the tuple to access the hidden states from the desired layer.659                prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)]660                # We also need to apply the final LayerNorm here to not mess with the661                # representations. The `last_hidden_states` that we typically use for662                # obtaining the final prompt representations passes through the LayerNorm663                # layer.664                prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds)665 666        if self.text_encoder is not None:667            prompt_embeds_dtype = self.text_encoder.dtype668        elif self.unet is not None:669            prompt_embeds_dtype = self.unet.dtype670        else:671            prompt_embeds_dtype = prompt_embeds.dtype672 673        prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)674 675        bs_embed, seq_len, _ = prompt_embeds.shape676        # duplicate text embeddings for each generation per prompt, using mps friendly method677        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)678        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)679 680        # get unconditional embeddings for classifier free guidance681        if do_classifier_free_guidance and negative_prompt_embeds is None:682            uncond_tokens: List[str]683            if negative_prompt is None:684                uncond_tokens = [""] * batch_size685            elif prompt is not None and type(prompt) is not type(negative_prompt):686                raise TypeError(687                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="688                    f" {type(prompt)}."689                )690            elif isinstance(negative_prompt, str):691                uncond_tokens = [negative_prompt]692            elif batch_size != len(negative_prompt):693                raise ValueError(694                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"695                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"696                    " the batch size of `prompt`."697                )698            else:699                uncond_tokens = negative_prompt700 701            # textual inversion: procecss multi-vector tokens if necessary702            if isinstance(self, TextualInversionLoaderMixin):703                uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer)704 705            max_length = prompt_embeds.shape[1]706            uncond_input = self.tokenizer(707                uncond_tokens,708                padding="max_length",709                max_length=max_length,710                truncation=True,711                return_tensors="pt",712            )713 714            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:715                attention_mask = uncond_input.attention_mask.to(device)716            else:717                attention_mask = None718 719            negative_prompt_embeds = self.text_encoder(720                uncond_input.input_ids.to(device),721                attention_mask=attention_mask,722            )723            negative_prompt_embeds = negative_prompt_embeds[0]724 725        if do_classifier_free_guidance:726            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method727            seq_len = negative_prompt_embeds.shape[1]728 729            negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)730 731            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)732            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)733 734        if isinstance(self, StableDiffusionLoraLoaderMixin) and USE_PEFT_BACKEND:735            # Retrieve the original scale by scaling back the LoRA layers736            unscale_lora_layers(self.text_encoder, lora_scale)737 738        return text_inputs, prompt_embeds, negative_prompt_embeds739 740    def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):741        dtype = next(self.image_encoder.parameters()).dtype742 743        if not isinstance(image, torch.Tensor):744            image = self.feature_extractor(image, return_tensors="pt").pixel_values745 746        image = image.to(device=device, dtype=dtype)747        if output_hidden_states:748            image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]749            image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)750            uncond_image_enc_hidden_states = self.image_encoder(751                torch.zeros_like(image), output_hidden_states=True752            ).hidden_states[-2]753            uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(754                num_images_per_prompt, dim=0755            )756            return image_enc_hidden_states, uncond_image_enc_hidden_states757        else:758            image_embeds = self.image_encoder(image).image_embeds759            image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)760            uncond_image_embeds = torch.zeros_like(image_embeds)761 762            return image_embeds, uncond_image_embeds763 764    def run_safety_checker(self, image, device, dtype):765        if self.safety_checker is None:766            has_nsfw_concept = None767        else:768            if torch.is_tensor(image):769                feature_extractor_input = self.image_processor.postprocess(image, output_type="pil")770            else:771                feature_extractor_input = self.image_processor.numpy_to_pil(image)772            safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device)773            image, has_nsfw_concept = self.safety_checker(774                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)775            )776        return image, has_nsfw_concept777 778    def decode_latents(self, latents):779        deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead"780        deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False)781 782        latents = 1 / self.vae.config.scaling_factor * latents783        image = self.vae.decode(latents, return_dict=False)[0]784        image = (image / 2 + 0.5).clamp(0, 1)785        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16786        image = image.cpu().permute(0, 2, 3, 1).float().numpy()787        return image788 789    def prepare_extra_step_kwargs(self, generator, eta):790        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature791        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.792        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502793        # and should be between [0, 1]794 795        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())796        extra_step_kwargs = {}797        if accepts_eta:798            extra_step_kwargs["eta"] = eta799 800        # check if the scheduler accepts generator801        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())802        if accepts_generator:803            extra_step_kwargs["generator"] = generator804        return extra_step_kwargs805 806    def check_inputs(807        self,808        prompt,809        height,810        width,811        boxdiff_phrases,812        boxdiff_boxes,813        callback_steps,814        negative_prompt=None,815        prompt_embeds=None,816        negative_prompt_embeds=None,817        callback_on_step_end_tensor_inputs=None,818    ):819        if height % 8 != 0 or width % 8 != 0:820            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")821 822        if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):823            raise ValueError(824                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"825                f" {type(callback_steps)}."826            )827        if callback_on_step_end_tensor_inputs is not None and not all(828            k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs829        ):830            raise ValueError(831                f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"832            )833 834        if prompt is not None and prompt_embeds is not None:835            raise ValueError(836                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"837                " only forward one of the two."838            )839        elif prompt is None and prompt_embeds is None:840            raise ValueError(841                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."842            )843        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):844            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")845 846        if negative_prompt is not None and negative_prompt_embeds is not None:847            raise ValueError(848                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"849                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."850            )851 852        if prompt_embeds is not None and negative_prompt_embeds is not None:853            if prompt_embeds.shape != negative_prompt_embeds.shape:854                raise ValueError(855                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"856                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"857                    f" {negative_prompt_embeds.shape}."858                )859 860        if boxdiff_phrases is not None or boxdiff_boxes is not None:861            if not (boxdiff_phrases is not None and boxdiff_boxes is not None):862                raise ValueError("Either both `boxdiff_phrases` and `boxdiff_boxes` must be passed or none of them.")863 864            if not isinstance(boxdiff_phrases, list) or not isinstance(boxdiff_boxes, list):865                raise ValueError("`boxdiff_phrases` and `boxdiff_boxes` must be lists.")866 867            if len(boxdiff_phrases) != len(boxdiff_boxes):868                raise ValueError(869                    "`boxdiff_phrases` and `boxdiff_boxes` must have the same length,"870                    f" got: `boxdiff_phrases` {len(boxdiff_phrases)} != `boxdiff_boxes`"871                    f" {len(boxdiff_boxes)}."872                )873 874    def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):875        shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor)876        if isinstance(generator, list) and len(generator) != batch_size:877            raise ValueError(878                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"879                f" size of {batch_size}. Make sure the batch size matches the length of the generators."880            )881 882        if latents is None:883            latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)884        else:885            latents = latents.to(device)886 887        # scale the initial noise by the standard deviation required by the scheduler888        latents = latents * self.scheduler.init_noise_sigma889        return latents890 891    def enable_freeu(self, s1: float, s2: float, b1: float, b2: float):892        r"""Enables the FreeU mechanism as in https://arxiv.org/abs/2309.11497.893 894        The suffixes after the scaling factors represent the stages where they are being applied.895 896        Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of the values897        that are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL.898 899        Args:900            s1 (`float`):901                Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to902                mitigate "oversmoothing effect" in the enhanced denoising process.903            s2 (`float`):904                Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to905                mitigate "oversmoothing effect" in the enhanced denoising process.906            b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features.907            b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features.908        """909        if not hasattr(self, "unet"):910            raise ValueError("The pipeline must have `unet` for using FreeU.")911        self.unet.enable_freeu(s1=s1, s2=s2, b1=b1, b2=b2)912 913    def disable_freeu(self):914        """Disables the FreeU mechanism if enabled."""915        self.unet.disable_freeu()916 917    # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.fuse_qkv_projections918    def fuse_qkv_projections(self, unet: bool = True, vae: bool = True):919        """920        Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query,921        key, value) are fused. For cross-attention modules, key and value projection matrices are fused.922 923        <Tip warning={true}>924 925        This API is 🧪 experimental.926 927        </Tip>928 929        Args:930            unet (`bool`, defaults to `True`): To apply fusion on the UNet.931            vae (`bool`, defaults to `True`): To apply fusion on the VAE.932        """933        self.fusing_unet = False934        self.fusing_vae = False935 936        if unet:937            self.fusing_unet = True938            self.unet.fuse_qkv_projections()939            self.unet.set_attn_processor(FusedAttnProcessor2_0())940 941        if vae:942            if not isinstance(self.vae, AutoencoderKL):943                raise ValueError("`fuse_qkv_projections()` is only supported for the VAE of type `AutoencoderKL`.")944 945            self.fusing_vae = True946            self.vae.fuse_qkv_projections()947            self.vae.set_attn_processor(FusedAttnProcessor2_0())948 949    # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.unfuse_qkv_projections950    def unfuse_qkv_projections(self, unet: bool = True, vae: bool = True):951        """Disable QKV projection fusion if enabled.952 953        <Tip warning={true}>954 955        This API is 🧪 experimental.956 957        </Tip>958 959        Args:960            unet (`bool`, defaults to `True`): To apply fusion on the UNet.961            vae (`bool`, defaults to `True`): To apply fusion on the VAE.962 963        """964        if unet:965            if not self.fusing_unet:966                logger.warning("The UNet was not initially fused for QKV projections. Doing nothing.")967            else:968                self.unet.unfuse_qkv_projections()969                self.fusing_unet = False970 971        if vae:972            if not self.fusing_vae:973                logger.warning("The VAE was not initially fused for QKV projections. Doing nothing.")974            else:975                self.vae.unfuse_qkv_projections()976                self.fusing_vae = False977 978    # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding979    def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32):980        """981        See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298982 983        Args:984            timesteps (`torch.Tensor`):985                generate embedding vectors at these timesteps986            embedding_dim (`int`, *optional*, defaults to 512):987                dimension of the embeddings to generate988            dtype:989                data type of the generated embeddings990 991        Returns:992            `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)`993        """994        assert len(w.shape) == 1995        w = w * 1000.0996 997        half_dim = embedding_dim // 2998        emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)999        emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)1000        emb = w.to(dtype)[:, None] * emb[None, :]1001        emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)1002        if embedding_dim % 2 == 1:  # zero pad1003            emb = torch.nn.functional.pad(emb, (0, 1))1004        assert emb.shape == (w.shape[0], embedding_dim)1005        return emb1006 1007    @property1008    def guidance_scale(self):1009        return self._guidance_scale1010 1011    @property1012    def guidance_rescale(self):1013        return self._guidance_rescale1014 1015    @property1016    def clip_skip(self):1017        return self._clip_skip1018 1019    # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)1020    # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`1021    # corresponds to doing no classifier free guidance.1022    @property1023    def do_classifier_free_guidance(self):1024        return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None1025 1026    @property1027    def cross_attention_kwargs(self):1028        return self._cross_attention_kwargs1029 1030    @property1031    def num_timesteps(self):1032        return self._num_timesteps1033 1034    @property1035    def interrupt(self):1036        return self._interrupt1037 1038    def _compute_max_attention_per_index(1039        self,1040        attention_maps: torch.Tensor,1041        indices_to_alter: List[int],1042        smooth_attentions: bool = False,1043        sigma: float = 0.5,1044        kernel_size: int = 3,1045        normalize_eot: bool = False,1046        bboxes: List[int] = None,1047        L: int = 1,1048        P: float = 0.2,1049    ) -> List[torch.Tensor]:1050        """Computes the maximum attention value for each of the tokens we wish to alter."""1051        last_idx = -11052        if normalize_eot:1053            prompt = self.prompt1054            if isinstance(self.prompt, list):1055                prompt = self.prompt[0]1056            last_idx = len(self.tokenizer(prompt)["input_ids"]) - 11057        attention_for_text = attention_maps[:, :, 1:last_idx]1058        attention_for_text *= 1001059        attention_for_text = torch.nn.functional.softmax(attention_for_text, dim=-1)1060 1061        # Shift indices since we removed the first token "1:last_idx"1062        indices_to_alter = [index - 1 for index in indices_to_alter]1063 1064        # Extract the maximum values1065        max_indices_list_fg = []1066        max_indices_list_bg = []1067        dist_x = []1068        dist_y = []1069 1070        cnt = 01071        for i in indices_to_alter:1072            image = attention_for_text[:, :, i]1073 1074            # TODO1075            # box = [max(round(b / (512 / image.shape[0])), 0) for b in bboxes[cnt]]1076            # x1, y1, x2, y2 = box1077            H, W = image.shape1078            x1 = min(max(round(bboxes[cnt][0] * W), 0), W)1079            y1 = min(max(round(bboxes[cnt][1] * H), 0), H)1080            x2 = min(max(round(bboxes[cnt][2] * W), 0), W)1081            y2 = min(max(round(bboxes[cnt][3] * H), 0), H)1082            box = [x1, y1, x2, y2]1083            cnt += 11084 1085            # coordinates to masks1086            obj_mask = torch.zeros_like(image)1087            ones_mask = torch.ones([y2 - y1, x2 - x1], dtype=obj_mask.dtype).to(obj_mask.device)1088            obj_mask[y1:y2, x1:x2] = ones_mask1089            bg_mask = 1 - obj_mask1090 1091            if smooth_attentions:1092                smoothing = GaussianSmoothing(channels=1, kernel_size=kernel_size, sigma=sigma, dim=2).to(image.device)1093                input = F.pad(image.unsqueeze(0).unsqueeze(0), (1, 1, 1, 1), mode="reflect")1094                image = smoothing(input).squeeze(0).squeeze(0)1095 1096            # Inner-Box constraint1097            k = (obj_mask.sum() * P).long()1098            max_indices_list_fg.append((image * obj_mask).reshape(-1).topk(k)[0].mean())1099 1100            # Outer-Box constraint1101            k = (bg_mask.sum() * P).long()1102            max_indices_list_bg.append((image * bg_mask).reshape(-1).topk(k)[0].mean())1103 1104            # Corner Constraint1105            gt_proj_x = torch.max(obj_mask, dim=0)[0]1106            gt_proj_y = torch.max(obj_mask, dim=1)[0]1107            corner_mask_x = torch.zeros_like(gt_proj_x)1108            corner_mask_y = torch.zeros_like(gt_proj_y)1109 1110            # create gt according to the number config.L1111            N = gt_proj_x.shape[0]1112            corner_mask_x[max(box[0] - L, 0) : min(box[0] + L + 1, N)] = 1.01113            corner_mask_x[max(box[2] - L, 0) : min(box[2] + L + 1, N)] = 1.01114            corner_mask_y[max(box[1] - L, 0) : min(box[1] + L + 1, N)] = 1.01115            corner_mask_y[max(box[3] - L, 0) : min(box[3] + L + 1, N)] = 1.01116            dist_x.append((F.l1_loss(image.max(dim=0)[0], gt_proj_x, reduction="none") * corner_mask_x).mean())1117            dist_y.append((F.l1_loss(image.max(dim=1)[0], gt_proj_y, reduction="none") * corner_mask_y).mean())1118 1119        return max_indices_list_fg, max_indices_list_bg, dist_x, dist_y1120 1121    def _aggregate_and_get_max_attention_per_token(1122        self,1123        attention_store: AttentionStore,1124        indices_to_alter: List[int],1125        attention_res: int = 16,1126        smooth_attentions: bool = False,1127        sigma: float = 0.5,1128        kernel_size: int = 3,1129        normalize_eot: bool = False,1130        bboxes: List[int] = None,1131        L: int = 1,1132        P: float = 0.2,1133    ):1134        """Aggregates the attention for each token and computes the max activation value for each token to alter."""1135        attention_maps = aggregate_attention(1136            attention_store=attention_store,1137            res=attention_res,1138            from_where=("up", "down", "mid"),1139            is_cross=True,1140            select=0,1141        )1142        max_attention_per_index_fg, max_attention_per_index_bg, dist_x, dist_y = self._compute_max_attention_per_index(1143            attention_maps=attention_maps,1144            indices_to_alter=indices_to_alter,1145            smooth_attentions=smooth_attentions,1146            sigma=sigma,1147            kernel_size=kernel_size,1148            normalize_eot=normalize_eot,1149            bboxes=bboxes,1150            L=L,1151            P=P,1152        )1153        return max_attention_per_index_fg, max_attention_per_index_bg, dist_x, dist_y1154 1155    @staticmethod1156    def _compute_loss(1157        max_attention_per_index_fg: List[torch.Tensor],1158        max_attention_per_index_bg: List[torch.Tensor],1159        dist_x: List[torch.Tensor],1160        dist_y: List[torch.Tensor],1161        return_losses: bool = False,1162    ) -> torch.Tensor:1163        """Computes the attend-and-excite loss using the maximum attention value for each token."""1164        losses_fg = [max(0, 1.0 - curr_max) for curr_max in max_attention_per_index_fg]1165        losses_bg = [max(0, curr_max) for curr_max in max_attention_per_index_bg]1166        loss = sum(losses_fg) + sum(losses_bg) + sum(dist_x) + sum(dist_y)1167        if return_losses:1168            return max(losses_fg), losses_fg1169        else:1170            return max(losses_fg), loss1171 1172    @staticmethod1173    def _update_latent(latents: torch.Tensor, loss: torch.Tensor, step_size: float) -> torch.Tensor:1174        """Update the latent according to the computed loss."""1175        grad_cond = torch.autograd.grad(loss.requires_grad_(True), [latents], retain_graph=True)[0]1176        latents = latents - step_size * grad_cond1177        return latents1178 1179    def _perform_iterative_refinement_step(1180        self,1181        latents: torch.Tensor,1182        indices_to_alter: List[int],1183        loss_fg: torch.Tensor,1184        threshold: float,1185        text_embeddings: torch.Tensor,1186        text_input,1187        attention_store: AttentionStore,1188        step_size: float,1189        t: int,1190        attention_res: int = 16,1191        smooth_attentions: bool = True,1192        sigma: float = 0.5,1193        kernel_size: int = 3,1194        max_refinement_steps: int = 20,1195        normalize_eot: bool = False,1196        bboxes: List[int] = None,1197        L: int = 1,1198        P: float = 0.2,1199    ):1200        """

Showing the first 1,200 of 1706 lines. Download the file for the rest.