diffusers/community-pipelines-mirror

1# Copyright 2024 Jingyang Zhang and The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import abc16import inspect17import math18import numbers19from typing import Any, Callable, Dict, List, Optional, Union20 21import numpy as np22import torch23import torch.nn as nn24import torch.nn.functional as F25from packaging import version26from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection27 28from diffusers.configuration_utils import FrozenDict29from diffusers.image_processor import PipelineImageInput, VaeImageProcessor30from diffusers.loaders import (31    FromSingleFileMixin,32    IPAdapterMixin,33    StableDiffusionLoraLoaderMixin,34    TextualInversionLoaderMixin,35)36from diffusers.models import AutoencoderKL, ImageProjection, UNet2DConditionModel37from diffusers.models.attention_processor import Attention, FusedAttnProcessor2_038from diffusers.models.lora import adjust_lora_scale_text_encoder39from diffusers.pipelines.pipeline_utils import DiffusionPipeline40from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput41from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker42from diffusers.schedulers import KarrasDiffusionSchedulers43from diffusers.utils import (44    USE_PEFT_BACKEND,45    deprecate,46    logging,47    replace_example_docstring,48    scale_lora_layers,49    unscale_lora_layers,50)51from diffusers.utils.torch_utils import randn_tensor52 53 54logger = logging.get_logger(__name__)  # pylint: disable=invalid-name55 56EXAMPLE_DOC_STRING = """57    Examples:58        ```py59        >>> import torch60        >>> from diffusers import StableDiffusionPipeline61 62        >>> pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16)63        >>> pipe = pipe.to("cuda")64 65        >>> prompt = "a photo of an astronaut riding a horse on mars"66        >>> image = pipe(prompt).images[0]67        ```68"""69 70 71class GaussianSmoothing(nn.Module):72    """73    Copied from official repo: https://github.com/showlab/BoxDiff/blob/master/utils/gaussian_smoothing.py74    Apply gaussian smoothing on a75    1d, 2d or 3d tensor. Filtering is performed seperately for each channel76    in the input using a depthwise convolution.77    Arguments:78        channels (int, sequence): Number of channels of the input tensors. Output will79            have this number of channels as well.80        kernel_size (int, sequence): Size of the gaussian kernel.81        sigma (float, sequence): Standard deviation of the gaussian kernel.82        dim (int, optional): The number of dimensions of the data.83            Default value is 2 (spatial).84    """85 86    def __init__(self, channels, kernel_size, sigma, dim=2):87        super(GaussianSmoothing, self).__init__()88        if isinstance(kernel_size, numbers.Number):89            kernel_size = [kernel_size] * dim90        if isinstance(sigma, numbers.Number):91            sigma = [sigma] * dim92 93        # The gaussian kernel is the product of the94        # gaussian function of each dimension.95        kernel = 196        meshgrids = torch.meshgrid([torch.arange(size, dtype=torch.float32) for size in kernel_size])97        for size, std, mgrid in zip(kernel_size, sigma, meshgrids):98            mean = (size - 1) / 299            kernel *= 1 / (std * math.sqrt(2 * math.pi)) * torch.exp(-(((mgrid - mean) / (2 * std)) ** 2))100 101        # Make sure sum of values in gaussian kernel equals 1.102        kernel = kernel / torch.sum(kernel)103 104        # Reshape to depthwise convolutional weight105        kernel = kernel.view(1, 1, *kernel.size())106        kernel = kernel.repeat(channels, *[1] * (kernel.dim() - 1))107 108        self.register_buffer("weight", kernel)109        self.groups = channels110 111        if dim == 1:112            self.conv = F.conv1d113        elif dim == 2:114            self.conv = F.conv2d115        elif dim == 3:116            self.conv = F.conv3d117        else:118            raise RuntimeError("Only 1, 2 and 3 dimensions are supported. Received {}.".format(dim))119 120    def forward(self, input):121        """122        Apply gaussian filter to input.123        Arguments:124            input (torch.Tensor): Input to apply gaussian filter on.125        Returns:126            filtered (torch.Tensor): Filtered output.127        """128        return self.conv(input, weight=self.weight.to(input.dtype), groups=self.groups)129 130 131class AttendExciteCrossAttnProcessor:132    def __init__(self, attnstore, place_in_unet):133        super().__init__()134        self.attnstore = attnstore135        self.place_in_unet = place_in_unet136 137    def __call__(138        self,139        attn: Attention,140        hidden_states: torch.FloatTensor,141        encoder_hidden_states: Optional[torch.FloatTensor] = None,142        attention_mask: Optional[torch.FloatTensor] = None,143    ) -> torch.Tensor:144        batch_size, sequence_length, _ = hidden_states.shape145        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size=1)146        query = attn.to_q(hidden_states)147 148        is_cross = encoder_hidden_states is not None149        encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states150        key = attn.to_k(encoder_hidden_states)151        value = attn.to_v(encoder_hidden_states)152 153        query = attn.head_to_batch_dim(query)154        key = attn.head_to_batch_dim(key)155        value = attn.head_to_batch_dim(value)156 157        attention_probs = attn.get_attention_scores(query, key, attention_mask)158        self.attnstore(attention_probs, is_cross, self.place_in_unet)159 160        hidden_states = torch.bmm(attention_probs, value)161        hidden_states = attn.batch_to_head_dim(hidden_states)162 163        # linear proj164        hidden_states = attn.to_out[0](hidden_states)165        # dropout166        hidden_states = attn.to_out[1](hidden_states)167 168        return hidden_states169 170 171class AttentionControl(abc.ABC):172    def step_callback(self, x_t):173        return x_t174 175    def between_steps(self):176        return177 178    # @property179    # def num_uncond_att_layers(self):180    #     return 0181 182    @abc.abstractmethod183    def forward(self, attn, is_cross: bool, place_in_unet: str):184        raise NotImplementedError185 186    def __call__(self, attn, is_cross: bool, place_in_unet: str):187        if self.cur_att_layer >= self.num_uncond_att_layers:188            self.forward(attn, is_cross, place_in_unet)189        self.cur_att_layer += 1190        if self.cur_att_layer == self.num_att_layers + self.num_uncond_att_layers:191            self.cur_att_layer = 0192            self.cur_step += 1193            self.between_steps()194 195    def reset(self):196        self.cur_step = 0197        self.cur_att_layer = 0198 199    def __init__(self):200        self.cur_step = 0201        self.num_att_layers = -1202        self.cur_att_layer = 0203 204 205class AttentionStore(AttentionControl):206    @staticmethod207    def get_empty_store():208        return {"down_cross": [], "mid_cross": [], "up_cross": [], "down_self": [], "mid_self": [], "up_self": []}209 210    def forward(self, attn, is_cross: bool, place_in_unet: str):211        key = f"{place_in_unet}_{'cross' if is_cross else 'self'}"212        if attn.shape[1] <= 32**2:  # avoid memory overhead213            self.step_store[key].append(attn)214        return attn215 216    def between_steps(self):217        self.attention_store = self.step_store218        if self.save_global_store:219            with torch.no_grad():220                if len(self.global_store) == 0:221                    self.global_store = self.step_store222                else:223                    for key in self.global_store:224                        for i in range(len(self.global_store[key])):225                            self.global_store[key][i] += self.step_store[key][i].detach()226        self.step_store = self.get_empty_store()227        self.step_store = self.get_empty_store()228 229    def get_average_attention(self):230        average_attention = self.attention_store231        return average_attention232 233    def get_average_global_attention(self):234        average_attention = {235            key: [item / self.cur_step for item in self.global_store[key]] for key in self.attention_store236        }237        return average_attention238 239    def reset(self):240        super(AttentionStore, self).reset()241        self.step_store = self.get_empty_store()242        self.attention_store = {}243        self.global_store = {}244 245    def __init__(self, save_global_store=False):246        """247        Initialize an empty AttentionStore248        :param step_index: used to visualize only a specific step in the diffusion process249        """250        super(AttentionStore, self).__init__()251        self.save_global_store = save_global_store252        self.step_store = self.get_empty_store()253        self.attention_store = {}254        self.global_store = {}255        self.curr_step_index = 0256        self.num_uncond_att_layers = 0257 258 259def aggregate_attention(260    attention_store: AttentionStore, res: int, from_where: List[str], is_cross: bool, select: int261) -> torch.Tensor:262    """Aggregates the attention across the different layers and heads at the specified resolution."""263    out = []264    attention_maps = attention_store.get_average_attention()265 266    # for k, v in attention_maps.items():267    #     for vv in v:268    #         print(vv.shape)269    # exit()270 271    num_pixels = res**2272    for location in from_where:273        for item in attention_maps[f"{location}_{'cross' if is_cross else 'self'}"]:274            if item.shape[1] == num_pixels:275                cross_maps = item.reshape(1, -1, res, res, item.shape[-1])[select]276                out.append(cross_maps)277    out = torch.cat(out, dim=0)278    out = out.sum(0) / out.shape[0]279    return out280 281 282def register_attention_control(model, controller):283    attn_procs = {}284    cross_att_count = 0285    for name in model.unet.attn_processors.keys():286        # cross_attention_dim = None if name.endswith("attn1.processor") else model.unet.config.cross_attention_dim287        if name.startswith("mid_block"):288            # hidden_size = model.unet.config.block_out_channels[-1]289            place_in_unet = "mid"290        elif name.startswith("up_blocks"):291            # block_id = int(name[len("up_blocks.")])292            # hidden_size = list(reversed(model.unet.config.block_out_channels))[block_id]293            place_in_unet = "up"294        elif name.startswith("down_blocks"):295            # block_id = int(name[len("down_blocks.")])296            # hidden_size = model.unet.config.block_out_channels[block_id]297            place_in_unet = "down"298        else:299            continue300 301        cross_att_count += 1302        attn_procs[name] = AttendExciteCrossAttnProcessor(attnstore=controller, place_in_unet=place_in_unet)303    model.unet.set_attn_processor(attn_procs)304    controller.num_att_layers = cross_att_count305 306 307def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):308    """309    Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and310    Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4311    """312    std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)313    std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)314    # rescale the results from guidance (fixes overexposure)315    noise_pred_rescaled = noise_cfg * (std_text / std_cfg)316    # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images317    noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg318    return noise_cfg319 320 321def retrieve_timesteps(322    scheduler,323    num_inference_steps: Optional[int] = None,324    device: Optional[Union[str, torch.device]] = None,325    timesteps: Optional[List[int]] = None,326    **kwargs,327):328    """329    Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles330    custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.331 332    Args:333        scheduler (`SchedulerMixin`):334            The scheduler to get timesteps from.335        num_inference_steps (`int`):336            The number of diffusion steps used when generating samples with a pre-trained model. If used,337            `timesteps` must be `None`.338        device (`str` or `torch.device`, *optional*):339            The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.340        timesteps (`List[int]`, *optional*):341                Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default342                timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps`343                must be `None`.344 345    Returns:346        `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the347        second element is the number of inference steps.348    """349    if timesteps is not None:350        accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())351        if not accepts_timesteps:352            raise ValueError(353                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"354                f" timestep schedules. Please check whether you are using the correct scheduler."355            )356        scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)357        timesteps = scheduler.timesteps358        num_inference_steps = len(timesteps)359    else:360        scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)361        timesteps = scheduler.timesteps362    return timesteps, num_inference_steps363 364 365class StableDiffusionBoxDiffPipeline(366    DiffusionPipeline, TextualInversionLoaderMixin, StableDiffusionLoraLoaderMixin, IPAdapterMixin, FromSingleFileMixin367):368    r"""369    Pipeline for text-to-image generation using Stable Diffusion with BoxDiff.370 371    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods372    implemented for all pipelines (downloading, saving, running on a particular device, etc.).373 374    The pipeline also inherits the following loading methods:375        - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings376        - [`~loaders.StableDiffusionLoraLoaderMixin.load_lora_weights`] for loading LoRA weights377        - [`~loaders.StableDiffusionLoraLoaderMixin.save_lora_weights`] for saving LoRA weights378        - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files379        - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters380 381    Args:382        vae ([`AutoencoderKL`]):383            Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.384        text_encoder ([`~transformers.CLIPTextModel`]):385            Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).386        tokenizer ([`~transformers.CLIPTokenizer`]):387            A `CLIPTokenizer` to tokenize text.388        unet ([`UNet2DConditionModel`]):389            A `UNet2DConditionModel` to denoise the encoded image latents.390        scheduler ([`SchedulerMixin`]):391            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of392            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].393        safety_checker ([`StableDiffusionSafetyChecker`]):394            Classification module that estimates whether generated images could be considered offensive or harmful.395            Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details396            about a model's potential harms.397        feature_extractor ([`~transformers.CLIPImageProcessor`]):398            A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`.399    """400 401    model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae"402    _optional_components = ["safety_checker", "feature_extractor", "image_encoder"]403    _exclude_from_cpu_offload = ["safety_checker"]404    _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]405 406    def __init__(407        self,408        vae: AutoencoderKL,409        text_encoder: CLIPTextModel,410        tokenizer: CLIPTokenizer,411        unet: UNet2DConditionModel,412        scheduler: KarrasDiffusionSchedulers,413        safety_checker: StableDiffusionSafetyChecker,414        feature_extractor: CLIPImageProcessor,415        image_encoder: CLIPVisionModelWithProjection = None,416        requires_safety_checker: bool = True,417    ):418        super().__init__()419 420        if scheduler is not None and getattr(scheduler.config, "steps_offset", 1) != 1:421            deprecation_message = (422                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"423                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "424                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"425                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"426                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"427                " file"428            )429            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)430            new_config = dict(scheduler.config)431            new_config["steps_offset"] = 1432            scheduler._internal_dict = FrozenDict(new_config)433 434        if scheduler is not None and getattr(scheduler.config, "clip_sample", False) is True:435            deprecation_message = (436                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."437                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"438                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"439                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"440                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"441            )442            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)443            new_config = dict(scheduler.config)444            new_config["clip_sample"] = False445            scheduler._internal_dict = FrozenDict(new_config)446 447        if safety_checker is None and requires_safety_checker:448            logger.warning(449                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"450                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"451                " results in services or applications open to the public. Both the diffusers team and Hugging Face"452                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"453                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"454                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."455            )456 457        if safety_checker is not None and feature_extractor is None:458            raise ValueError(459                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"460                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."461            )462 463        is_unet_version_less_0_9_0 = (464            unet is not None465            and hasattr(unet.config, "_diffusers_version")466            and version.parse(version.parse(unet.config._diffusers_version).base_version) < version.parse("0.9.0.dev0")467        )468        is_unet_sample_size_less_64 = (469            unet is not None and hasattr(unet.config, "sample_size") and unet.config.sample_size < 64470        )471        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:472            deprecation_message = (473                "The configuration file of the unet has set the default `sample_size` to smaller than"474                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"475                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"476                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"477                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"478                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"479                " in the config might lead to incorrect results in future versions. If you have downloaded this"480                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"481                " the `unet/config.json` file"482            )483            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)484            new_config = dict(unet.config)485            new_config["sample_size"] = 64486            unet._internal_dict = FrozenDict(new_config)487 488        self.register_modules(489            vae=vae,490            text_encoder=text_encoder,491            tokenizer=tokenizer,492            unet=unet,493            scheduler=scheduler,494            safety_checker=safety_checker,495            feature_extractor=feature_extractor,496            image_encoder=image_encoder,497        )498        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8499        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)500        self.register_to_config(requires_safety_checker=requires_safety_checker)501 502    def enable_vae_slicing(self):503        r"""504        Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to505        compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.506        """507        self.vae.enable_slicing()508 509    def disable_vae_slicing(self):510        r"""511        Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to512        computing decoding in one step.513        """514        self.vae.disable_slicing()515 516    def enable_vae_tiling(self):517        r"""518        Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to519        compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow520        processing larger images.521        """522        self.vae.enable_tiling()523 524    def disable_vae_tiling(self):525        r"""526        Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to527        computing decoding in one step.528        """529        self.vae.disable_tiling()530 531    def _encode_prompt(532        self,533        prompt,534        device,535        num_images_per_prompt,536        do_classifier_free_guidance,537        negative_prompt=None,538        prompt_embeds: Optional[torch.FloatTensor] = None,539        negative_prompt_embeds: Optional[torch.FloatTensor] = None,540        lora_scale: Optional[float] = None,541        **kwargs,542    ):543        deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple."544        deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False)545 546        prompt_embeds_tuple = self.encode_prompt(547            prompt=prompt,548            device=device,549            num_images_per_prompt=num_images_per_prompt,550            do_classifier_free_guidance=do_classifier_free_guidance,551            negative_prompt=negative_prompt,552            prompt_embeds=prompt_embeds,553            negative_prompt_embeds=negative_prompt_embeds,554            lora_scale=lora_scale,555            **kwargs,556        )557 558        # concatenate for backwards comp559        prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]])560 561        return prompt_embeds562 563    def encode_prompt(564        self,565        prompt,566        device,567        num_images_per_prompt,568        do_classifier_free_guidance,569        negative_prompt=None,570        prompt_embeds: Optional[torch.FloatTensor] = None,571        negative_prompt_embeds: Optional[torch.FloatTensor] = None,572        lora_scale: Optional[float] = None,573        clip_skip: Optional[int] = None,574    ):575        r"""576        Encodes the prompt into text encoder hidden states.577 578        Args:579            prompt (`str` or `List[str]`, *optional*):580                prompt to be encoded581            device: (`torch.device`):582                torch device583            num_images_per_prompt (`int`):584                number of images that should be generated per prompt585            do_classifier_free_guidance (`bool`):586                whether to use classifier free guidance or not587            negative_prompt (`str` or `List[str]`, *optional*):588                The prompt or prompts not to guide the image generation. If not defined, one has to pass589                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is590                less than `1`).591            prompt_embeds (`torch.FloatTensor`, *optional*):592                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not593                provided, text embeddings will be generated from `prompt` input argument.594            negative_prompt_embeds (`torch.FloatTensor`, *optional*):595                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt596                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input597                argument.598            lora_scale (`float`, *optional*):599                A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.600            clip_skip (`int`, *optional*):601                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that602                the output of the pre-final layer will be used for computing the prompt embeddings.603        """604        # set lora scale so that monkey patched LoRA605        # function of text encoder can correctly access it606        if lora_scale is not None and isinstance(self, StableDiffusionLoraLoaderMixin):607            self._lora_scale = lora_scale608 609            # dynamically adjust the LoRA scale610            if not USE_PEFT_BACKEND:611                adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)612            else:613                scale_lora_layers(self.text_encoder, lora_scale)614 615        if prompt is not None and isinstance(prompt, str):616            batch_size = 1617        elif prompt is not None and isinstance(prompt, list):618            batch_size = len(prompt)619        else:620            batch_size = prompt_embeds.shape[0]621 622        if prompt_embeds is None:623            # textual inversion: procecss multi-vector tokens if necessary624            if isinstance(self, TextualInversionLoaderMixin):625                prompt = self.maybe_convert_prompt(prompt, self.tokenizer)626 627            text_inputs = self.tokenizer(628                prompt,629                padding="max_length",630                max_length=self.tokenizer.model_max_length,631                truncation=True,632                return_tensors="pt",633            )634            text_input_ids = text_inputs.input_ids635            untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids636 637            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(638                text_input_ids, untruncated_ids639            ):640                removed_text = self.tokenizer.batch_decode(641                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]642                )643                logger.warning(644                    "The following part of your input was truncated because CLIP can only handle sequences up to"645                    f" {self.tokenizer.model_max_length} tokens: {removed_text}"646                )647 648            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:649                attention_mask = text_inputs.attention_mask.to(device)650            else:651                attention_mask = None652 653            if clip_skip is None:654                prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask)655                prompt_embeds = prompt_embeds[0]656            else:657                prompt_embeds = self.text_encoder(658                    text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True659                )660                # Access the `hidden_states` first, that contains a tuple of661                # all the hidden states from the encoder layers. Then index into662                # the tuple to access the hidden states from the desired layer.663                prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)]664                # We also need to apply the final LayerNorm here to not mess with the665                # representations. The `last_hidden_states` that we typically use for666                # obtaining the final prompt representations passes through the LayerNorm667                # layer.668                prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds)669 670        if self.text_encoder is not None:671            prompt_embeds_dtype = self.text_encoder.dtype672        elif self.unet is not None:673            prompt_embeds_dtype = self.unet.dtype674        else:675            prompt_embeds_dtype = prompt_embeds.dtype676 677        prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)678 679        bs_embed, seq_len, _ = prompt_embeds.shape680        # duplicate text embeddings for each generation per prompt, using mps friendly method681        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)682        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)683 684        # get unconditional embeddings for classifier free guidance685        if do_classifier_free_guidance and negative_prompt_embeds is None:686            uncond_tokens: List[str]687            if negative_prompt is None:688                uncond_tokens = [""] * batch_size689            elif prompt is not None and type(prompt) is not type(negative_prompt):690                raise TypeError(691                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="692                    f" {type(prompt)}."693                )694            elif isinstance(negative_prompt, str):695                uncond_tokens = [negative_prompt]696            elif batch_size != len(negative_prompt):697                raise ValueError(698                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"699                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"700                    " the batch size of `prompt`."701                )702            else:703                uncond_tokens = negative_prompt704 705            # textual inversion: procecss multi-vector tokens if necessary706            if isinstance(self, TextualInversionLoaderMixin):707                uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer)708 709            max_length = prompt_embeds.shape[1]710            uncond_input = self.tokenizer(711                uncond_tokens,712                padding="max_length",713                max_length=max_length,714                truncation=True,715                return_tensors="pt",716            )717 718            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:719                attention_mask = uncond_input.attention_mask.to(device)720            else:721                attention_mask = None722 723            negative_prompt_embeds = self.text_encoder(724                uncond_input.input_ids.to(device),725                attention_mask=attention_mask,726            )727            negative_prompt_embeds = negative_prompt_embeds[0]728 729        if do_classifier_free_guidance:730            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method731            seq_len = negative_prompt_embeds.shape[1]732 733            negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)734 735            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)736            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)737 738        if isinstance(self, StableDiffusionLoraLoaderMixin) and USE_PEFT_BACKEND:739            # Retrieve the original scale by scaling back the LoRA layers740            unscale_lora_layers(self.text_encoder, lora_scale)741 742        return text_inputs, prompt_embeds, negative_prompt_embeds743 744    def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):745        dtype = next(self.image_encoder.parameters()).dtype746 747        if not isinstance(image, torch.Tensor):748            image = self.feature_extractor(image, return_tensors="pt").pixel_values749 750        image = image.to(device=device, dtype=dtype)751        if output_hidden_states:752            image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]753            image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)754            uncond_image_enc_hidden_states = self.image_encoder(755                torch.zeros_like(image), output_hidden_states=True756            ).hidden_states[-2]757            uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(758                num_images_per_prompt, dim=0759            )760            return image_enc_hidden_states, uncond_image_enc_hidden_states761        else:762            image_embeds = self.image_encoder(image).image_embeds763            image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)764            uncond_image_embeds = torch.zeros_like(image_embeds)765 766            return image_embeds, uncond_image_embeds767 768    def run_safety_checker(self, image, device, dtype):769        if self.safety_checker is None:770            has_nsfw_concept = None771        else:772            if torch.is_tensor(image):773                feature_extractor_input = self.image_processor.postprocess(image, output_type="pil")774            else:775                feature_extractor_input = self.image_processor.numpy_to_pil(image)776            safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device)777            image, has_nsfw_concept = self.safety_checker(778                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)779            )780        return image, has_nsfw_concept781 782    def decode_latents(self, latents):783        deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead"784        deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False)785 786        latents = 1 / self.vae.config.scaling_factor * latents787        image = self.vae.decode(latents, return_dict=False)[0]788        image = (image / 2 + 0.5).clamp(0, 1)789        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16790        image = image.cpu().permute(0, 2, 3, 1).float().numpy()791        return image792 793    def prepare_extra_step_kwargs(self, generator, eta):794        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature795        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.796        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502797        # and should be between [0, 1]798 799        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())800        extra_step_kwargs = {}801        if accepts_eta:802            extra_step_kwargs["eta"] = eta803 804        # check if the scheduler accepts generator805        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())806        if accepts_generator:807            extra_step_kwargs["generator"] = generator808        return extra_step_kwargs809 810    def check_inputs(811        self,812        prompt,813        height,814        width,815        boxdiff_phrases,816        boxdiff_boxes,817        callback_steps,818        negative_prompt=None,819        prompt_embeds=None,820        negative_prompt_embeds=None,821        callback_on_step_end_tensor_inputs=None,822    ):823        if height % 8 != 0 or width % 8 != 0:824            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")825 826        if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):827            raise ValueError(828                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"829                f" {type(callback_steps)}."830            )831        if callback_on_step_end_tensor_inputs is not None and not all(832            k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs833        ):834            raise ValueError(835                f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"836            )837 838        if prompt is not None and prompt_embeds is not None:839            raise ValueError(840                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"841                " only forward one of the two."842            )843        elif prompt is None and prompt_embeds is None:844            raise ValueError(845                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."846            )847        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):848            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")849 850        if negative_prompt is not None and negative_prompt_embeds is not None:851            raise ValueError(852                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"853                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."854            )855 856        if prompt_embeds is not None and negative_prompt_embeds is not None:857            if prompt_embeds.shape != negative_prompt_embeds.shape:858                raise ValueError(859                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"860                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"861                    f" {negative_prompt_embeds.shape}."862                )863 864        if boxdiff_phrases is not None or boxdiff_boxes is not None:865            if not (boxdiff_phrases is not None and boxdiff_boxes is not None):866                raise ValueError("Either both `boxdiff_phrases` and `boxdiff_boxes` must be passed or none of them.")867 868            if not isinstance(boxdiff_phrases, list) or not isinstance(boxdiff_boxes, list):869                raise ValueError("`boxdiff_phrases` and `boxdiff_boxes` must be lists.")870 871            if len(boxdiff_phrases) != len(boxdiff_boxes):872                raise ValueError(873                    "`boxdiff_phrases` and `boxdiff_boxes` must have the same length,"874                    f" got: `boxdiff_phrases` {len(boxdiff_phrases)} != `boxdiff_boxes`"875                    f" {len(boxdiff_boxes)}."876                )877 878    def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):879        shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor)880        if isinstance(generator, list) and len(generator) != batch_size:881            raise ValueError(882                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"883                f" size of {batch_size}. Make sure the batch size matches the length of the generators."884            )885 886        if latents is None:887            latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)888        else:889            latents = latents.to(device)890 891        # scale the initial noise by the standard deviation required by the scheduler892        latents = latents * self.scheduler.init_noise_sigma893        return latents894 895    def enable_freeu(self, s1: float, s2: float, b1: float, b2: float):896        r"""Enables the FreeU mechanism as in https://arxiv.org/abs/2309.11497.897 898        The suffixes after the scaling factors represent the stages where they are being applied.899 900        Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of the values901        that are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL.902 903        Args:904            s1 (`float`):905                Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to906                mitigate "oversmoothing effect" in the enhanced denoising process.907            s2 (`float`):908                Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to909                mitigate "oversmoothing effect" in the enhanced denoising process.910            b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features.911            b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features.912        """913        if not hasattr(self, "unet"):914            raise ValueError("The pipeline must have `unet` for using FreeU.")915        self.unet.enable_freeu(s1=s1, s2=s2, b1=b1, b2=b2)916 917    def disable_freeu(self):918        """Disables the FreeU mechanism if enabled."""919        self.unet.disable_freeu()920 921    # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.fuse_qkv_projections922    def fuse_qkv_projections(self, unet: bool = True, vae: bool = True):923        """924        Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query,925        key, value) are fused. For cross-attention modules, key and value projection matrices are fused.926 927        <Tip warning={true}>928 929        This API is 🧪 experimental.930 931        </Tip>932 933        Args:934            unet (`bool`, defaults to `True`): To apply fusion on the UNet.935            vae (`bool`, defaults to `True`): To apply fusion on the VAE.936        """937        self.fusing_unet = False938        self.fusing_vae = False939 940        if unet:941            self.fusing_unet = True942            self.unet.fuse_qkv_projections()943            self.unet.set_attn_processor(FusedAttnProcessor2_0())944 945        if vae:946            if not isinstance(self.vae, AutoencoderKL):947                raise ValueError("`fuse_qkv_projections()` is only supported for the VAE of type `AutoencoderKL`.")948 949            self.fusing_vae = True950            self.vae.fuse_qkv_projections()951            self.vae.set_attn_processor(FusedAttnProcessor2_0())952 953    # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.unfuse_qkv_projections954    def unfuse_qkv_projections(self, unet: bool = True, vae: bool = True):955        """Disable QKV projection fusion if enabled.956 957        <Tip warning={true}>958 959        This API is 🧪 experimental.960 961        </Tip>962 963        Args:964            unet (`bool`, defaults to `True`): To apply fusion on the UNet.965            vae (`bool`, defaults to `True`): To apply fusion on the VAE.966 967        """968        if unet:969            if not self.fusing_unet:970                logger.warning("The UNet was not initially fused for QKV projections. Doing nothing.")971            else:972                self.unet.unfuse_qkv_projections()973                self.fusing_unet = False974 975        if vae:976            if not self.fusing_vae:977                logger.warning("The VAE was not initially fused for QKV projections. Doing nothing.")978            else:979                self.vae.unfuse_qkv_projections()980                self.fusing_vae = False981 982    # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding983    def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32):984        """985        See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298986 987        Args:988            timesteps (`torch.Tensor`):989                generate embedding vectors at these timesteps990            embedding_dim (`int`, *optional*, defaults to 512):991                dimension of the embeddings to generate992            dtype:993                data type of the generated embeddings994 995        Returns:996            `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)`997        """998        assert len(w.shape) == 1999        w = w * 1000.01000 1001        half_dim = embedding_dim // 21002        emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)1003        emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)1004        emb = w.to(dtype)[:, None] * emb[None, :]1005        emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)1006        if embedding_dim % 2 == 1:  # zero pad1007            emb = torch.nn.functional.pad(emb, (0, 1))1008        assert emb.shape == (w.shape[0], embedding_dim)1009        return emb1010 1011    @property1012    def guidance_scale(self):1013        return self._guidance_scale1014 1015    @property1016    def guidance_rescale(self):1017        return self._guidance_rescale1018 1019    @property1020    def clip_skip(self):1021        return self._clip_skip1022 1023    # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)1024    # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`1025    # corresponds to doing no classifier free guidance.1026    @property1027    def do_classifier_free_guidance(self):1028        return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None1029 1030    @property1031    def cross_attention_kwargs(self):1032        return self._cross_attention_kwargs1033 1034    @property1035    def num_timesteps(self):1036        return self._num_timesteps1037 1038    @property1039    def interrupt(self):1040        return self._interrupt1041 1042    def _compute_max_attention_per_index(1043        self,1044        attention_maps: torch.Tensor,1045        indices_to_alter: List[int],1046        smooth_attentions: bool = False,1047        sigma: float = 0.5,1048        kernel_size: int = 3,1049        normalize_eot: bool = False,1050        bboxes: List[int] = None,1051        L: int = 1,1052        P: float = 0.2,1053    ) -> List[torch.Tensor]:1054        """Computes the maximum attention value for each of the tokens we wish to alter."""1055        last_idx = -11056        if normalize_eot:1057            prompt = self.prompt1058            if isinstance(self.prompt, list):1059                prompt = self.prompt[0]1060            last_idx = len(self.tokenizer(prompt)["input_ids"]) - 11061        attention_for_text = attention_maps[:, :, 1:last_idx]1062        attention_for_text *= 1001063        attention_for_text = torch.nn.functional.softmax(attention_for_text, dim=-1)1064 1065        # Shift indices since we removed the first token "1:last_idx"1066        indices_to_alter = [index - 1 for index in indices_to_alter]1067 1068        # Extract the maximum values1069        max_indices_list_fg = []1070        max_indices_list_bg = []1071        dist_x = []1072        dist_y = []1073 1074        cnt = 01075        for i in indices_to_alter:1076            image = attention_for_text[:, :, i]1077 1078            # TODO1079            # box = [max(round(b / (512 / image.shape[0])), 0) for b in bboxes[cnt]]1080            # x1, y1, x2, y2 = box1081            H, W = image.shape1082            x1 = min(max(round(bboxes[cnt][0] * W), 0), W)1083            y1 = min(max(round(bboxes[cnt][1] * H), 0), H)1084            x2 = min(max(round(bboxes[cnt][2] * W), 0), W)1085            y2 = min(max(round(bboxes[cnt][3] * H), 0), H)1086            box = [x1, y1, x2, y2]1087            cnt += 11088 1089            # coordinates to masks1090            obj_mask = torch.zeros_like(image)1091            ones_mask = torch.ones([y2 - y1, x2 - x1], dtype=obj_mask.dtype).to(obj_mask.device)1092            obj_mask[y1:y2, x1:x2] = ones_mask1093            bg_mask = 1 - obj_mask1094 1095            if smooth_attentions:1096                smoothing = GaussianSmoothing(channels=1, kernel_size=kernel_size, sigma=sigma, dim=2).to(image.device)1097                input = F.pad(image.unsqueeze(0).unsqueeze(0), (1, 1, 1, 1), mode="reflect")1098                image = smoothing(input).squeeze(0).squeeze(0)1099 1100            # Inner-Box constraint1101            k = (obj_mask.sum() * P).long()1102            max_indices_list_fg.append((image * obj_mask).reshape(-1).topk(k)[0].mean())1103 1104            # Outer-Box constraint1105            k = (bg_mask.sum() * P).long()1106            max_indices_list_bg.append((image * bg_mask).reshape(-1).topk(k)[0].mean())1107 1108            # Corner Constraint1109            gt_proj_x = torch.max(obj_mask, dim=0)[0]1110            gt_proj_y = torch.max(obj_mask, dim=1)[0]1111            corner_mask_x = torch.zeros_like(gt_proj_x)1112            corner_mask_y = torch.zeros_like(gt_proj_y)1113 1114            # create gt according to the number config.L1115            N = gt_proj_x.shape[0]1116            corner_mask_x[max(box[0] - L, 0) : min(box[0] + L + 1, N)] = 1.01117            corner_mask_x[max(box[2] - L, 0) : min(box[2] + L + 1, N)] = 1.01118            corner_mask_y[max(box[1] - L, 0) : min(box[1] + L + 1, N)] = 1.01119            corner_mask_y[max(box[3] - L, 0) : min(box[3] + L + 1, N)] = 1.01120            dist_x.append((F.l1_loss(image.max(dim=0)[0], gt_proj_x, reduction="none") * corner_mask_x).mean())1121            dist_y.append((F.l1_loss(image.max(dim=1)[0], gt_proj_y, reduction="none") * corner_mask_y).mean())1122 1123        return max_indices_list_fg, max_indices_list_bg, dist_x, dist_y1124 1125    def _aggregate_and_get_max_attention_per_token(1126        self,1127        attention_store: AttentionStore,1128        indices_to_alter: List[int],1129        attention_res: int = 16,1130        smooth_attentions: bool = False,1131        sigma: float = 0.5,1132        kernel_size: int = 3,1133        normalize_eot: bool = False,1134        bboxes: List[int] = None,1135        L: int = 1,1136        P: float = 0.2,1137    ):1138        """Aggregates the attention for each token and computes the max activation value for each token to alter."""1139        attention_maps = aggregate_attention(1140            attention_store=attention_store,1141            res=attention_res,1142            from_where=("up", "down", "mid"),1143            is_cross=True,1144            select=0,1145        )1146        max_attention_per_index_fg, max_attention_per_index_bg, dist_x, dist_y = self._compute_max_attention_per_index(1147            attention_maps=attention_maps,1148            indices_to_alter=indices_to_alter,1149            smooth_attentions=smooth_attentions,1150            sigma=sigma,1151            kernel_size=kernel_size,1152            normalize_eot=normalize_eot,1153            bboxes=bboxes,1154            L=L,1155            P=P,1156        )1157        return max_attention_per_index_fg, max_attention_per_index_bg, dist_x, dist_y1158 1159    @staticmethod1160    def _compute_loss(1161        max_attention_per_index_fg: List[torch.Tensor],1162        max_attention_per_index_bg: List[torch.Tensor],1163        dist_x: List[torch.Tensor],1164        dist_y: List[torch.Tensor],1165        return_losses: bool = False,1166    ) -> torch.Tensor:1167        """Computes the attend-and-excite loss using the maximum attention value for each token."""1168        losses_fg = [max(0, 1.0 - curr_max) for curr_max in max_attention_per_index_fg]1169        losses_bg = [max(0, curr_max) for curr_max in max_attention_per_index_bg]1170        loss = sum(losses_fg) + sum(losses_bg) + sum(dist_x) + sum(dist_y)1171        if return_losses:1172            return max(losses_fg), losses_fg1173        else:1174            return max(losses_fg), loss1175 1176    @staticmethod1177    def _update_latent(latents: torch.Tensor, loss: torch.Tensor, step_size: float) -> torch.Tensor:1178        """Update the latent according to the computed loss."""1179        grad_cond = torch.autograd.grad(loss.requires_grad_(True), [latents], retain_graph=True)[0]1180        latents = latents - step_size * grad_cond1181        return latents1182 1183    def _perform_iterative_refinement_step(1184        self,1185        latents: torch.Tensor,1186        indices_to_alter: List[int],1187        loss_fg: torch.Tensor,1188        threshold: float,1189        text_embeddings: torch.Tensor,1190        text_input,1191        attention_store: AttentionStore,1192        step_size: float,1193        t: int,1194        attention_res: int = 16,1195        smooth_attentions: bool = True,1196        sigma: float = 0.5,1197        kernel_size: int = 3,1198        max_refinement_steps: int = 20,1199        normalize_eot: bool = False,1200        bboxes: List[int] = None,

Showing the first 1,200 of 1710 lines. Download the file for the rest.