diffusers/community-pipelines-mirror

1# A diffuser version implementation of Zero1to3 (https://github.com/cvlab-columbia/zero123), ICCV 20232# by Xin Kong3 4import inspect5from typing import Any, Callable, Dict, List, Optional, Union6 7import kornia8import numpy as np9import PIL.Image10import torch11from packaging import version12from transformers import CLIPFeatureExtractor, CLIPVisionModelWithProjection13 14# from ...configuration_utils import FrozenDict15# from ...models import AutoencoderKL, UNet2DConditionModel16# from ...schedulers import KarrasDiffusionSchedulers17# from ...utils import (18#     deprecate,19#     is_accelerate_available,20#     is_accelerate_version,21#     logging,22#     randn_tensor,23#     replace_example_docstring,24# )25# from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin26# from . import StableDiffusionPipelineOutput27# from .safety_checker import StableDiffusionSafetyChecker28from diffusers import AutoencoderKL, DiffusionPipeline, StableDiffusionMixin, UNet2DConditionModel29from diffusers.configuration_utils import ConfigMixin, FrozenDict30from diffusers.models.modeling_utils import ModelMixin31from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput, StableDiffusionSafetyChecker32from diffusers.schedulers import KarrasDiffusionSchedulers33from diffusers.utils import (34    deprecate,35    logging,36    replace_example_docstring,37)38from diffusers.utils.torch_utils import randn_tensor39 40 41logger = logging.get_logger(__name__)  # pylint: disable=invalid-name42# todo43EXAMPLE_DOC_STRING = """44    Examples:45        ```py46        >>> import torch47        >>> from diffusers import StableDiffusionPipeline48 49        >>> pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16)50        >>> pipe = pipe.to("cuda")51 52        >>> prompt = "a photo of an astronaut riding a horse on mars"53        >>> image = pipe(prompt).images[0]54        ```55"""56 57 58class CCProjection(ModelMixin, ConfigMixin):59    def __init__(self, in_channel=772, out_channel=768):60        super().__init__()61        self.in_channel = in_channel62        self.out_channel = out_channel63        self.projection = torch.nn.Linear(in_channel, out_channel)64 65    def forward(self, x):66        return self.projection(x)67 68 69class Zero1to3StableDiffusionPipeline(DiffusionPipeline, StableDiffusionMixin):70    r"""71    Pipeline for single view conditioned novel view generation using Zero1to3.72 73    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the74    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)75 76    Args:77        vae ([`AutoencoderKL`]):78            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.79        image_encoder ([`CLIPVisionModelWithProjection`]):80            Frozen CLIP image-encoder. Stable Diffusion Image Variation uses the vision portion of81            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPVisionModelWithProjection),82            specifically the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.83        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.84        scheduler ([`SchedulerMixin`]):85            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of86            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].87        safety_checker ([`StableDiffusionSafetyChecker`]):88            Classification module that estimates whether generated images could be considered offensive or harmful.89            Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details.90        feature_extractor ([`CLIPFeatureExtractor`]):91            Model that extracts features from generated images to be used as inputs for the `safety_checker`.92        cc_projection ([`CCProjection`]):93            Projection layer to project the concated CLIP features and pose embeddings to the original CLIP feature size.94    """95 96    _optional_components = ["safety_checker", "feature_extractor"]97 98    def __init__(99        self,100        vae: AutoencoderKL,101        image_encoder: CLIPVisionModelWithProjection,102        unet: UNet2DConditionModel,103        scheduler: KarrasDiffusionSchedulers,104        safety_checker: StableDiffusionSafetyChecker,105        feature_extractor: CLIPFeatureExtractor,106        cc_projection: CCProjection,107        requires_safety_checker: bool = True,108    ):109        super().__init__()110 111        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:112            deprecation_message = (113                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"114                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "115                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"116                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"117                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"118                " file"119            )120            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)121            new_config = dict(scheduler.config)122            new_config["steps_offset"] = 1123            scheduler._internal_dict = FrozenDict(new_config)124 125        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:126            deprecation_message = (127                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."128                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"129                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"130                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"131                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"132            )133            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)134            new_config = dict(scheduler.config)135            new_config["clip_sample"] = False136            scheduler._internal_dict = FrozenDict(new_config)137 138        if safety_checker is None and requires_safety_checker:139            logger.warning(140                f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"141                " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"142                " results in services or applications open to the public. Both the diffusers team and Hugging Face"143                " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"144                " it only for use-cases that involve analyzing network behavior or auditing its results. For more"145                " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."146            )147 148        if safety_checker is not None and feature_extractor is None:149            raise ValueError(150                "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"151                " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."152            )153 154        is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(155            version.parse(unet.config._diffusers_version).base_version156        ) < version.parse("0.9.0.dev0")157        is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64158        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:159            deprecation_message = (160                "The configuration file of the unet has set the default `sample_size` to smaller than"161                " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"162                " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"163                " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"164                " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"165                " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"166                " in the config might lead to incorrect results in future versions. If you have downloaded this"167                " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"168                " the `unet/config.json` file"169            )170            deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)171            new_config = dict(unet.config)172            new_config["sample_size"] = 64173            unet._internal_dict = FrozenDict(new_config)174 175        self.register_modules(176            vae=vae,177            image_encoder=image_encoder,178            unet=unet,179            scheduler=scheduler,180            safety_checker=safety_checker,181            feature_extractor=feature_extractor,182            cc_projection=cc_projection,183        )184        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)185        self.register_to_config(requires_safety_checker=requires_safety_checker)186        # self.model_mode = None187 188    def _encode_prompt(189        self,190        prompt,191        device,192        num_images_per_prompt,193        do_classifier_free_guidance,194        negative_prompt=None,195        prompt_embeds: Optional[torch.Tensor] = None,196        negative_prompt_embeds: Optional[torch.Tensor] = None,197    ):198        r"""199        Encodes the prompt into text encoder hidden states.200 201        Args:202             prompt (`str` or `List[str]`, *optional*):203                prompt to be encoded204            device: (`torch.device`):205                torch device206            num_images_per_prompt (`int`):207                number of images that should be generated per prompt208            do_classifier_free_guidance (`bool`):209                whether to use classifier free guidance or not210            negative_prompt (`str` or `List[str]`, *optional*):211                The prompt or prompts not to guide the image generation. If not defined, one has to pass212                `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.213                Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).214            prompt_embeds (`torch.Tensor`, *optional*):215                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not216                provided, text embeddings will be generated from `prompt` input argument.217            negative_prompt_embeds (`torch.Tensor`, *optional*):218                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt219                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input220                argument.221        """222        if prompt is not None and isinstance(prompt, str):223            batch_size = 1224        elif prompt is not None and isinstance(prompt, list):225            batch_size = len(prompt)226        else:227            batch_size = prompt_embeds.shape[0]228 229        if prompt_embeds is None:230            text_inputs = self.tokenizer(231                prompt,232                padding="max_length",233                max_length=self.tokenizer.model_max_length,234                truncation=True,235                return_tensors="pt",236            )237            text_input_ids = text_inputs.input_ids238            untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids239 240            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(241                text_input_ids, untruncated_ids242            ):243                removed_text = self.tokenizer.batch_decode(244                    untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]245                )246                logger.warning(247                    "The following part of your input was truncated because CLIP can only handle sequences up to"248                    f" {self.tokenizer.model_max_length} tokens: {removed_text}"249                )250 251            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:252                attention_mask = text_inputs.attention_mask.to(device)253            else:254                attention_mask = None255 256            prompt_embeds = self.text_encoder(257                text_input_ids.to(device),258                attention_mask=attention_mask,259            )260            prompt_embeds = prompt_embeds[0]261 262        prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)263 264        bs_embed, seq_len, _ = prompt_embeds.shape265        # duplicate text embeddings for each generation per prompt, using mps friendly method266        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)267        prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)268 269        # get unconditional embeddings for classifier free guidance270        if do_classifier_free_guidance and negative_prompt_embeds is None:271            uncond_tokens: List[str]272            if negative_prompt is None:273                uncond_tokens = [""] * batch_size274            elif type(prompt) is not type(negative_prompt):275                raise TypeError(276                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="277                    f" {type(prompt)}."278                )279            elif isinstance(negative_prompt, str):280                uncond_tokens = [negative_prompt]281            elif batch_size != len(negative_prompt):282                raise ValueError(283                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"284                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"285                    " the batch size of `prompt`."286                )287            else:288                uncond_tokens = negative_prompt289 290            max_length = prompt_embeds.shape[1]291            uncond_input = self.tokenizer(292                uncond_tokens,293                padding="max_length",294                max_length=max_length,295                truncation=True,296                return_tensors="pt",297            )298 299            if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:300                attention_mask = uncond_input.attention_mask.to(device)301            else:302                attention_mask = None303 304            negative_prompt_embeds = self.text_encoder(305                uncond_input.input_ids.to(device),306                attention_mask=attention_mask,307            )308            negative_prompt_embeds = negative_prompt_embeds[0]309 310        if do_classifier_free_guidance:311            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method312            seq_len = negative_prompt_embeds.shape[1]313 314            negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)315 316            negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)317            negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)318 319            # For classifier free guidance, we need to do two forward passes.320            # Here we concatenate the unconditional and text embeddings into a single batch321            # to avoid doing two forward passes322            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])323 324        return prompt_embeds325 326    def CLIP_preprocess(self, x):327        dtype = x.dtype328        # following openai's implementation329        # TODO HF OpenAI CLIP preprocessing issue https://github.com/huggingface/transformers/issues/22505#issuecomment-1650170741330        # follow openai preprocessing to keep exact same, input tensor [-1, 1], otherwise the preprocessing will be different, https://github.com/huggingface/transformers/pull/22608331        if isinstance(x, torch.Tensor):332            if x.min() < -1.0 or x.max() > 1.0:333                raise ValueError("Expected input tensor to have values in the range [-1, 1]")334        x = kornia.geometry.resize(335            x.to(torch.float32), (224, 224), interpolation="bicubic", align_corners=True, antialias=False336        ).to(dtype=dtype)337        x = (x + 1.0) / 2.0338        # renormalize according to clip339        x = kornia.enhance.normalize(340            x, torch.Tensor([0.48145466, 0.4578275, 0.40821073]), torch.Tensor([0.26862954, 0.26130258, 0.27577711])341        )342        return x343 344    # from image_variation345    def _encode_image(self, image, device, num_images_per_prompt, do_classifier_free_guidance):346        dtype = next(self.image_encoder.parameters()).dtype347        if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)):348            raise ValueError(349                f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}"350            )351 352        if isinstance(image, torch.Tensor):353            # Batch single image354            if image.ndim == 3:355                assert image.shape[0] == 3, "Image outside a batch should be of shape (3, H, W)"356                image = image.unsqueeze(0)357 358            assert image.ndim == 4, "Image must have 4 dimensions"359 360            # Check image is in [-1, 1]361            if image.min() < -1 or image.max() > 1:362                raise ValueError("Image should be in [-1, 1] range")363        else:364            # preprocess image365            if isinstance(image, (PIL.Image.Image, np.ndarray)):366                image = [image]367 368            if isinstance(image, list) and isinstance(image[0], PIL.Image.Image):369                image = [np.array(i.convert("RGB"))[None, :] for i in image]370                image = np.concatenate(image, axis=0)371            elif isinstance(image, list) and isinstance(image[0], np.ndarray):372                image = np.concatenate([i[None, :] for i in image], axis=0)373 374            image = image.transpose(0, 3, 1, 2)375            image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0376 377        image = image.to(device=device, dtype=dtype)378 379        image = self.CLIP_preprocess(image)380        # if not isinstance(image, torch.Tensor):381        #     # 0-255382        #     print("Warning: image is processed by hf's preprocess, which is different from openai original's.")383        #     image = self.feature_extractor(images=image, return_tensors="pt").pixel_values384        image_embeddings = self.image_encoder(image).image_embeds.to(dtype=dtype)385        image_embeddings = image_embeddings.unsqueeze(1)386 387        # duplicate image embeddings for each generation per prompt, using mps friendly method388        bs_embed, seq_len, _ = image_embeddings.shape389        image_embeddings = image_embeddings.repeat(1, num_images_per_prompt, 1)390        image_embeddings = image_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1)391 392        if do_classifier_free_guidance:393            negative_prompt_embeds = torch.zeros_like(image_embeddings)394 395            # For classifier free guidance, we need to do two forward passes.396            # Here we concatenate the unconditional and text embeddings into a single batch397            # to avoid doing two forward passes398            image_embeddings = torch.cat([negative_prompt_embeds, image_embeddings])399 400        return image_embeddings401 402    def _encode_pose(self, pose, device, num_images_per_prompt, do_classifier_free_guidance):403        dtype = next(self.cc_projection.parameters()).dtype404        if isinstance(pose, torch.Tensor):405            pose_embeddings = pose.unsqueeze(1).to(device=device, dtype=dtype)406        else:407            if isinstance(pose[0], list):408                pose = torch.Tensor(pose)409            else:410                pose = torch.Tensor([pose])411            x, y, z = pose[:, 0].unsqueeze(1), pose[:, 1].unsqueeze(1), pose[:, 2].unsqueeze(1)412            pose_embeddings = (413                torch.cat([torch.deg2rad(x), torch.sin(torch.deg2rad(y)), torch.cos(torch.deg2rad(y)), z], dim=-1)414                .unsqueeze(1)415                .to(device=device, dtype=dtype)416            )  # B, 1, 4417        # duplicate pose embeddings for each generation per prompt, using mps friendly method418        bs_embed, seq_len, _ = pose_embeddings.shape419        pose_embeddings = pose_embeddings.repeat(1, num_images_per_prompt, 1)420        pose_embeddings = pose_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1)421        if do_classifier_free_guidance:422            negative_prompt_embeds = torch.zeros_like(pose_embeddings)423 424            # For classifier free guidance, we need to do two forward passes.425            # Here we concatenate the unconditional and text embeddings into a single batch426            # to avoid doing two forward passes427            pose_embeddings = torch.cat([negative_prompt_embeds, pose_embeddings])428        return pose_embeddings429 430    def _encode_image_with_pose(self, image, pose, device, num_images_per_prompt, do_classifier_free_guidance):431        img_prompt_embeds = self._encode_image(image, device, num_images_per_prompt, False)432        pose_prompt_embeds = self._encode_pose(pose, device, num_images_per_prompt, False)433        prompt_embeds = torch.cat([img_prompt_embeds, pose_prompt_embeds], dim=-1)434        prompt_embeds = self.cc_projection(prompt_embeds)435        # prompt_embeds = img_prompt_embeds436        # follow 0123, add negative prompt, after projection437        if do_classifier_free_guidance:438            negative_prompt = torch.zeros_like(prompt_embeds)439            prompt_embeds = torch.cat([negative_prompt, prompt_embeds])440        return prompt_embeds441 442    def run_safety_checker(self, image, device, dtype):443        if self.safety_checker is not None:444            safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)445            image, has_nsfw_concept = self.safety_checker(446                images=image, clip_input=safety_checker_input.pixel_values.to(dtype)447            )448        else:449            has_nsfw_concept = None450        return image, has_nsfw_concept451 452    def decode_latents(self, latents):453        latents = 1 / self.vae.config.scaling_factor * latents454        image = self.vae.decode(latents).sample455        image = (image / 2 + 0.5).clamp(0, 1)456        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16457        image = image.cpu().permute(0, 2, 3, 1).float().numpy()458        return image459 460    def prepare_extra_step_kwargs(self, generator, eta):461        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature462        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.463        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502464        # and should be between [0, 1]465 466        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())467        extra_step_kwargs = {}468        if accepts_eta:469            extra_step_kwargs["eta"] = eta470 471        # check if the scheduler accepts generator472        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())473        if accepts_generator:474            extra_step_kwargs["generator"] = generator475        return extra_step_kwargs476 477    def check_inputs(self, image, height, width, callback_steps):478        if (479            not isinstance(image, torch.Tensor)480            and not isinstance(image, PIL.Image.Image)481            and not isinstance(image, list)482        ):483            raise ValueError(484                "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is"485                f" {type(image)}"486            )487 488        if height % 8 != 0 or width % 8 != 0:489            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")490 491        if (callback_steps is None) or (492            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)493        ):494            raise ValueError(495                f"`callback_steps` has to be a positive integer but is {callback_steps} of type"496                f" {type(callback_steps)}."497            )498 499    def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):500        shape = (501            batch_size,502            num_channels_latents,503            int(height) // self.vae_scale_factor,504            int(width) // self.vae_scale_factor,505        )506        if isinstance(generator, list) and len(generator) != batch_size:507            raise ValueError(508                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"509                f" size of {batch_size}. Make sure the batch size matches the length of the generators."510            )511 512        if latents is None:513            latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)514        else:515            latents = latents.to(device)516 517        # scale the initial noise by the standard deviation required by the scheduler518        latents = latents * self.scheduler.init_noise_sigma519        return latents520 521    def prepare_img_latents(self, image, batch_size, dtype, device, generator=None, do_classifier_free_guidance=False):522        if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)):523            raise ValueError(524                f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}"525            )526 527        if isinstance(image, torch.Tensor):528            # Batch single image529            if image.ndim == 3:530                assert image.shape[0] == 3, "Image outside a batch should be of shape (3, H, W)"531                image = image.unsqueeze(0)532 533            assert image.ndim == 4, "Image must have 4 dimensions"534 535            # Check image is in [-1, 1]536            if image.min() < -1 or image.max() > 1:537                raise ValueError("Image should be in [-1, 1] range")538        else:539            # preprocess image540            if isinstance(image, (PIL.Image.Image, np.ndarray)):541                image = [image]542 543            if isinstance(image, list) and isinstance(image[0], PIL.Image.Image):544                image = [np.array(i.convert("RGB"))[None, :] for i in image]545                image = np.concatenate(image, axis=0)546            elif isinstance(image, list) and isinstance(image[0], np.ndarray):547                image = np.concatenate([i[None, :] for i in image], axis=0)548 549            image = image.transpose(0, 3, 1, 2)550            image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0551 552        image = image.to(device=device, dtype=dtype)553 554        if isinstance(generator, list) and len(generator) != batch_size:555            raise ValueError(556                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"557                f" size of {batch_size}. Make sure the batch size matches the length of the generators."558            )559 560        if isinstance(generator, list):561            init_latents = [562                self.vae.encode(image[i : i + 1]).latent_dist.mode(generator[i])563                for i in range(batch_size)  # sample564            ]565            init_latents = torch.cat(init_latents, dim=0)566        else:567            init_latents = self.vae.encode(image).latent_dist.mode()568 569        # init_latents = self.vae.config.scaling_factor * init_latents  # todo in original zero123's inference gradio_new.py, model.encode_first_stage() is not scaled by scaling_factor570        if batch_size > init_latents.shape[0]:571            # init_latents = init_latents.repeat(batch_size // init_latents.shape[0], 1, 1, 1)572            num_images_per_prompt = batch_size // init_latents.shape[0]573            # duplicate image latents for each generation per prompt, using mps friendly method574            bs_embed, emb_c, emb_h, emb_w = init_latents.shape575            init_latents = init_latents.unsqueeze(1)576            init_latents = init_latents.repeat(1, num_images_per_prompt, 1, 1, 1)577            init_latents = init_latents.view(bs_embed * num_images_per_prompt, emb_c, emb_h, emb_w)578 579        # init_latents = torch.cat([init_latents]*2) if do_classifier_free_guidance else init_latents   # follow zero123580        init_latents = (581            torch.cat([torch.zeros_like(init_latents), init_latents]) if do_classifier_free_guidance else init_latents582        )583 584        init_latents = init_latents.to(device=device, dtype=dtype)585        return init_latents586 587    # def load_cc_projection(self, pretrained_weights=None):588    #     self.cc_projection = torch.nn.Linear(772, 768)589    #     torch.nn.init.eye_(list(self.cc_projection.parameters())[0][:768, :768])590    #     torch.nn.init.zeros_(list(self.cc_projection.parameters())[1])591    #     if pretrained_weights is not None:592    #         self.cc_projection.load_state_dict(pretrained_weights)593 594    @torch.no_grad()595    @replace_example_docstring(EXAMPLE_DOC_STRING)596    def __call__(597        self,598        input_imgs: Union[torch.Tensor, PIL.Image.Image] = None,599        prompt_imgs: Union[torch.Tensor, PIL.Image.Image] = None,600        poses: Union[List[float], List[List[float]]] = None,601        torch_dtype=torch.float32,602        height: Optional[int] = None,603        width: Optional[int] = None,604        num_inference_steps: int = 50,605        guidance_scale: float = 3.0,606        negative_prompt: Optional[Union[str, List[str]]] = None,607        num_images_per_prompt: Optional[int] = 1,608        eta: float = 0.0,609        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,610        latents: Optional[torch.Tensor] = None,611        prompt_embeds: Optional[torch.Tensor] = None,612        negative_prompt_embeds: Optional[torch.Tensor] = None,613        output_type: Optional[str] = "pil",614        return_dict: bool = True,615        callback: Optional[Callable[[int, int, torch.Tensor], None]] = None,616        callback_steps: int = 1,617        cross_attention_kwargs: Optional[Dict[str, Any]] = None,618        controlnet_conditioning_scale: float = 1.0,619    ):620        r"""621        Function invoked when calling the pipeline for generation.622 623        Args:624            input_imgs (`PIL` or `List[PIL]`, *optional*):625                The single input image for each 3D object626            prompt_imgs (`PIL` or `List[PIL]`, *optional*):627                Same as input_imgs, but will be used later as an image prompt condition, encoded by CLIP feature628            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):629                The height in pixels of the generated image.630            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):631                The width in pixels of the generated image.632            num_inference_steps (`int`, *optional*, defaults to 50):633                The number of denoising steps. More denoising steps usually lead to a higher quality image at the634                expense of slower inference.635            guidance_scale (`float`, *optional*, defaults to 7.5):636                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).637                `guidance_scale` is defined as `w` of equation 2. of [Imagen638                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >639                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,640                usually at the expense of lower image quality.641            negative_prompt (`str` or `List[str]`, *optional*):642                The prompt or prompts not to guide the image generation. If not defined, one has to pass643                `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.644                Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).645            num_images_per_prompt (`int`, *optional*, defaults to 1):646                The number of images to generate per prompt.647            eta (`float`, *optional*, defaults to 0.0):648                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to649                [`schedulers.DDIMScheduler`], will be ignored for others.650            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):651                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)652                to make generation deterministic.653            latents (`torch.Tensor`, *optional*):654                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image655                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents656                tensor will ge generated by sampling using the supplied random `generator`.657            prompt_embeds (`torch.Tensor`, *optional*):658                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not659                provided, text embeddings will be generated from `prompt` input argument.660            negative_prompt_embeds (`torch.Tensor`, *optional*):661                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt662                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input663                argument.664            output_type (`str`, *optional*, defaults to `"pil"`):665                The output format of the generate image. Choose between666                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.667            return_dict (`bool`, *optional*, defaults to `True`):668                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a669                plain tuple.670            callback (`Callable`, *optional*):671                A function that will be called every `callback_steps` steps during inference. The function will be672                called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`.673            callback_steps (`int`, *optional*, defaults to 1):674                The frequency at which the `callback` function will be called. If not specified, the callback will be675                called at every step.676            cross_attention_kwargs (`dict`, *optional*):677                A kwargs dictionary that if specified is passed along to the `AttnProcessor` as defined under678                `self.processor` in679                [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py).680 681        Examples:682 683        Returns:684            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:685            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.686            When returning a tuple, the first element is a list with the generated images, and the second element is a687            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"688            (nsfw) content, according to the `safety_checker`.689        """690        # 0. Default height and width to unet691        height = height or self.unet.config.sample_size * self.vae_scale_factor692        width = width or self.unet.config.sample_size * self.vae_scale_factor693 694        # 1. Check inputs. Raise error if not correct695        # input_image = hint_imgs696        self.check_inputs(input_imgs, height, width, callback_steps)697 698        # 2. Define call parameters699        if isinstance(input_imgs, PIL.Image.Image):700            batch_size = 1701        elif isinstance(input_imgs, list):702            batch_size = len(input_imgs)703        else:704            batch_size = input_imgs.shape[0]705        device = self._execution_device706        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)707        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`708        # corresponds to doing no classifier free guidance.709        do_classifier_free_guidance = guidance_scale > 1.0710 711        # 3. Encode input image with pose as prompt712        prompt_embeds = self._encode_image_with_pose(713            prompt_imgs, poses, device, num_images_per_prompt, do_classifier_free_guidance714        )715 716        # 4. Prepare timesteps717        self.scheduler.set_timesteps(num_inference_steps, device=device)718        timesteps = self.scheduler.timesteps719 720        # 5. Prepare latent variables721        latents = self.prepare_latents(722            batch_size * num_images_per_prompt,723            4,724            height,725            width,726            prompt_embeds.dtype,727            device,728            generator,729            latents,730        )731 732        # 6. Prepare image latents733        img_latents = self.prepare_img_latents(734            input_imgs,735            batch_size * num_images_per_prompt,736            prompt_embeds.dtype,737            device,738            generator,739            do_classifier_free_guidance,740        )741 742        # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline743        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)744 745        # 7. Denoising loop746        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order747        with self.progress_bar(total=num_inference_steps) as progress_bar:748            for i, t in enumerate(timesteps):749                # expand the latents if we are doing classifier free guidance750                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents751                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)752                latent_model_input = torch.cat([latent_model_input, img_latents], dim=1)753 754                # predict the noise residual755                noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=prompt_embeds).sample756 757                # perform guidance758                if do_classifier_free_guidance:759                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)760                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)761 762                # compute the previous noisy sample x_t -> x_t-1763                # latents = self.scheduler.step(noise_pred.to(dtype=torch.float32), t, latents.to(dtype=torch.float32)).prev_sample.to(prompt_embeds.dtype)764                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]765 766                # call the callback, if provided767                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):768                    progress_bar.update()769                    if callback is not None and i % callback_steps == 0:770                        step_idx = i // getattr(self.scheduler, "order", 1)771                        callback(step_idx, t, latents)772 773        # 8. Post-processing774        has_nsfw_concept = None775        if output_type == "latent":776            image = latents777        elif output_type == "pil":778            # 8. Post-processing779            image = self.decode_latents(latents)780            # 10. Convert to PIL781            image = self.numpy_to_pil(image)782        else:783            # 8. Post-processing784            image = self.decode_latents(latents)785 786        # Offload last model to CPU787        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:788            self.final_offload_hook.offload()789 790        if not return_dict:791            return (image, has_nsfw_concept)792 793        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)794