declare-lab/tango2
92
1# coding=utf-82# Copyright 2023 HuggingFace Inc.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import gc17import json18import os19import random20import shutil21import sys22import tempfile23import unittest24import unittest.mock as mock25 26import numpy as np27import PIL28import requests_mock29import safetensors.torch30import torch31from parameterized import parameterized32from PIL import Image33from requests.exceptions import HTTPError34from transformers import CLIPImageProcessor, CLIPModel, CLIPTextConfig, CLIPTextModel, CLIPTokenizer35 36from diffusers import (37 AutoencoderKL,38 DDIMPipeline,39 DDIMScheduler,40 DDPMPipeline,41 DDPMScheduler,42 DiffusionPipeline,43 DPMSolverMultistepScheduler,44 EulerAncestralDiscreteScheduler,45 EulerDiscreteScheduler,46 LMSDiscreteScheduler,47 PNDMScheduler,48 StableDiffusionImg2ImgPipeline,49 StableDiffusionInpaintPipelineLegacy,50 StableDiffusionPipeline,51 UNet2DConditionModel,52 UNet2DModel,53 UniPCMultistepScheduler,54 logging,55)56from diffusers.schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME57from diffusers.utils import (58 CONFIG_NAME,59 WEIGHTS_NAME,60 floats_tensor,61 is_flax_available,62 nightly,63 require_torch_2,64 slow,65 torch_device,66)67from diffusers.utils.testing_utils import CaptureLogger, get_tests_dir, load_numpy, require_compel, require_torch_gpu68 69 70torch.backends.cuda.matmul.allow_tf32 = False71 72 73class DownloadTests(unittest.TestCase):74 def test_one_request_upon_cached(self):75 # TODO: For some reason this test fails on MPS where no HEAD call is made.76 if torch_device == "mps":77 return78 79 with tempfile.TemporaryDirectory() as tmpdirname:80 with requests_mock.mock(real_http=True) as m:81 DiffusionPipeline.download(82 "hf-internal-testing/tiny-stable-diffusion-pipe", safety_checker=None, cache_dir=tmpdirname83 )84 85 download_requests = [r.method for r in m.request_history]86 assert download_requests.count("HEAD") == 15, "15 calls to files"87 assert download_requests.count("GET") == 17, "15 calls to files + model_info + model_index.json"88 assert (89 len(download_requests) == 3290 ), "2 calls per file (15 files) + send_telemetry, model_info and model_index.json"91 92 with requests_mock.mock(real_http=True) as m:93 DiffusionPipeline.download(94 "hf-internal-testing/tiny-stable-diffusion-pipe", safety_checker=None, cache_dir=tmpdirname95 )96 97 cache_requests = [r.method for r in m.request_history]98 assert cache_requests.count("HEAD") == 1, "model_index.json is only HEAD"99 assert cache_requests.count("GET") == 1, "model info is only GET"100 assert (101 len(cache_requests) == 2102 ), "We should call only `model_info` to check for _commit hash and `send_telemetry`"103 104 def test_download_only_pytorch(self):105 with tempfile.TemporaryDirectory() as tmpdirname:106 # pipeline has Flax weights107 tmpdirname = DiffusionPipeline.download(108 "hf-internal-testing/tiny-stable-diffusion-pipe", safety_checker=None, cache_dir=tmpdirname109 )110 111 all_root_files = [t[-1] for t in os.walk(os.path.join(tmpdirname))]112 files = [item for sublist in all_root_files for item in sublist]113 114 # None of the downloaded files should be a flax file even if we have some here:115 # https://huggingface.co/hf-internal-testing/tiny-stable-diffusion-pipe/blob/main/unet/diffusion_flax_model.msgpack116 assert not any(f.endswith(".msgpack") for f in files)117 # We need to never convert this tiny model to safetensors for this test to pass118 assert not any(f.endswith(".safetensors") for f in files)119 120 def test_force_safetensors_error(self):121 with tempfile.TemporaryDirectory() as tmpdirname:122 # pipeline has Flax weights123 with self.assertRaises(EnvironmentError):124 tmpdirname = DiffusionPipeline.download(125 "hf-internal-testing/tiny-stable-diffusion-pipe-no-safetensors",126 safety_checker=None,127 cache_dir=tmpdirname,128 use_safetensors=True,129 )130 131 def test_returned_cached_folder(self):132 prompt = "hello"133 pipe = StableDiffusionPipeline.from_pretrained(134 "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None135 )136 _, local_path = StableDiffusionPipeline.from_pretrained(137 "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None, return_cached_folder=True138 )139 pipe_2 = StableDiffusionPipeline.from_pretrained(local_path)140 141 pipe = pipe.to(torch_device)142 pipe_2 = pipe_2.to(torch_device)143 144 generator = torch.manual_seed(0)145 out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images146 147 generator = torch.manual_seed(0)148 out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images149 150 assert np.max(np.abs(out - out_2)) < 1e-3151 152 def test_download_safetensors(self):153 with tempfile.TemporaryDirectory() as tmpdirname:154 # pipeline has Flax weights155 tmpdirname = DiffusionPipeline.download(156 "hf-internal-testing/tiny-stable-diffusion-pipe-safetensors",157 safety_checker=None,158 cache_dir=tmpdirname,159 )160 161 all_root_files = [t[-1] for t in os.walk(os.path.join(tmpdirname))]162 files = [item for sublist in all_root_files for item in sublist]163 164 # None of the downloaded files should be a pytorch file even if we have some here:165 # https://huggingface.co/hf-internal-testing/tiny-stable-diffusion-pipe/blob/main/unet/diffusion_flax_model.msgpack166 assert not any(f.endswith(".bin") for f in files)167 168 def test_download_no_safety_checker(self):169 prompt = "hello"170 pipe = StableDiffusionPipeline.from_pretrained(171 "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None172 )173 pipe = pipe.to(torch_device)174 generator = torch.manual_seed(0)175 out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images176 177 pipe_2 = StableDiffusionPipeline.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch")178 pipe_2 = pipe_2.to(torch_device)179 generator = torch.manual_seed(0)180 out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images181 182 assert np.max(np.abs(out - out_2)) < 1e-3183 184 def test_load_no_safety_checker_explicit_locally(self):185 prompt = "hello"186 pipe = StableDiffusionPipeline.from_pretrained(187 "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None188 )189 pipe = pipe.to(torch_device)190 generator = torch.manual_seed(0)191 out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images192 193 with tempfile.TemporaryDirectory() as tmpdirname:194 pipe.save_pretrained(tmpdirname)195 pipe_2 = StableDiffusionPipeline.from_pretrained(tmpdirname, safety_checker=None)196 pipe_2 = pipe_2.to(torch_device)197 198 generator = torch.manual_seed(0)199 200 out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images201 202 assert np.max(np.abs(out - out_2)) < 1e-3203 204 def test_load_no_safety_checker_default_locally(self):205 prompt = "hello"206 pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch")207 pipe = pipe.to(torch_device)208 209 generator = torch.manual_seed(0)210 out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images211 212 with tempfile.TemporaryDirectory() as tmpdirname:213 pipe.save_pretrained(tmpdirname)214 pipe_2 = StableDiffusionPipeline.from_pretrained(tmpdirname)215 pipe_2 = pipe_2.to(torch_device)216 217 generator = torch.manual_seed(0)218 219 out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images220 221 assert np.max(np.abs(out - out_2)) < 1e-3222 223 def test_cached_files_are_used_when_no_internet(self):224 # A mock response for an HTTP head request to emulate server down225 response_mock = mock.Mock()226 response_mock.status_code = 500227 response_mock.headers = {}228 response_mock.raise_for_status.side_effect = HTTPError229 response_mock.json.return_value = {}230 231 # Download this model to make sure it's in the cache.232 orig_pipe = StableDiffusionPipeline.from_pretrained(233 "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None234 )235 orig_comps = {k: v for k, v in orig_pipe.components.items() if hasattr(v, "parameters")}236 237 # Under the mock environment we get a 500 error when trying to reach the model.238 with mock.patch("requests.request", return_value=response_mock):239 # Download this model to make sure it's in the cache.240 pipe = StableDiffusionPipeline.from_pretrained(241 "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None, local_files_only=True242 )243 comps = {k: v for k, v in pipe.components.items() if hasattr(v, "parameters")}244 245 for m1, m2 in zip(orig_comps.values(), comps.values()):246 for p1, p2 in zip(m1.parameters(), m2.parameters()):247 if p1.data.ne(p2.data).sum() > 0:248 assert False, "Parameters not the same!"249 250 def test_download_from_variant_folder(self):251 for safe_avail in [False, True]:252 import diffusers253 254 diffusers.utils.import_utils._safetensors_available = safe_avail255 256 other_format = ".bin" if safe_avail else ".safetensors"257 with tempfile.TemporaryDirectory() as tmpdirname:258 tmpdirname = StableDiffusionPipeline.download(259 "hf-internal-testing/stable-diffusion-all-variants", cache_dir=tmpdirname260 )261 all_root_files = [t[-1] for t in os.walk(tmpdirname)]262 files = [item for sublist in all_root_files for item in sublist]263 264 # None of the downloaded files should be a variant file even if we have some here:265 # https://huggingface.co/hf-internal-testing/stable-diffusion-all-variants/tree/main/unet266 assert len(files) == 15, f"We should only download 15 files, not {len(files)}"267 assert not any(f.endswith(other_format) for f in files)268 # no variants269 assert not any(len(f.split(".")) == 3 for f in files)270 271 diffusers.utils.import_utils._safetensors_available = True272 273 def test_download_variant_all(self):274 for safe_avail in [False, True]:275 import diffusers276 277 diffusers.utils.import_utils._safetensors_available = safe_avail278 279 other_format = ".bin" if safe_avail else ".safetensors"280 this_format = ".safetensors" if safe_avail else ".bin"281 variant = "fp16"282 283 with tempfile.TemporaryDirectory() as tmpdirname:284 tmpdirname = StableDiffusionPipeline.download(285 "hf-internal-testing/stable-diffusion-all-variants", cache_dir=tmpdirname, variant=variant286 )287 all_root_files = [t[-1] for t in os.walk(tmpdirname)]288 files = [item for sublist in all_root_files for item in sublist]289 290 # None of the downloaded files should be a non-variant file even if we have some here:291 # https://huggingface.co/hf-internal-testing/stable-diffusion-all-variants/tree/main/unet292 assert len(files) == 15, f"We should only download 15 files, not {len(files)}"293 # unet, vae, text_encoder, safety_checker294 assert len([f for f in files if f.endswith(f"{variant}{this_format}")]) == 4295 # all checkpoints should have variant ending296 assert not any(f.endswith(this_format) and not f.endswith(f"{variant}{this_format}") for f in files)297 assert not any(f.endswith(other_format) for f in files)298 299 diffusers.utils.import_utils._safetensors_available = True300 301 def test_download_variant_partly(self):302 for safe_avail in [False, True]:303 import diffusers304 305 diffusers.utils.import_utils._safetensors_available = safe_avail306 307 other_format = ".bin" if safe_avail else ".safetensors"308 this_format = ".safetensors" if safe_avail else ".bin"309 variant = "no_ema"310 311 with tempfile.TemporaryDirectory() as tmpdirname:312 tmpdirname = StableDiffusionPipeline.download(313 "hf-internal-testing/stable-diffusion-all-variants", cache_dir=tmpdirname, variant=variant314 )315 all_root_files = [t[-1] for t in os.walk(tmpdirname)]316 files = [item for sublist in all_root_files for item in sublist]317 318 unet_files = os.listdir(os.path.join(tmpdirname, "unet"))319 320 # Some of the downloaded files should be a non-variant file, check:321 # https://huggingface.co/hf-internal-testing/stable-diffusion-all-variants/tree/main/unet322 assert len(files) == 15, f"We should only download 15 files, not {len(files)}"323 # only unet has "no_ema" variant324 assert f"diffusion_pytorch_model.{variant}{this_format}" in unet_files325 assert len([f for f in files if f.endswith(f"{variant}{this_format}")]) == 1326 # vae, safety_checker and text_encoder should have no variant327 assert sum(f.endswith(this_format) and not f.endswith(f"{variant}{this_format}") for f in files) == 3328 assert not any(f.endswith(other_format) for f in files)329 330 diffusers.utils.import_utils._safetensors_available = True331 332 def test_download_broken_variant(self):333 for safe_avail in [False, True]:334 import diffusers335 336 diffusers.utils.import_utils._safetensors_available = safe_avail337 # text encoder is missing no variant and "no_ema" variant weights, so the following can't work338 for variant in [None, "no_ema"]:339 with self.assertRaises(OSError) as error_context:340 with tempfile.TemporaryDirectory() as tmpdirname:341 tmpdirname = StableDiffusionPipeline.from_pretrained(342 "hf-internal-testing/stable-diffusion-broken-variants",343 cache_dir=tmpdirname,344 variant=variant,345 )346 347 assert "Error no file name" in str(error_context.exception)348 349 # text encoder has fp16 variants so we can load it350 with tempfile.TemporaryDirectory() as tmpdirname:351 tmpdirname = StableDiffusionPipeline.download(352 "hf-internal-testing/stable-diffusion-broken-variants", cache_dir=tmpdirname, variant="fp16"353 )354 355 all_root_files = [t[-1] for t in os.walk(tmpdirname)]356 files = [item for sublist in all_root_files for item in sublist]357 358 # None of the downloaded files should be a non-variant file even if we have some here:359 # https://huggingface.co/hf-internal-testing/stable-diffusion-broken-variants/tree/main/unet360 assert len(files) == 15, f"We should only download 15 files, not {len(files)}"361 # only unet has "no_ema" variant362 363 diffusers.utils.import_utils._safetensors_available = True364 365 def test_text_inversion_download(self):366 pipe = StableDiffusionPipeline.from_pretrained(367 "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None368 )369 pipe = pipe.to(torch_device)370 371 num_tokens = len(pipe.tokenizer)372 373 # single token load local374 with tempfile.TemporaryDirectory() as tmpdirname:375 ten = {"<*>": torch.ones((32,))}376 torch.save(ten, os.path.join(tmpdirname, "learned_embeds.bin"))377 378 pipe.load_textual_inversion(tmpdirname)379 380 token = pipe.tokenizer.convert_tokens_to_ids("<*>")381 assert token == num_tokens, "Added token must be at spot `num_tokens`"382 assert pipe.text_encoder.get_input_embeddings().weight[-1].sum().item() == 32383 assert pipe._maybe_convert_prompt("<*>", pipe.tokenizer) == "<*>"384 385 prompt = "hey <*>"386 out = pipe(prompt, num_inference_steps=1, output_type="numpy").images387 assert out.shape == (1, 128, 128, 3)388 389 # single token load local with weight name390 with tempfile.TemporaryDirectory() as tmpdirname:391 ten = {"<**>": 2 * torch.ones((1, 32))}392 torch.save(ten, os.path.join(tmpdirname, "learned_embeds.bin"))393 394 pipe.load_textual_inversion(tmpdirname, weight_name="learned_embeds.bin")395 396 token = pipe.tokenizer.convert_tokens_to_ids("<**>")397 assert token == num_tokens + 1, "Added token must be at spot `num_tokens`"398 assert pipe.text_encoder.get_input_embeddings().weight[-1].sum().item() == 64399 assert pipe._maybe_convert_prompt("<**>", pipe.tokenizer) == "<**>"400 401 prompt = "hey <**>"402 out = pipe(prompt, num_inference_steps=1, output_type="numpy").images403 assert out.shape == (1, 128, 128, 3)404 405 # multi token load406 with tempfile.TemporaryDirectory() as tmpdirname:407 ten = {"<***>": torch.cat([3 * torch.ones((1, 32)), 4 * torch.ones((1, 32)), 5 * torch.ones((1, 32))])}408 torch.save(ten, os.path.join(tmpdirname, "learned_embeds.bin"))409 410 pipe.load_textual_inversion(tmpdirname)411 412 token = pipe.tokenizer.convert_tokens_to_ids("<***>")413 token_1 = pipe.tokenizer.convert_tokens_to_ids("<***>_1")414 token_2 = pipe.tokenizer.convert_tokens_to_ids("<***>_2")415 416 assert token == num_tokens + 2, "Added token must be at spot `num_tokens`"417 assert token_1 == num_tokens + 3, "Added token must be at spot `num_tokens`"418 assert token_2 == num_tokens + 4, "Added token must be at spot `num_tokens`"419 assert pipe.text_encoder.get_input_embeddings().weight[-3].sum().item() == 96420 assert pipe.text_encoder.get_input_embeddings().weight[-2].sum().item() == 128421 assert pipe.text_encoder.get_input_embeddings().weight[-1].sum().item() == 160422 assert pipe._maybe_convert_prompt("<***>", pipe.tokenizer) == "<***><***>_1<***>_2"423 424 prompt = "hey <***>"425 out = pipe(prompt, num_inference_steps=1, output_type="numpy").images426 assert out.shape == (1, 128, 128, 3)427 428 # multi token load a1111429 with tempfile.TemporaryDirectory() as tmpdirname:430 ten = {431 "string_to_param": {432 "*": torch.cat([3 * torch.ones((1, 32)), 4 * torch.ones((1, 32)), 5 * torch.ones((1, 32))])433 },434 "name": "<****>",435 }436 torch.save(ten, os.path.join(tmpdirname, "a1111.bin"))437 438 pipe.load_textual_inversion(tmpdirname, weight_name="a1111.bin")439 440 token = pipe.tokenizer.convert_tokens_to_ids("<****>")441 token_1 = pipe.tokenizer.convert_tokens_to_ids("<****>_1")442 token_2 = pipe.tokenizer.convert_tokens_to_ids("<****>_2")443 444 assert token == num_tokens + 5, "Added token must be at spot `num_tokens`"445 assert token_1 == num_tokens + 6, "Added token must be at spot `num_tokens`"446 assert token_2 == num_tokens + 7, "Added token must be at spot `num_tokens`"447 assert pipe.text_encoder.get_input_embeddings().weight[-3].sum().item() == 96448 assert pipe.text_encoder.get_input_embeddings().weight[-2].sum().item() == 128449 assert pipe.text_encoder.get_input_embeddings().weight[-1].sum().item() == 160450 assert pipe._maybe_convert_prompt("<****>", pipe.tokenizer) == "<****><****>_1<****>_2"451 452 prompt = "hey <****>"453 out = pipe(prompt, num_inference_steps=1, output_type="numpy").images454 assert out.shape == (1, 128, 128, 3)455 456 457class CustomPipelineTests(unittest.TestCase):458 def test_load_custom_pipeline(self):459 pipeline = DiffusionPipeline.from_pretrained(460 "google/ddpm-cifar10-32", custom_pipeline="hf-internal-testing/diffusers-dummy-pipeline"461 )462 pipeline = pipeline.to(torch_device)463 # NOTE that `"CustomPipeline"` is not a class that is defined in this library, but solely on the Hub464 # under https://huggingface.co/hf-internal-testing/diffusers-dummy-pipeline/blob/main/pipeline.py#L24465 assert pipeline.__class__.__name__ == "CustomPipeline"466 467 def test_load_custom_github(self):468 pipeline = DiffusionPipeline.from_pretrained(469 "google/ddpm-cifar10-32", custom_pipeline="one_step_unet", custom_revision="main"470 )471 472 # make sure that on "main" pipeline gives only ones because of: https://github.com/huggingface/diffusers/pull/1690473 with torch.no_grad():474 output = pipeline()475 476 assert output.numel() == output.sum()477 478 # hack since Python doesn't like overwriting modules: https://stackoverflow.com/questions/3105801/unload-a-module-in-python479 # Could in the future work with hashes instead.480 del sys.modules["diffusers_modules.git.one_step_unet"]481 482 pipeline = DiffusionPipeline.from_pretrained(483 "google/ddpm-cifar10-32", custom_pipeline="one_step_unet", custom_revision="0.10.2"484 )485 with torch.no_grad():486 output = pipeline()487 488 assert output.numel() != output.sum()489 490 assert pipeline.__class__.__name__ == "UnetSchedulerOneForwardPipeline"491 492 def test_run_custom_pipeline(self):493 pipeline = DiffusionPipeline.from_pretrained(494 "google/ddpm-cifar10-32", custom_pipeline="hf-internal-testing/diffusers-dummy-pipeline"495 )496 pipeline = pipeline.to(torch_device)497 images, output_str = pipeline(num_inference_steps=2, output_type="np")498 499 assert images[0].shape == (1, 32, 32, 3)500 501 # compare output to https://huggingface.co/hf-internal-testing/diffusers-dummy-pipeline/blob/main/pipeline.py#L102502 assert output_str == "This is a test"503 504 def test_local_custom_pipeline_repo(self):505 local_custom_pipeline_path = get_tests_dir("fixtures/custom_pipeline")506 pipeline = DiffusionPipeline.from_pretrained(507 "google/ddpm-cifar10-32", custom_pipeline=local_custom_pipeline_path508 )509 pipeline = pipeline.to(torch_device)510 images, output_str = pipeline(num_inference_steps=2, output_type="np")511 512 assert pipeline.__class__.__name__ == "CustomLocalPipeline"513 assert images[0].shape == (1, 32, 32, 3)514 # compare to https://github.com/huggingface/diffusers/blob/main/tests/fixtures/custom_pipeline/pipeline.py#L102515 assert output_str == "This is a local test"516 517 def test_local_custom_pipeline_file(self):518 local_custom_pipeline_path = get_tests_dir("fixtures/custom_pipeline")519 local_custom_pipeline_path = os.path.join(local_custom_pipeline_path, "what_ever.py")520 pipeline = DiffusionPipeline.from_pretrained(521 "google/ddpm-cifar10-32", custom_pipeline=local_custom_pipeline_path522 )523 pipeline = pipeline.to(torch_device)524 images, output_str = pipeline(num_inference_steps=2, output_type="np")525 526 assert pipeline.__class__.__name__ == "CustomLocalPipeline"527 assert images[0].shape == (1, 32, 32, 3)528 # compare to https://github.com/huggingface/diffusers/blob/main/tests/fixtures/custom_pipeline/pipeline.py#L102529 assert output_str == "This is a local test"530 531 @slow532 @require_torch_gpu533 def test_download_from_git(self):534 clip_model_id = "laion/CLIP-ViT-B-32-laion2B-s34B-b79K"535 536 feature_extractor = CLIPImageProcessor.from_pretrained(clip_model_id)537 clip_model = CLIPModel.from_pretrained(clip_model_id, torch_dtype=torch.float16)538 539 pipeline = DiffusionPipeline.from_pretrained(540 "CompVis/stable-diffusion-v1-4",541 custom_pipeline="clip_guided_stable_diffusion",542 clip_model=clip_model,543 feature_extractor=feature_extractor,544 torch_dtype=torch.float16,545 )546 pipeline.enable_attention_slicing()547 pipeline = pipeline.to(torch_device)548 549 # NOTE that `"CLIPGuidedStableDiffusion"` is not a class that is defined in the pypi package of th e library, but solely on the community examples folder of GitHub under:550 # https://github.com/huggingface/diffusers/blob/main/examples/community/clip_guided_stable_diffusion.py551 assert pipeline.__class__.__name__ == "CLIPGuidedStableDiffusion"552 553 image = pipeline("a prompt", num_inference_steps=2, output_type="np").images[0]554 assert image.shape == (512, 512, 3)555 556 557class PipelineFastTests(unittest.TestCase):558 def tearDown(self):559 # clean up the VRAM after each test560 super().tearDown()561 gc.collect()562 torch.cuda.empty_cache()563 564 import diffusers565 566 diffusers.utils.import_utils._safetensors_available = True567 568 def dummy_image(self):569 batch_size = 1570 num_channels = 3571 sizes = (32, 32)572 573 image = floats_tensor((batch_size, num_channels) + sizes, rng=random.Random(0)).to(torch_device)574 return image575 576 def dummy_uncond_unet(self, sample_size=32):577 torch.manual_seed(0)578 model = UNet2DModel(579 block_out_channels=(32, 64),580 layers_per_block=2,581 sample_size=sample_size,582 in_channels=3,583 out_channels=3,584 down_block_types=("DownBlock2D", "AttnDownBlock2D"),585 up_block_types=("AttnUpBlock2D", "UpBlock2D"),586 )587 return model588 589 def dummy_cond_unet(self, sample_size=32):590 torch.manual_seed(0)591 model = UNet2DConditionModel(592 block_out_channels=(32, 64),593 layers_per_block=2,594 sample_size=sample_size,595 in_channels=4,596 out_channels=4,597 down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"),598 up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"),599 cross_attention_dim=32,600 )601 return model602 603 @property604 def dummy_vae(self):605 torch.manual_seed(0)606 model = AutoencoderKL(607 block_out_channels=[32, 64],608 in_channels=3,609 out_channels=3,610 down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"],611 up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"],612 latent_channels=4,613 )614 return model615 616 @property617 def dummy_text_encoder(self):618 torch.manual_seed(0)619 config = CLIPTextConfig(620 bos_token_id=0,621 eos_token_id=2,622 hidden_size=32,623 intermediate_size=37,624 layer_norm_eps=1e-05,625 num_attention_heads=4,626 num_hidden_layers=5,627 pad_token_id=1,628 vocab_size=1000,629 )630 return CLIPTextModel(config)631 632 @property633 def dummy_extractor(self):634 def extract(*args, **kwargs):635 class Out:636 def __init__(self):637 self.pixel_values = torch.ones([0])638 639 def to(self, device):640 self.pixel_values.to(device)641 return self642 643 return Out()644 645 return extract646 647 @parameterized.expand(648 [649 [DDIMScheduler, DDIMPipeline, 32],650 [DDPMScheduler, DDPMPipeline, 32],651 [DDIMScheduler, DDIMPipeline, (32, 64)],652 [DDPMScheduler, DDPMPipeline, (64, 32)],653 ]654 )655 def test_uncond_unet_components(self, scheduler_fn=DDPMScheduler, pipeline_fn=DDPMPipeline, sample_size=32):656 unet = self.dummy_uncond_unet(sample_size)657 scheduler = scheduler_fn()658 pipeline = pipeline_fn(unet, scheduler).to(torch_device)659 660 generator = torch.manual_seed(0)661 out_image = pipeline(662 generator=generator,663 num_inference_steps=2,664 output_type="np",665 ).images666 sample_size = (sample_size, sample_size) if isinstance(sample_size, int) else sample_size667 assert out_image.shape == (1, *sample_size, 3)668 669 def test_stable_diffusion_components(self):670 """Test that components property works correctly"""671 unet = self.dummy_cond_unet()672 scheduler = PNDMScheduler(skip_prk_steps=True)673 vae = self.dummy_vae674 bert = self.dummy_text_encoder675 tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")676 677 image = self.dummy_image().cpu().permute(0, 2, 3, 1)[0]678 init_image = Image.fromarray(np.uint8(image)).convert("RGB")679 mask_image = Image.fromarray(np.uint8(image + 4)).convert("RGB").resize((32, 32))680 681 # make sure here that pndm scheduler skips prk682 inpaint = StableDiffusionInpaintPipelineLegacy(683 unet=unet,684 scheduler=scheduler,685 vae=vae,686 text_encoder=bert,687 tokenizer=tokenizer,688 safety_checker=None,689 feature_extractor=self.dummy_extractor,690 ).to(torch_device)691 img2img = StableDiffusionImg2ImgPipeline(**inpaint.components).to(torch_device)692 text2img = StableDiffusionPipeline(**inpaint.components).to(torch_device)693 694 prompt = "A painting of a squirrel eating a burger"695 696 generator = torch.manual_seed(0)697 image_inpaint = inpaint(698 [prompt],699 generator=generator,700 num_inference_steps=2,701 output_type="np",702 image=init_image,703 mask_image=mask_image,704 ).images705 image_img2img = img2img(706 [prompt],707 generator=generator,708 num_inference_steps=2,709 output_type="np",710 image=init_image,711 ).images712 image_text2img = text2img(713 [prompt],714 generator=generator,715 num_inference_steps=2,716 output_type="np",717 ).images718 719 assert image_inpaint.shape == (1, 32, 32, 3)720 assert image_img2img.shape == (1, 32, 32, 3)721 assert image_text2img.shape == (1, 64, 64, 3)722 723 @require_torch_gpu724 def test_pipe_false_offload_warn(self):725 unet = self.dummy_cond_unet()726 scheduler = PNDMScheduler(skip_prk_steps=True)727 vae = self.dummy_vae728 bert = self.dummy_text_encoder729 tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")730 731 sd = StableDiffusionPipeline(732 unet=unet,733 scheduler=scheduler,734 vae=vae,735 text_encoder=bert,736 tokenizer=tokenizer,737 safety_checker=None,738 feature_extractor=self.dummy_extractor,739 )740 741 sd.enable_model_cpu_offload()742 743 logger = logging.get_logger("diffusers.pipelines.pipeline_utils")744 with CaptureLogger(logger) as cap_logger:745 sd.to("cuda")746 747 assert "It is strongly recommended against doing so" in str(cap_logger)748 749 sd = StableDiffusionPipeline(750 unet=unet,751 scheduler=scheduler,752 vae=vae,753 text_encoder=bert,754 tokenizer=tokenizer,755 safety_checker=None,756 feature_extractor=self.dummy_extractor,757 )758 759 def test_set_scheduler(self):760 unet = self.dummy_cond_unet()761 scheduler = PNDMScheduler(skip_prk_steps=True)762 vae = self.dummy_vae763 bert = self.dummy_text_encoder764 tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")765 766 sd = StableDiffusionPipeline(767 unet=unet,768 scheduler=scheduler,769 vae=vae,770 text_encoder=bert,771 tokenizer=tokenizer,772 safety_checker=None,773 feature_extractor=self.dummy_extractor,774 )775 776 sd.scheduler = DDIMScheduler.from_config(sd.scheduler.config)777 assert isinstance(sd.scheduler, DDIMScheduler)778 sd.scheduler = DDPMScheduler.from_config(sd.scheduler.config)779 assert isinstance(sd.scheduler, DDPMScheduler)780 sd.scheduler = PNDMScheduler.from_config(sd.scheduler.config)781 assert isinstance(sd.scheduler, PNDMScheduler)782 sd.scheduler = LMSDiscreteScheduler.from_config(sd.scheduler.config)783 assert isinstance(sd.scheduler, LMSDiscreteScheduler)784 sd.scheduler = EulerDiscreteScheduler.from_config(sd.scheduler.config)785 assert isinstance(sd.scheduler, EulerDiscreteScheduler)786 sd.scheduler = EulerAncestralDiscreteScheduler.from_config(sd.scheduler.config)787 assert isinstance(sd.scheduler, EulerAncestralDiscreteScheduler)788 sd.scheduler = DPMSolverMultistepScheduler.from_config(sd.scheduler.config)789 assert isinstance(sd.scheduler, DPMSolverMultistepScheduler)790 791 def test_set_scheduler_consistency(self):792 unet = self.dummy_cond_unet()793 pndm = PNDMScheduler.from_config("hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler")794 ddim = DDIMScheduler.from_config("hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler")795 vae = self.dummy_vae796 bert = self.dummy_text_encoder797 tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")798 799 sd = StableDiffusionPipeline(800 unet=unet,801 scheduler=pndm,802 vae=vae,803 text_encoder=bert,804 tokenizer=tokenizer,805 safety_checker=None,806 feature_extractor=self.dummy_extractor,807 )808 809 pndm_config = sd.scheduler.config810 sd.scheduler = DDPMScheduler.from_config(pndm_config)811 sd.scheduler = PNDMScheduler.from_config(sd.scheduler.config)812 pndm_config_2 = sd.scheduler.config813 pndm_config_2 = {k: v for k, v in pndm_config_2.items() if k in pndm_config}814 815 assert dict(pndm_config) == dict(pndm_config_2)816 817 sd = StableDiffusionPipeline(818 unet=unet,819 scheduler=ddim,820 vae=vae,821 text_encoder=bert,822 tokenizer=tokenizer,823 safety_checker=None,824 feature_extractor=self.dummy_extractor,825 )826 827 ddim_config = sd.scheduler.config828 sd.scheduler = LMSDiscreteScheduler.from_config(ddim_config)829 sd.scheduler = DDIMScheduler.from_config(sd.scheduler.config)830 ddim_config_2 = sd.scheduler.config831 ddim_config_2 = {k: v for k, v in ddim_config_2.items() if k in ddim_config}832 833 assert dict(ddim_config) == dict(ddim_config_2)834 835 def test_save_safe_serialization(self):836 pipeline = StableDiffusionPipeline.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch")837 with tempfile.TemporaryDirectory() as tmpdirname:838 pipeline.save_pretrained(tmpdirname, safe_serialization=True)839 840 # Validate that the VAE safetensor exists and are of the correct format841 vae_path = os.path.join(tmpdirname, "vae", "diffusion_pytorch_model.safetensors")842 assert os.path.exists(vae_path), f"Could not find {vae_path}"843 _ = safetensors.torch.load_file(vae_path)844 845 # Validate that the UNet safetensor exists and are of the correct format846 unet_path = os.path.join(tmpdirname, "unet", "diffusion_pytorch_model.safetensors")847 assert os.path.exists(unet_path), f"Could not find {unet_path}"848 _ = safetensors.torch.load_file(unet_path)849 850 # Validate that the text encoder safetensor exists and are of the correct format851 text_encoder_path = os.path.join(tmpdirname, "text_encoder", "model.safetensors")852 assert os.path.exists(text_encoder_path), f"Could not find {text_encoder_path}"853 _ = safetensors.torch.load_file(text_encoder_path)854 855 pipeline = StableDiffusionPipeline.from_pretrained(tmpdirname)856 assert pipeline.unet is not None857 assert pipeline.vae is not None858 assert pipeline.text_encoder is not None859 assert pipeline.scheduler is not None860 assert pipeline.feature_extractor is not None861 862 def test_no_pytorch_download_when_doing_safetensors(self):863 # by default we don't download864 with tempfile.TemporaryDirectory() as tmpdirname:865 _ = StableDiffusionPipeline.from_pretrained(866 "hf-internal-testing/diffusers-stable-diffusion-tiny-all", cache_dir=tmpdirname867 )868 869 path = os.path.join(870 tmpdirname,871 "models--hf-internal-testing--diffusers-stable-diffusion-tiny-all",872 "snapshots",873 "07838d72e12f9bcec1375b0482b80c1d399be843",874 "unet",875 )876 # safetensors exists877 assert os.path.exists(os.path.join(path, "diffusion_pytorch_model.safetensors"))878 # pytorch does not879 assert not os.path.exists(os.path.join(path, "diffusion_pytorch_model.bin"))880 881 def test_no_safetensors_download_when_doing_pytorch(self):882 # mock diffusers safetensors not available883 import diffusers884 885 diffusers.utils.import_utils._safetensors_available = False886 887 with tempfile.TemporaryDirectory() as tmpdirname:888 _ = StableDiffusionPipeline.from_pretrained(889 "hf-internal-testing/diffusers-stable-diffusion-tiny-all", cache_dir=tmpdirname890 )891 892 path = os.path.join(893 tmpdirname,894 "models--hf-internal-testing--diffusers-stable-diffusion-tiny-all",895 "snapshots",896 "07838d72e12f9bcec1375b0482b80c1d399be843",897 "unet",898 )899 # safetensors does not exists900 assert not os.path.exists(os.path.join(path, "diffusion_pytorch_model.safetensors"))901 # pytorch does902 assert os.path.exists(os.path.join(path, "diffusion_pytorch_model.bin"))903 904 diffusers.utils.import_utils._safetensors_available = True905 906 def test_optional_components(self):907 unet = self.dummy_cond_unet()908 pndm = PNDMScheduler.from_config("hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler")909 vae = self.dummy_vae910 bert = self.dummy_text_encoder911 tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")912 913 orig_sd = StableDiffusionPipeline(914 unet=unet,915 scheduler=pndm,916 vae=vae,917 text_encoder=bert,918 tokenizer=tokenizer,919 safety_checker=unet,920 feature_extractor=self.dummy_extractor,921 )922 sd = orig_sd923 924 assert sd.config.requires_safety_checker is True925 926 with tempfile.TemporaryDirectory() as tmpdirname:927 sd.save_pretrained(tmpdirname)928 929 # Test that passing None works930 sd = StableDiffusionPipeline.from_pretrained(931 tmpdirname, feature_extractor=None, safety_checker=None, requires_safety_checker=False932 )933 934 assert sd.config.requires_safety_checker is False935 assert sd.config.safety_checker == (None, None)936 assert sd.config.feature_extractor == (None, None)937 938 with tempfile.TemporaryDirectory() as tmpdirname:939 sd.save_pretrained(tmpdirname)940 941 # Test that loading previous None works942 sd = StableDiffusionPipeline.from_pretrained(tmpdirname)943 944 assert sd.config.requires_safety_checker is False945 assert sd.config.safety_checker == (None, None)946 assert sd.config.feature_extractor == (None, None)947 948 orig_sd.save_pretrained(tmpdirname)949 950 # Test that loading without any directory works951 shutil.rmtree(os.path.join(tmpdirname, "safety_checker"))952 with open(os.path.join(tmpdirname, sd.config_name)) as f:953 config = json.load(f)954 config["safety_checker"] = [None, None]955 with open(os.path.join(tmpdirname, sd.config_name), "w") as f:956 json.dump(config, f)957 958 sd = StableDiffusionPipeline.from_pretrained(tmpdirname, requires_safety_checker=False)959 sd.save_pretrained(tmpdirname)960 sd = StableDiffusionPipeline.from_pretrained(tmpdirname)961 962 assert sd.config.requires_safety_checker is False963 assert sd.config.safety_checker == (None, None)964 assert sd.config.feature_extractor == (None, None)965 966 # Test that loading from deleted model index works967 with open(os.path.join(tmpdirname, sd.config_name)) as f:968 config = json.load(f)969 del config["safety_checker"]970 del config["feature_extractor"]971 with open(os.path.join(tmpdirname, sd.config_name), "w") as f:972 json.dump(config, f)973 974 sd = StableDiffusionPipeline.from_pretrained(tmpdirname)975 976 assert sd.config.requires_safety_checker is False977 assert sd.config.safety_checker == (None, None)978 assert sd.config.feature_extractor == (None, None)979 980 with tempfile.TemporaryDirectory() as tmpdirname:981 sd.save_pretrained(tmpdirname)982 983 # Test that partially loading works984 sd = StableDiffusionPipeline.from_pretrained(tmpdirname, feature_extractor=self.dummy_extractor)985 986 assert sd.config.requires_safety_checker is False987 assert sd.config.safety_checker == (None, None)988 assert sd.config.feature_extractor != (None, None)989 990 # Test that partially loading works991 sd = StableDiffusionPipeline.from_pretrained(992 tmpdirname,993 feature_extractor=self.dummy_extractor,994 safety_checker=unet,995 requires_safety_checker=[True, True],996 )997 998 assert sd.config.requires_safety_checker == [True, True]999 assert sd.config.safety_checker != (None, None)1000 assert sd.config.feature_extractor != (None, None)1001 1002 with tempfile.TemporaryDirectory() as tmpdirname:1003 sd.save_pretrained(tmpdirname)1004 sd = StableDiffusionPipeline.from_pretrained(tmpdirname, feature_extractor=self.dummy_extractor)1005 1006 assert sd.config.requires_safety_checker == [True, True]1007 assert sd.config.safety_checker != (None, None)1008 assert sd.config.feature_extractor != (None, None)1009 1010 1011@slow1012@require_torch_gpu1013class PipelineSlowTests(unittest.TestCase):1014 def tearDown(self):1015 # clean up the VRAM after each test1016 super().tearDown()1017 gc.collect()1018 torch.cuda.empty_cache()1019 1020 def test_smart_download(self):1021 model_id = "hf-internal-testing/unet-pipeline-dummy"1022 with tempfile.TemporaryDirectory() as tmpdirname:1023 _ = DiffusionPipeline.from_pretrained(model_id, cache_dir=tmpdirname, force_download=True)1024 local_repo_name = "--".join(["models"] + model_id.split("/"))1025 snapshot_dir = os.path.join(tmpdirname, local_repo_name, "snapshots")1026 snapshot_dir = os.path.join(snapshot_dir, os.listdir(snapshot_dir)[0])1027 1028 # inspect all downloaded files to make sure that everything is included1029 assert os.path.isfile(os.path.join(snapshot_dir, DiffusionPipeline.config_name))1030 assert os.path.isfile(os.path.join(snapshot_dir, CONFIG_NAME))1031 assert os.path.isfile(os.path.join(snapshot_dir, SCHEDULER_CONFIG_NAME))1032 assert os.path.isfile(os.path.join(snapshot_dir, WEIGHTS_NAME))1033 assert os.path.isfile(os.path.join(snapshot_dir, "scheduler", SCHEDULER_CONFIG_NAME))1034 assert os.path.isfile(os.path.join(snapshot_dir, "unet", WEIGHTS_NAME))1035 assert os.path.isfile(os.path.join(snapshot_dir, "unet", WEIGHTS_NAME))1036 # let's make sure the super large numpy file:1037 # https://huggingface.co/hf-internal-testing/unet-pipeline-dummy/blob/main/big_array.npy1038 # is not downloaded, but all the expected ones1039 assert not os.path.isfile(os.path.join(snapshot_dir, "big_array.npy"))1040 1041 def test_warning_unused_kwargs(self):1042 model_id = "hf-internal-testing/unet-pipeline-dummy"1043 logger = logging.get_logger("diffusers.pipelines")1044 with tempfile.TemporaryDirectory() as tmpdirname:1045 with CaptureLogger(logger) as cap_logger:1046 DiffusionPipeline.from_pretrained(1047 model_id,1048 not_used=True,1049 cache_dir=tmpdirname,1050 force_download=True,1051 )1052 1053 assert (1054 cap_logger.out.strip().split("\n")[-1]1055 == "Keyword arguments {'not_used': True} are not expected by DDPMPipeline and will be ignored."1056 )1057 1058 def test_from_save_pretrained(self):1059 # 1. Load models1060 model = UNet2DModel(1061 block_out_channels=(32, 64),1062 layers_per_block=2,1063 sample_size=32,1064 in_channels=3,1065 out_channels=3,1066 down_block_types=("DownBlock2D", "AttnDownBlock2D"),1067 up_block_types=("AttnUpBlock2D", "UpBlock2D"),1068 )1069 scheduler = DDPMScheduler(num_train_timesteps=10)1070 1071 ddpm = DDPMPipeline(model, scheduler)1072 ddpm.to(torch_device)1073 ddpm.set_progress_bar_config(disable=None)1074 1075 with tempfile.TemporaryDirectory() as tmpdirname:1076 ddpm.save_pretrained(tmpdirname)1077 new_ddpm = DDPMPipeline.from_pretrained(tmpdirname)1078 new_ddpm.to(torch_device)1079 1080 generator = torch.Generator(device=torch_device).manual_seed(0)1081 image = ddpm(generator=generator, num_inference_steps=5, output_type="numpy").images1082 1083 generator = torch.Generator(device=torch_device).manual_seed(0)1084 new_image = new_ddpm(generator=generator, num_inference_steps=5, output_type="numpy").images1085 1086 assert np.abs(image - new_image).sum() < 1e-5, "Models don't give the same forward pass"1087 1088 @require_torch_21089 def test_from_save_pretrained_dynamo(self):1090 # 1. Load models1091 model = UNet2DModel(1092 block_out_channels=(32, 64),1093 layers_per_block=2,1094 sample_size=32,1095 in_channels=3,1096 out_channels=3,1097 down_block_types=("DownBlock2D", "AttnDownBlock2D"),1098 up_block_types=("AttnUpBlock2D", "UpBlock2D"),1099 )1100 model = torch.compile(model)1101 scheduler = DDPMScheduler(num_train_timesteps=10)1102 1103 ddpm = DDPMPipeline(model, scheduler)1104 ddpm.to(torch_device)1105 ddpm.set_progress_bar_config(disable=None)1106 1107 with tempfile.TemporaryDirectory() as tmpdirname:1108 ddpm.save_pretrained(tmpdirname)1109 new_ddpm = DDPMPipeline.from_pretrained(tmpdirname)1110 new_ddpm.to(torch_device)1111 1112 generator = torch.Generator(device=torch_device).manual_seed(0)1113 image = ddpm(generator=generator, num_inference_steps=5, output_type="numpy").images1114 1115 generator = torch.Generator(device=torch_device).manual_seed(0)1116 new_image = new_ddpm(generator=generator, num_inference_steps=5, output_type="numpy").images1117 1118 assert np.abs(image - new_image).sum() < 1e-5, "Models don't give the same forward pass"1119 1120 def test_from_pretrained_hub(self):1121 model_path = "google/ddpm-cifar10-32"1122 1123 scheduler = DDPMScheduler(num_train_timesteps=10)1124 1125 ddpm = DDPMPipeline.from_pretrained(model_path, scheduler=scheduler)1126 ddpm = ddpm.to(torch_device)1127 ddpm.set_progress_bar_config(disable=None)1128 1129 ddpm_from_hub = DiffusionPipeline.from_pretrained(model_path, scheduler=scheduler)1130 ddpm_from_hub = ddpm_from_hub.to(torch_device)1131 ddpm_from_hub.set_progress_bar_config(disable=None)1132 1133 generator = torch.Generator(device=torch_device).manual_seed(0)1134 image = ddpm(generator=generator, num_inference_steps=5, output_type="numpy").images1135 1136 generator = torch.Generator(device=torch_device).manual_seed(0)1137 new_image = ddpm_from_hub(generator=generator, num_inference_steps=5, output_type="numpy").images1138 1139 assert np.abs(image - new_image).sum() < 1e-5, "Models don't give the same forward pass"1140 1141 def test_from_pretrained_hub_pass_model(self):1142 model_path = "google/ddpm-cifar10-32"1143 1144 scheduler = DDPMScheduler(num_train_timesteps=10)1145 1146 # pass unet into DiffusionPipeline1147 unet = UNet2DModel.from_pretrained(model_path)1148 ddpm_from_hub_custom_model = DiffusionPipeline.from_pretrained(model_path, unet=unet, scheduler=scheduler)1149 ddpm_from_hub_custom_model = ddpm_from_hub_custom_model.to(torch_device)1150 ddpm_from_hub_custom_model.set_progress_bar_config(disable=None)1151 1152 ddpm_from_hub = DiffusionPipeline.from_pretrained(model_path, scheduler=scheduler)1153 ddpm_from_hub = ddpm_from_hub.to(torch_device)1154 ddpm_from_hub_custom_model.set_progress_bar_config(disable=None)1155 1156 generator = torch.Generator(device=torch_device).manual_seed(0)1157 image = ddpm_from_hub_custom_model(generator=generator, num_inference_steps=5, output_type="numpy").images1158 1159 generator = torch.Generator(device=torch_device).manual_seed(0)1160 new_image = ddpm_from_hub(generator=generator, num_inference_steps=5, output_type="numpy").images1161 1162 assert np.abs(image - new_image).sum() < 1e-5, "Models don't give the same forward pass"1163 1164 def test_output_format(self):1165 model_path = "google/ddpm-cifar10-32"1166 1167 scheduler = DDIMScheduler.from_pretrained(model_path)1168 pipe = DDIMPipeline.from_pretrained(model_path, scheduler=scheduler)1169 pipe.to(torch_device)1170 pipe.set_progress_bar_config(disable=None)1171 1172 images = pipe(output_type="numpy").images1173 assert images.shape == (1, 32, 32, 3)1174 assert isinstance(images, np.ndarray)1175 1176 images = pipe(output_type="pil", num_inference_steps=4).images1177 assert isinstance(images, list)1178 assert len(images) == 11179 assert isinstance(images[0], PIL.Image.Image)1180 1181 # use PIL by default1182 images = pipe(num_inference_steps=4).images1183 assert isinstance(images, list)1184 assert isinstance(images[0], PIL.Image.Image)1185 1186 def test_from_flax_from_pt(self):1187 pipe_pt = StableDiffusionPipeline.from_pretrained(1188 "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None1189 )1190 pipe_pt.to(torch_device)1191 1192 if not is_flax_available():1193 raise ImportError("Make sure flax is installed.")1194 1195 from diffusers import FlaxStableDiffusionPipeline1196 1197 with tempfile.TemporaryDirectory() as tmpdirname:1198 pipe_pt.save_pretrained(tmpdirname)1199 1200 pipe_flax, params = FlaxStableDiffusionPipeline.from_pretrained(