edessa/EMG

1import os2import sys3 4import numpy as np5import torch6from diffusers import FlowMatchEulerDiscreteScheduler7from omegaconf import OmegaConf8from PIL import Image9 10current_file_path = os.path.abspath(__file__)11project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))]12for project_root in project_roots:13    sys.path.insert(0, project_root) if project_root not in sys.path else None14 15from videox_fun.dist import set_multi_gpus_devices, shard_model16from videox_fun.models import (AutoencoderKL, AutoTokenizer, Qwen3ForCausalLM,17                               ZImageControlTransformer2DModel)18from videox_fun.models.cache_utils import get_teacache_coefficients19from videox_fun.pipeline import ZImageControlPipeline20from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler21from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler22from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8,23                                               convert_weight_dtype_wrapper)24from videox_fun.utils.lora_utils import merge_lora, unmerge_lora25from videox_fun.utils.utils import (filter_kwargs, get_image, get_image_latent,26                                    get_image_to_video_latent,27                                    get_video_to_video_latent,28                                    save_videos_grid)29 30# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload].31# model_full_load means that the entire model will be moved to the GPU.32# 33# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU,34# and the transformer model has been quantized to float8, which can save more GPU memory. 35# 36# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory.37# 38# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, 39# and the transformer model has been quantized to float8, which can save more GPU memory. 40# 41# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, 42# resulting in slower speeds but saving a large amount of GPU memory.43GPU_memory_mode     = "model_cpu_offload"44# Multi GPUs config45# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. 46# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4.47# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1.48ulysses_degree      = 149ring_degree         = 150# Use FSDP to save more GPU memory in multi gpus.51fsdp_dit            = False52fsdp_text_encoder   = False53# Compile will give a speedup in fixed resolution and need a little GPU memory. 54# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload.55compile_dit         = False56 57# Config and model path58config_path         = "config/z_image/z_image_control_2.1.yaml"59# model path60model_name          = "models/Diffusion_Transformer/Z-Image"61 62# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++"63sampler_name        = "Flow"64 65# Load pretrained model if need66transformer_path    = "models/Personalized_Model/Z-Image-Fun-Controlnet-Tile-2.1.safetensors" 67vae_path            = None68lora_path           = None69 70# Other params71sample_size         = [2048, 2048]72 73# Use torch.float16 if GPU does not support torch.bfloat1674# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat1675weight_dtype        = torch.bfloat1676control_image       = "asset/low_res.png"77# The inpaint_image and mask_image is useless in tile model, just set them to None.78inpaint_image       = None79mask_image          = None80control_context_scale = 0.8581 82# Please use as detailed a prompt as possible to describe the object that needs to be generated.83prompt              = "这是一张充满都市气息的户外人物肖像照片。画面中是一位年轻男性，他展现出时尚而自信的形象。人物拥有精心打理的短发发型，两侧修剪得较短，顶部保留一定长度，呈现出流行的Undercut造型。他佩戴着一副时尚的浅色墨镜或透明镜框眼镜，为整体造型增添了潮流感。脸上洋溢着温和友善的笑容，神情放松自然，给人以阳光开朗的印象。他身穿一件经典的牛仔外套，这件单品永不过时，展现出休闲又有型的穿衣风格。牛仔外套的蓝色调与整体氛围十分协调，领口处隐约可见内搭的衣物。照片的背景是典型的城市街景，可以看到模糊的建筑物、街道和行人，营造出繁华都市的氛围。背景经过了恰当的虚化处理，使人物主体更加突出。光线明亮而柔和，可能是白天的自然光，为照片带来清新通透的视觉效果。整张照片构图专业，景深控制得当，完美捕捉了一个现代都市年轻人充满活力和自信的瞬间，展现出积极向上的生活态度。"84negative_prompt     = "低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。构图混乱。文字模糊，扭曲。"85guidance_scale      = 4.086seed                = 4387num_inference_steps = 2088lora_weight         = 0.5589save_path           = "samples/z-image-t2i-control"90 91device = set_multi_gpus_devices(ulysses_degree, ring_degree)92config = OmegaConf.load(config_path)93 94transformer = ZImageControlTransformer2DModel.from_pretrained(95    model_name, 96    subfolder="transformer",97    low_cpu_mem_usage=True,98    torch_dtype=weight_dtype,99    transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']),100).to(weight_dtype)101 102if transformer_path is not None:103    print(f"From checkpoint: {transformer_path}")104    if transformer_path.endswith("safetensors"):105        from safetensors.torch import load_file, safe_open106        state_dict = load_file(transformer_path)107    else:108        state_dict = torch.load(transformer_path, map_location="cpu")109    state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict110 111    m, u = transformer.load_state_dict(state_dict, strict=False)112    print(f"missing keys: {len(m)}, unexpected keys: {len(u)}")113 114# Get Vae115vae = AutoencoderKL.from_pretrained(116    model_name, 117    subfolder="vae"118).to(weight_dtype)119 120if vae_path is not None:121    print(f"From checkpoint: {vae_path}")122    if vae_path.endswith("safetensors"):123        from safetensors.torch import load_file, safe_open124        state_dict = load_file(vae_path)125    else:126        state_dict = torch.load(vae_path, map_location="cpu")127    state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict128 129    m, u = vae.load_state_dict(state_dict, strict=False)130    print(f"missing keys: {len(m)}, unexpected keys: {len(u)}")131 132# Get tokenizer and text_encoder133tokenizer = AutoTokenizer.from_pretrained(134    model_name, subfolder="tokenizer"135)136text_encoder = Qwen3ForCausalLM.from_pretrained(137    model_name, subfolder="text_encoder", torch_dtype=weight_dtype,138    low_cpu_mem_usage=True,139)140 141# Get Scheduler142Chosen_Scheduler = scheduler_dict = {143    "Flow": FlowMatchEulerDiscreteScheduler,144    "Flow_Unipc": FlowUniPCMultistepScheduler,145    "Flow_DPM++": FlowDPMSolverMultistepScheduler,146}[sampler_name]147scheduler = Chosen_Scheduler.from_pretrained(148    model_name, 149    subfolder="scheduler"150)151 152pipeline = ZImageControlPipeline(153    vae=vae,154    tokenizer=tokenizer,155    text_encoder=text_encoder,156    transformer=transformer,157    scheduler=scheduler,158)159 160if ulysses_degree > 1 or ring_degree > 1:161    from functools import partial162    transformer.enable_multi_gpus_inference()163    if fsdp_dit:164        shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.layers))165        pipeline.transformer = shard_fn(pipeline.transformer)166        print("Add FSDP DIT")167    if fsdp_text_encoder:168        shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(text_encoder.model.layers))169        text_encoder = shard_fn(text_encoder)170        print("Add FSDP TEXT ENCODER")171 172if compile_dit:173    for i in range(len(pipeline.transformer.transformer_blocks)):174        pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i])175    print("Add Compile")176 177if GPU_memory_mode == "sequential_cpu_offload":178    pipeline.enable_sequential_cpu_offload(device=device)179elif GPU_memory_mode == "model_cpu_offload_and_qfloat8":180    convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device)181    convert_weight_dtype_wrapper(transformer, weight_dtype)182    pipeline.enable_model_cpu_offload(device=device)183elif GPU_memory_mode == "model_cpu_offload":184    pipeline.enable_model_cpu_offload(device=device)185elif GPU_memory_mode == "model_full_load_and_qfloat8":186    convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device)187    convert_weight_dtype_wrapper(transformer, weight_dtype)188    pipeline.to(device=device)189else:190    pipeline.to(device=device)191 192generator = torch.Generator(device=device).manual_seed(seed)193 194if lora_path is not None:195    pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype)196 197with torch.no_grad():198    if inpaint_image is not None:199        inpaint_image = get_image_latent(inpaint_image, sample_size=sample_size)[:, :, 0]200    else:201        inpaint_image = torch.zeros([1, 3, sample_size[0], sample_size[1]])202 203    if mask_image is not None:204        mask_image = get_image_latent(mask_image, sample_size=sample_size)[:, :1, 0]205    else:206        mask_image = torch.ones([1, 1, sample_size[0], sample_size[1]]) * 255207 208    if control_image is not None:209        control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0]210 211    sample = pipeline(212        prompt      = prompt, 213        negative_prompt = negative_prompt,214        height      = sample_size[0],215        width       = sample_size[1],216        generator   = generator,217        guidance_scale = guidance_scale,218        image               = inpaint_image,219        mask_image          = mask_image,220        control_image       = control_image,221        num_inference_steps = num_inference_steps,222        control_context_scale = control_context_scale,223    ).images224 225if lora_path is not None:226    pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype)227 228def save_results():229    if not os.path.exists(save_path):230        os.makedirs(save_path, exist_ok=True)231 232    index = len([path for path in os.listdir(save_path)]) + 1233    prefix = str(index).zfill(8)234    video_path = os.path.join(save_path, prefix + ".png")235    image = sample[0]236    image.save(video_path)237 238if ulysses_degree * ring_degree > 1:239    import torch.distributed as dist240    if dist.get_rank() == 0:241        save_results()242else:243    save_results()