Note1/LTX-2.3-Lora
0
1import os2import subprocess3import sys4 5# Disable torch.compile / dynamo before any torch import6os.environ["TORCH_COMPILE_DISABLE"] = "1"7os.environ["TORCHDYNAMO_DISABLE"] = "1"8 9# Install xformers for memory-efficient attention10subprocess.run([sys.executable, "-m", "pip", "install", "xformers==0.0.32.post2", "--no-build-isolation"], check=False)11 12# Clone LTX-2 repo and install packages13LTX_REPO_URL = "https://github.com/Lightricks/LTX-2.git"14LTX_REPO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "LTX-2")15 16LTX_COMMIT = "ae855f8538843825f9015a419cf4ba5edaf5eec2" # known working commit with decode_video17 18if not os.path.exists(LTX_REPO_DIR):19 print(f"Cloning {LTX_REPO_URL}...")20 subprocess.run(["git", "clone", LTX_REPO_URL, LTX_REPO_DIR], check=True)21 subprocess.run(["git", "checkout", LTX_COMMIT], cwd=LTX_REPO_DIR, check=True)22 23print("Installing ltx-core and ltx-pipelines from cloned repo...")24subprocess.run(25 [sys.executable, "-m", "pip", "install", "--force-reinstall", "--no-deps", "-e",26 os.path.join(LTX_REPO_DIR, "packages", "ltx-core"),27 "-e", os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines")],28 check=True,29)30 31sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines", "src"))32sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-core", "src"))33 34import logging35import random36import tempfile37from pathlib import Path38import gc39import hashlib40 41import torch42torch._dynamo.config.suppress_errors = True43torch._dynamo.config.disable = True44 45import spaces46import gradio as gr47import numpy as np48from huggingface_hub import hf_hub_download, snapshot_download49 50from ltx_core.components.diffusion_steps import EulerDiffusionStep51from ltx_core.components.noisers import GaussianNoiser52from ltx_core.model.audio_vae import encode_audio as vae_encode_audio53from ltx_core.model.upsampler import upsample_video54from ltx_core.model.video_vae import TilingConfig, get_video_chunks_number, decode_video as vae_decode_video55from ltx_core.quantization import QuantizationPolicy56from ltx_core.types import Audio, AudioLatentShape, VideoPixelShape57from ltx_pipelines.distilled import DistilledPipeline58from ltx_pipelines.utils import euler_denoising_loop59from ltx_pipelines.utils.args import ImageConditioningInput60from ltx_pipelines.utils.constants import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES61from ltx_pipelines.utils.helpers import (62 cleanup_memory,63 combined_image_conditionings,64 denoise_video_only,65 encode_prompts,66 simple_denoising_func,67)68from ltx_pipelines.utils.media_io import decode_audio_from_file, encode_video69from ltx_core.loader.primitives import LoraPathStrengthAndSDOps70from ltx_core.loader.sd_ops import LTXV_LORA_COMFY_RENAMING_MAP71 72# Force-patch xformers attention into the LTX attention module.73from ltx_core.model.transformer import attention as _attn_mod74print(f"[ATTN] Before patch: memory_efficient_attention={_attn_mod.memory_efficient_attention}")75try:76 from xformers.ops import memory_efficient_attention as _mea77 _attn_mod.memory_efficient_attention = _mea78 print(f"[ATTN] After patch: memory_efficient_attention={_attn_mod.memory_efficient_attention}")79except Exception as e:80 print(f"[ATTN] xformers patch FAILED: {type(e).__name__}: {e}")81 82logging.getLogger().setLevel(logging.INFO)83 84MAX_SEED = np.iinfo(np.int32).max85DEFAULT_PROMPT = (86 "An astronaut hatches from a fragile egg on the surface of the Moon, "87 "the shell cracking and peeling apart in gentle low-gravity motion. "88 "Fine lunar dust lifts and drifts outward with each movement, floating "89 "in slow arcs before settling back onto the ground."90)91DEFAULT_FRAME_RATE = 24.092 93# Resolution presets: (width, height)94RESOLUTIONS = {95 "high": {"16:9": (1536, 1024), "9:16": (1024, 1536), "1:1": (1024, 1024), "9:7": (1408, 1088), "7:9": (1088, 1408), "19:13": (1472, 1008), "13:19": (1008, 1472)},96 "low": {"16:9": (768, 512), "9:16": (512, 768), "1:1": (768, 768), "9:7": (704, 544), "7:9": (544, 704), "19:13": (736, 504), "13:19": (504, 736)},97}98 99 100class LTX23DistilledA2VPipeline(DistilledPipeline):101 """DistilledPipeline with optional audio conditioning."""102 103 def __call__(104 self,105 prompt: str,106 seed: int,107 height: int,108 width: int,109 num_frames: int,110 frame_rate: float,111 images: list[ImageConditioningInput],112 audio_path: str | None = None,113 tiling_config: TilingConfig | None = None,114 enhance_prompt: bool = False,115 ):116 # Standard path when no audio input is provided.117 print(prompt)118 if audio_path is None:119 return super().__call__(120 prompt=prompt,121 seed=seed,122 height=height,123 width=width,124 num_frames=num_frames,125 frame_rate=frame_rate,126 images=images,127 tiling_config=tiling_config,128 enhance_prompt=enhance_prompt,129 )130 131 generator = torch.Generator(device=self.device).manual_seed(seed)132 noiser = GaussianNoiser(generator=generator)133 stepper = EulerDiffusionStep()134 dtype = torch.bfloat16135 136 (ctx_p,) = encode_prompts(137 [prompt],138 self.model_ledger,139 enhance_first_prompt=enhance_prompt,140 enhance_prompt_image=images[0].path if len(images) > 0 else None,141 )142 video_context, audio_context = ctx_p.video_encoding, ctx_p.audio_encoding143 144 video_duration = num_frames / frame_rate145 decoded_audio = decode_audio_from_file(audio_path, self.device, 0.0, video_duration)146 if decoded_audio is None:147 raise ValueError(f"Could not extract audio stream from {audio_path}")148 149 encoded_audio_latent = vae_encode_audio(decoded_audio, self.model_ledger.audio_encoder())150 audio_shape = AudioLatentShape.from_duration(batch=1, duration=video_duration, channels=8, mel_bins=16)151 expected_frames = audio_shape.frames152 actual_frames = encoded_audio_latent.shape[2]153 154 if actual_frames > expected_frames:155 encoded_audio_latent = encoded_audio_latent[:, :, :expected_frames, :]156 elif actual_frames < expected_frames:157 pad = torch.zeros(158 encoded_audio_latent.shape[0],159 encoded_audio_latent.shape[1],160 expected_frames - actual_frames,161 encoded_audio_latent.shape[3],162 device=encoded_audio_latent.device,163 dtype=encoded_audio_latent.dtype,164 )165 encoded_audio_latent = torch.cat([encoded_audio_latent, pad], dim=2)166 167 video_encoder = self.model_ledger.video_encoder()168 transformer = self.model_ledger.transformer()169 stage_1_sigmas = torch.tensor(DISTILLED_SIGMA_VALUES, device=self.device)170 171 def denoising_loop(sigmas, video_state, audio_state, stepper):172 return euler_denoising_loop(173 sigmas=sigmas,174 video_state=video_state,175 audio_state=audio_state,176 stepper=stepper,177 denoise_fn=simple_denoising_func(178 video_context=video_context,179 audio_context=audio_context,180 transformer=transformer,181 ),182 )183 184 stage_1_output_shape = VideoPixelShape(185 batch=1,186 frames=num_frames,187 width=width // 2,188 height=height // 2,189 fps=frame_rate,190 )191 stage_1_conditionings = combined_image_conditionings(192 images=images,193 height=stage_1_output_shape.height,194 width=stage_1_output_shape.width,195 video_encoder=video_encoder,196 dtype=dtype,197 device=self.device,198 )199 video_state = denoise_video_only(200 output_shape=stage_1_output_shape,201 conditionings=stage_1_conditionings,202 noiser=noiser,203 sigmas=stage_1_sigmas,204 stepper=stepper,205 denoising_loop_fn=denoising_loop,206 components=self.pipeline_components,207 dtype=dtype,208 device=self.device,209 initial_audio_latent=encoded_audio_latent,210 )211 212 torch.cuda.synchronize()213 cleanup_memory()214 215 upscaled_video_latent = upsample_video(216 latent=video_state.latent[:1],217 video_encoder=video_encoder,218 upsampler=self.model_ledger.spatial_upsampler(),219 )220 stage_2_sigmas = torch.tensor(STAGE_2_DISTILLED_SIGMA_VALUES, device=self.device)221 stage_2_output_shape = VideoPixelShape(batch=1, frames=num_frames, width=width, height=height, fps=frame_rate)222 stage_2_conditionings = combined_image_conditionings(223 images=images,224 height=stage_2_output_shape.height,225 width=stage_2_output_shape.width,226 video_encoder=video_encoder,227 dtype=dtype,228 device=self.device,229 )230 video_state = denoise_video_only(231 output_shape=stage_2_output_shape,232 conditionings=stage_2_conditionings,233 noiser=noiser,234 sigmas=stage_2_sigmas,235 stepper=stepper,236 denoising_loop_fn=denoising_loop,237 components=self.pipeline_components,238 dtype=dtype,239 device=self.device,240 noise_scale=stage_2_sigmas[0],241 initial_video_latent=upscaled_video_latent,242 initial_audio_latent=encoded_audio_latent,243 )244 245 torch.cuda.synchronize()246 del transformer247 del video_encoder248 cleanup_memory()249 250 decoded_video = vae_decode_video(251 video_state.latent,252 self.model_ledger.video_decoder(),253 tiling_config,254 generator,255 )256 original_audio = Audio(257 waveform=decoded_audio.waveform.squeeze(0),258 sampling_rate=decoded_audio.sampling_rate,259 )260 return decoded_video, original_audio261 262 263# Model repos264LTX_MODEL_REPO = "Lightricks/LTX-2.3"265GEMMA_REPO ="Lightricks/gemma-3-12b-it-qat-q4_0-unquantized"266 267 268# Download model checkpoints269print("=" * 80)270print("Downloading LTX-2.3 distilled model + Gemma...")271print("=" * 80)272 273# LoRA cache directory and currently-applied key274LORA_CACHE_DIR = Path("lora_cache")275LORA_CACHE_DIR.mkdir(exist_ok=True)276current_lora_key: str | None = None277 278PENDING_LORA_KEY: str | None = None279PENDING_LORA_STATE: dict[str, torch.Tensor] | None = None280PENDING_LORA_STATUS: str = "No LoRA state prepared yet."281 282weights_dir = Path("weights")283weights_dir.mkdir(exist_ok=True)284checkpoint_path = hf_hub_download(285 repo_id=LTX_MODEL_REPO,286 filename="ltx-2.3-22b-distilled.safetensors",287 local_dir=str(weights_dir),288 local_dir_use_symlinks=False,289)290spatial_upsampler_path = hf_hub_download(repo_id=LTX_MODEL_REPO, filename="ltx-2.3-spatial-upscaler-x2-1.0.safetensors")291gemma_root = snapshot_download(repo_id=GEMMA_REPO)292 293# ---- Insert block (LoRA downloads) between lines 268 and 269 ----294# LoRA repo + download the requested LoRA adapters295LORA_REPO = "rahul7star/Ltx-2-3-Lora-Collection"296 297print("=" * 80)298print("Downloading LoRA adapters from dagloop5/LoRA...")299print("=" * 80)300paste_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="ltx23-kabapaste.safetensors")301ruri_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX23-ruri.safetensors")302transit_lora_path = hf_hub_download(repo_id="valiantcat/LTX-2.3-Transition-LORA", filename="ltx2.3-transition.safetensors")303pose_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2_3_NSFW_furry_concat_v2.safetensors")304general_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2.3_Reasoning_V1.safetensors")305motion_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="motion_helper.safetensors")306dreamlay_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="DR34ML4Y_LTXXX_PREVIEW_RC1.safetensors") # m15510n4ry, bl0wj0b, d0ubl3_bj, d0gg1e, c0wg1rl307mself_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="Furry Hyper Masturbation - LTX-2 I2V v1.safetensors") # Hyperfap308dramatic_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX-2.3 - Orgasm.safetensors") # "[He | She] is having am orgasm." (am or an?)309fluid_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="cr3ampi3_animation_i2v_ltx2_v1.0.safetensors") # cr3ampi3 animation., missionary animation, doggystyle bouncy animation, double penetration animation310liquid_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="liquid_wet_dr1pp_ltx2_v1.0_scaled.safetensors") # wet dr1pp311demopose_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="ltx23__demopose_d3m0p0s3.safetensors") # d3m0p0s3312 313print(f"Paste LoRA: {paste_lora_path }")314print(f"Ruri LoRA: {ruri_lora_path }")315print(f"Transit LoRA: {transit_lora_path }")316print(f"Pose LoRA: {pose_lora_path}")317print(f"General LoRA: {general_lora_path}")318print(f"Motion LoRA: {motion_lora_path}")319print(f"Dreamlay LoRA: {dreamlay_lora_path}")320print(f"Mself LoRA: {mself_lora_path}")321print(f"Dramatic LoRA: {dramatic_lora_path}")322print(f"Fluid LoRA: {fluid_lora_path}")323print(f"Liquid LoRA: {liquid_lora_path}")324print(f"Demopose LoRA: {demopose_lora_path}")325# ----------------------------------------------------------------326 327print(f"Checkpoint: {checkpoint_path}")328print(f"Spatial upsampler: {spatial_upsampler_path}")329print(f"Gemma root: {gemma_root}")330 331# Initialize pipeline WITH text encoder and optional audio support332# ---- Replace block (pipeline init) lines 275-281 ----333pipeline = LTX23DistilledA2VPipeline(334 distilled_checkpoint_path=checkpoint_path,335 spatial_upsampler_path=spatial_upsampler_path,336 gemma_root=gemma_root,337 loras=[],338 quantization=QuantizationPolicy.fp8_cast(), # keep FP8 quantization unchanged339)340# ----------------------------------------------------------------341 342def _make_lora_key(paste_strength: float,ruri_strength: float,transit_strength: float,pose_strength: float, general_strength: float, motion_strength: float, dreamlay_strength: float, mself_strength: float, dramatic_strength: float, fluid_strength: float, liquid_strength: float, demopose_strength: float) -> tuple[str, str]:343 ps = round(float(paste_strength), 2)344 rr = round(float(ruri_strength), 2)345 rt = round(float(transit_strength), 2)346 rp = round(float(pose_strength), 2)347 rg = round(float(general_strength), 2)348 rm = round(float(motion_strength), 2)349 rd = round(float(dreamlay_strength), 2)350 rs = round(float(mself_strength), 2)351 rr = round(float(dramatic_strength), 2)352 rf = round(float(fluid_strength), 2)353 rl = round(float(liquid_strength), 2)354 ro = round(float(demopose_strength), 2)355 key_str = f"{paste_lora_path}:{ps}|{ruri_lora_path}:{rr}|{transit_lora_path}:{rt}|{pose_lora_path}:{rp}|{general_lora_path}:{rg}|{motion_lora_path}:{rm}|{dreamlay_lora_path}:{rd}|{mself_lora_path}:{rs}|{dramatic_lora_path}:{rr}|{fluid_lora_path}:{rf}|{liquid_lora_path}:{rl}|{demopose_lora_path}:{ro}"356 key = hashlib.sha256(key_str.encode("utf-8")).hexdigest()357 return key, key_str358 359 360def prepare_lora_cache(361 paste_strength: float,362 ruri_strength: float,363 transit_strength: float,364 pose_strength: float,365 general_strength: float,366 motion_strength: float,367 dreamlay_strength: float,368 mself_strength: float,369 dramatic_strength: float,370 fluid_strength: float,371 liquid_strength: float,372 demopose_strength: float,373 progress=gr.Progress(track_tqdm=True),374):375 """376 CPU-only step:377 - checks cache378 - loads cached fused transformer state_dict, or379 - builds fused transformer on CPU and saves it380 The resulting state_dict is stored in memory and can be applied later.381 """382 global PENDING_LORA_KEY, PENDING_LORA_STATE, PENDING_LORA_STATUS383 384 ledger = pipeline.model_ledger385 key, _ = _make_lora_key(paste_strength,ruri_strength,transit_strength,pose_strength, general_strength, motion_strength, dreamlay_strength, mself_strength, dramatic_strength, fluid_strength, liquid_strength, demopose_strength)386 cache_path = LORA_CACHE_DIR / f"{key}.pt"387 388 progress(0.05, desc="Preparing LoRA state")389 if cache_path.exists():390 try:391 progress(0.20, desc="Loading cached fused state")392 state = torch.load(cache_path, map_location="cpu")393 PENDING_LORA_KEY = key394 PENDING_LORA_STATE = state395 PENDING_LORA_STATUS = f"Loaded cached LoRA state: {cache_path.name}"396 return PENDING_LORA_STATUS397 except Exception as e:398 print(f"[LoRA] Cache load failed: {type(e).__name__}: {e}")399 400 entries = [401 (paste_lora_path, round(float(paste_strength), 2)),402 (ruri_lora_path, round(float(ruri_strength), 2)),403 (transit_lora_path, round(float(transit_strength), 2)),404 (pose_lora_path, round(float(pose_strength), 2)),405 (general_lora_path, round(float(general_strength), 2)),406 (motion_lora_path, round(float(motion_strength), 2)),407 (dreamlay_lora_path, round(float(dreamlay_strength), 2)),408 (mself_lora_path, round(float(mself_strength), 2)),409 (dramatic_lora_path, round(float(dramatic_strength), 2)),410 (fluid_lora_path, round(float(fluid_strength), 2)),411 (liquid_lora_path, round(float(liquid_strength), 2)),412 (demopose_lora_path, round(float(demopose_strength), 2)),413 ]414 loras_for_builder = [415 LoraPathStrengthAndSDOps(path, strength, LTXV_LORA_COMFY_RENAMING_MAP)416 for path, strength in entries417 if path is not None and float(strength) != 0.0418 ]419 420 if not loras_for_builder:421 PENDING_LORA_KEY = None422 PENDING_LORA_STATE = None423 PENDING_LORA_STATUS = "No non-zero LoRA strengths selected; nothing to prepare."424 return PENDING_LORA_STATUS425 426 tmp_ledger = None427 new_transformer_cpu = None428 try:429 progress(0.35, desc="Building fused CPU transformer")430 tmp_ledger = pipeline.model_ledger.__class__(431 dtype=ledger.dtype,432 device=torch.device("cpu"),433 checkpoint_path=str(checkpoint_path),434 spatial_upsampler_path=str(spatial_upsampler_path),435 gemma_root_path=str(gemma_root),436 loras=tuple(loras_for_builder),437 quantization=getattr(ledger, "quantization", None),438 )439 new_transformer_cpu = tmp_ledger.transformer()440 441 progress(0.70, desc="Extracting fused state_dict")442 state = new_transformer_cpu.state_dict()443 torch.save(state, cache_path)444 445 PENDING_LORA_KEY = key446 PENDING_LORA_STATE = state447 PENDING_LORA_STATUS = f"Built and cached LoRA state: {cache_path.name}"448 return PENDING_LORA_STATUS449 450 except Exception as e:451 import traceback452 print(f"[LoRA] Prepare failed: {type(e).__name__}: {e}")453 print(traceback.format_exc())454 PENDING_LORA_KEY = None455 PENDING_LORA_STATE = None456 PENDING_LORA_STATUS = f"LoRA prepare failed: {type(e).__name__}: {e}"457 return PENDING_LORA_STATUS458 459 finally:460 try:461 del new_transformer_cpu462 except Exception:463 pass464 try:465 del tmp_ledger466 except Exception:467 pass468 gc.collect()469 470 471def apply_prepared_lora_state_to_pipeline():472 """473 Fast step: copy the already prepared CPU state into the live transformer.474 This is the only part that should remain near generation time.475 """476 global current_lora_key, PENDING_LORA_KEY, PENDING_LORA_STATE477 478 if PENDING_LORA_STATE is None or PENDING_LORA_KEY is None:479 print("[LoRA] No prepared LoRA state available; skipping.")480 return False481 482 if current_lora_key == PENDING_LORA_KEY:483 print("[LoRA] Prepared LoRA state already active; skipping.")484 return True485 486 existing_transformer = _transformer487 existing_params = {name: param for name, param in existing_transformer.named_parameters()}488 existing_buffers = {name: buf for name, buf in existing_transformer.named_buffers()}489 490 with torch.no_grad():491 for k, v in PENDING_LORA_STATE.items():492 if k in existing_params:493 existing_params[k].data.copy_(v.to(existing_params[k].device))494 elif k in existing_buffers:495 existing_buffers[k].data.copy_(v.to(existing_buffers[k].device))496 497 current_lora_key = PENDING_LORA_KEY498 print("[LoRA] Prepared LoRA state applied to the pipeline."+current_lora_key)499 return True500 501# ---- REPLACE PRELOAD BLOCK START ----502# Preload all models for ZeroGPU tensor packing.503print("Preloading all models (including Gemma and audio components)...")504ledger = pipeline.model_ledger505 506# Save the original factory methods so we can rebuild individual components later.507# These are bound callables on ledger that will call the builder when invoked.508_orig_transformer_factory = ledger.transformer509_orig_video_encoder_factory = ledger.video_encoder510_orig_video_decoder_factory = ledger.video_decoder511_orig_audio_encoder_factory = ledger.audio_encoder512_orig_audio_decoder_factory = ledger.audio_decoder513_orig_vocoder_factory = ledger.vocoder514_orig_spatial_upsampler_factory = ledger.spatial_upsampler515_orig_text_encoder_factory = ledger.text_encoder516_orig_gemma_embeddings_factory = ledger.gemma_embeddings_processor517 518# Call the original factories once to create the cached instances we will serve by default.519_transformer = _orig_transformer_factory()520_video_encoder = _orig_video_encoder_factory()521_video_decoder = _orig_video_decoder_factory()522_audio_encoder = _orig_audio_encoder_factory()523_audio_decoder = _orig_audio_decoder_factory()524_vocoder = _orig_vocoder_factory()525_spatial_upsampler = _orig_spatial_upsampler_factory()526_text_encoder = _orig_text_encoder_factory()527_embeddings_processor = _orig_gemma_embeddings_factory()528 529# Replace ledger methods with lightweight lambdas that return the cached instances.530# We keep the original factories above so we can call them later to rebuild components.531ledger.transformer = lambda: _transformer532ledger.video_encoder = lambda: _video_encoder533ledger.video_decoder = lambda: _video_decoder534ledger.audio_encoder = lambda: _audio_encoder535ledger.audio_decoder = lambda: _audio_decoder536ledger.vocoder = lambda: _vocoder537ledger.spatial_upsampler = lambda: _spatial_upsampler538ledger.text_encoder = lambda: _text_encoder539ledger.gemma_embeddings_processor = lambda: _embeddings_processor540 541print("All models preloaded (including Gemma text encoder and audio encoder)!")542# ---- REPLACE PRELOAD BLOCK END ----543 544print("=" * 80)545print("Pipeline ready!")546print("=" * 80)547 548 549def log_memory(tag: str):550 if torch.cuda.is_available():551 allocated = torch.cuda.memory_allocated() / 1024**3552 peak = torch.cuda.max_memory_allocated() / 1024**3553 free, total = torch.cuda.mem_get_info()554 print(f"[VRAM {tag}] allocated={allocated:.2f}GB peak={peak:.2f}GB free={free / 1024**3:.2f}GB total={total / 1024**3:.2f}GB")555 556 557def detect_aspect_ratio(image) -> str:558 if image is None:559 return "16:9"560 if hasattr(image, "size"):561 w, h = image.size562 elif hasattr(image, "shape"):563 h, w = image.shape[:2]564 else:565 return "16:9"566 ratio = w / h567 candidates = {"16:9": 16 / 9, "9:16": 9 / 16, "1:1": 1.0}568 return min(candidates, key=lambda k: abs(ratio - candidates[k]))569 570 571def on_image_upload(first_image, last_image, high_res):572 ref_image = first_image if first_image is not None else last_image573 aspect = detect_aspect_ratio(ref_image)574 tier = "high" if high_res else "low"575 w, h = RESOLUTIONS[tier][aspect]576 return gr.update(value=w), gr.update(value=h)577 578 579def on_highres_toggle(first_image, last_image, high_res):580 ref_image = first_image if first_image is not None else last_image581 aspect = detect_aspect_ratio(ref_image)582 tier = "high" if high_res else "low"583 w, h = RESOLUTIONS[tier][aspect]584 return gr.update(value=w), gr.update(value=h)585 586 587def get_gpu_duration(588 first_image,589 last_image,590 input_audio,591 prompt: str,592 duration: float,593 gpu_duration: float,594 enhance_prompt: bool = True,595 seed: int = 42,596 randomize_seed: bool = True,597 height: int = 1024,598 width: int = 1536,599 paste_strength: float = 0.0,600 ruri_strength: float = 0.0,601 transit_strength: float = 0.0,602 pose_strength: float = 0.0,603 general_strength: float = 0.0,604 motion_strength: float = 0.0,605 dreamlay_strength: float = 0.0,606 mself_strength: float = 0.0,607 dramatic_strength: float = 0.0,608 fluid_strength: float = 0.0,609 liquid_strength: float = 0.0,610 demopose_strength: float = 0.0,611 progress=None,612):613 return int(gpu_duration)614 615@spaces.GPU(duration=get_gpu_duration)616@torch.inference_mode()617def generate_video(618 first_image,619 last_image,620 input_audio,621 prompt: str,622 duration: float,623 gpu_duration: float,624 enhance_prompt: bool = True,625 seed: int = 42,626 randomize_seed: bool = True,627 height: int = 1024,628 width: int = 1536,629 paste_strength: float = 0.0,630 ruri_strength: float = 0.0,631 transit_strength: float = 0.0,632 pose_strength: float = 0.0,633 general_strength: float = 0.0,634 motion_strength: float = 0.0,635 dreamlay_strength: float = 0.0,636 mself_strength: float = 0.0,637 dramatic_strength: float = 0.0,638 fluid_strength: float = 0.0,639 liquid_strength: float = 0.0,640 demopose_strength: float = 0.0,641 progress=gr.Progress(track_tqdm=True),642):643 try:644 torch.cuda.reset_peak_memory_stats()645 log_memory("start")646 647 current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)648 649 frame_rate = DEFAULT_FRAME_RATE650 num_frames = int(duration * frame_rate) + 1651 num_frames = ((num_frames - 1 + 7) // 8) * 8 + 1652 653 print(f"Prompt:{prompt},Generating: {height}x{width}, {num_frames} frames ({duration}s), seed={current_seed},")654 655 images = []656 output_dir = Path("outputs")657 output_dir.mkdir(exist_ok=True)658 659 if first_image is not None:660 temp_first_path = output_dir / f"temp_first_{current_seed}.jpg"661 if hasattr(first_image, "save"):662 first_image.save(temp_first_path)663 else:664 temp_first_path = Path(first_image)665 images.append(ImageConditioningInput(path=str(temp_first_path), frame_idx=0, strength=1.0))666 667 if last_image is not None:668 temp_last_path = output_dir / f"temp_last_{current_seed}.jpg"669 if hasattr(last_image, "save"):670 last_image.save(temp_last_path)671 else:672 temp_last_path = Path(last_image)673 images.append(ImageConditioningInput(path=str(temp_last_path), frame_idx=num_frames - 1, strength=1.0))674 675 tiling_config = TilingConfig.default()676 video_chunks_number = get_video_chunks_number(num_frames, tiling_config)677 678 log_memory("before pipeline call")679 680 apply_prepared_lora_state_to_pipeline()681 682 video, audio = pipeline(683 prompt=prompt,684 seed=current_seed,685 height=int(height),686 width=int(width),687 num_frames=num_frames,688 frame_rate=frame_rate,689 images=images,690 audio_path=input_audio,691 tiling_config=tiling_config,692 enhance_prompt=enhance_prompt,693 )694 695 log_memory("after pipeline call")696 697 output_path = tempfile.mktemp(suffix=".mp4")698 encode_video(699 video=video,700 fps=frame_rate,701 audio=audio,702 output_path=output_path,703 video_chunks_number=video_chunks_number,704 )705 706 log_memory("after encode_video")707 return str(output_path), current_seed708 709 except Exception as e:710 import traceback711 log_memory("on error")712 print(f"Error: {str(e)}\n{traceback.format_exc()}")713 return None, current_seed714with gr.Blocks(title="LTX-2.3 Distilled") as demo:715 gr.Markdown("# LTX-2.3 F2LF with Fast Audio-Video Generation and Lora support")716 717 718 with gr.Row():719 with gr.Column():720 with gr.Row():721 first_image = gr.Image(label="First Frame (Optional)", type="pil")722 last_image = gr.Image(label="Last Frame (Optional)", type="pil")723 input_audio = gr.Audio(label="Audio Input (Optional)", type="filepath")724 prompt = gr.Textbox(725 label="Prompt",726 info="for best results - make it as elaborate as possible",727 value="Make this image come alive with cinematic motion, smooth animation",728 lines=3,729 placeholder="Describe the motion and animation you want...",730 )731 duration = gr.Slider(label="Duration (seconds)", minimum=1.0, maximum=30.0, value=10.0, step=0.1)732 733 734 generate_btn = gr.Button("Generate Video", variant="primary", size="lg")735 736 with gr.Accordion("Advanced Settings", open=True):737 seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, value=10, step=1)738 randomize_seed = gr.Checkbox(label="Randomize Seed", value=True)739 with gr.Row():740 width = gr.Number(label="Width", value=1536, precision=0)741 height = gr.Number(label="Height", value=1024, precision=0)742 with gr.Row():743 enhance_prompt = gr.Checkbox(label="Enhance Prompt", value=False)744 high_res = gr.Checkbox(label="High Resolution", value=True)745 with gr.Column():746 gr.Markdown("### LoRA adapter strengths (set to 0 to disable)")747 paste_strength = gr.Slider(label="Paste anything Lora strength", minimum=0.0, maximum=2.0, value=0.0, step=0.01)748 ruri_strength = gr.Slider(label="Ruri lady Lora strength", minimum=0.0, maximum=2.0, value=0.0, step=0.01)749 transit_strength = gr.Slider(label="Transition Lora strength", minimum=0.0, maximum=2.0, value=0.0, step=0.01)750 pose_strength = gr.Slider(label="Anthro Enhancer strength", minimum=0.0, maximum=2.0, value=0.0, step=0.01)751 general_strength = gr.Slider(label="Reasoning Enhancer strength", minimum=0.0, maximum=2.0, value=0.0, step=0.01)752 motion_strength = gr.Slider(label="Anthro Posing Helper strength", minimum=0.0, maximum=2.0, value=0.0, step=0.01)753 dreamlay_strength = gr.Slider(label="Dreamfly strength", minimum=0.0, maximum=2.0, value=0.0, step=0.01)754 mself_strength = gr.Slider(label="Mself strength", minimum=0.0, maximum=2.0, value=0.0, step=0.01)755 dramatic_strength = gr.Slider(label="Dramatic strength", minimum=0.0, maximum=2.0, value=0.0, step=0.01)756 fluid_strength = gr.Slider(label="Fluid Helper strength", minimum=0.0, maximum=2.0, value=0.0, step=0.01)757 liquid_strength = gr.Slider(label="Liquid Helper strength", minimum=0.0, maximum=2.0, value=0.0, step=0.01)758 demopose_strength = gr.Slider(label="Demopose Helper strength", minimum=0.0, maximum=2.0, value=0.0, step=0.01)759 760 prepare_lora_btn = gr.Button("Prepare / Load LoRA Cache", variant="secondary")761 lora_status = gr.Textbox(762 label="LoRA Cache Status",763 value="No LoRA state prepared yet.",764 interactive=False,765 )766 767 with gr.Column():768 output_video = gr.Video(label="Generated Video", autoplay=False)769 gpu_duration = gr.Slider(770 label="ZeroGPU duration (seconds)",771 minimum=40.0,772 maximum=240.0,773 value=85.0,774 step=1.0,775 )776 777 gr.Examples(778 examples=[779 [780 "asc.jpg",781 None,782 None,783 "A low-angle wide shot establishes a winding, wet asphalt road flanked by a dense, dark forest...",784 3.0,785 80.0,786 False,787 42,788 True,789 1024,790 1024,791 1.0,792 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,793 ],794 [None, "asc.jpg", "KABAPASTE The girl is squeezed from a tube like paste...", "paste lora"]795 ],796 inputs=[797 first_image, last_image, input_audio, prompt, duration, gpu_duration,798 enhance_prompt, seed, randomize_seed, height, width,799 paste_strength, pose_strength, general_strength, motion_strength,800 dreamlay_strength, mself_strength, dramatic_strength,801 fluid_strength, liquid_strength, demopose_strength,802 ],803 )804 805 # ✅ FIXED EXAMPLE OUTPUT SECTION (ONLY CHANGE)806 with gr.Column():807 gr.Markdown("## 🎬 Example Output")808 809 prompt_preview = gr.Textbox(810 label="Prompt",811 value="KABAPASTE The girl is squeezed from a tube like paste. The video is silent with the sound of rain and squeezing sounds. After the girl forms, her umbrella opens up.",812 interactive=False813 )814 815 video_preview = gr.Video(816 value="abc.mp4", 817 autoplay=True,818 interactive=False,819 width=150,820 height=150821 822 823 824 )825 826 first_image.change(827 fn=on_image_upload,828 inputs=[first_image, last_image, high_res],829 outputs=[width, height],830 )831 832 last_image.change(833 fn=on_image_upload,834 inputs=[first_image, last_image, high_res],835 outputs=[width, height],836 )837 838 high_res.change(839 fn=on_highres_toggle,840 inputs=[first_image, last_image, high_res],841 outputs=[width, height],842 )843 844 prepare_lora_btn.click(845 fn=prepare_lora_cache,846 inputs=[paste_strength, ruri_strength, transit_strength, pose_strength, general_strength, motion_strength, dreamlay_strength, mself_strength, dramatic_strength, fluid_strength, liquid_strength, demopose_strength],847 outputs=[lora_status],848 )849 850 generate_btn.click(851 fn=generate_video,852 inputs=[853 first_image, last_image, input_audio, prompt, duration, gpu_duration, enhance_prompt,854 seed, randomize_seed, height, width,855 paste_strength, ruri_strength, transit_strength, pose_strength, general_strength, motion_strength, dreamlay_strength, mself_strength, dramatic_strength, fluid_strength, liquid_strength, demopose_strength,856 ],857 outputs=[output_video, seed],858 )859 860 861css = """862.fillable{max-width: 1200px !important}863"""864 865if __name__ == "__main__":866 demo.launch(theme=gr.themes.Citrus(), css=css)867