multimodalart/StreamDiffusionV2-Realtime
0
1<!DOCTYPE html>2<html lang="en">3<head>4<meta charset="utf-8" />5<meta name="viewport" content="width=device-width, initial-scale=1" />6<title>LiveEdit · Realtime</title>7<style>8 :root { --bg:#0e0f13; --panel:#171922; --line:#2a2e3a; --fg:#e8e8ee; --accent:#c084fc; --good:#86efac; }9 * { box-sizing: border-box; }10 body { margin:0; background:var(--bg); color:var(--fg); font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif; }11 .wrap { max-width:1100px; margin:0 auto; padding:24px 18px 60px; }12 h1 { font-size:1.5rem; margin:0 0 4px; }13 .sub { color:#9aa0ad; font-size:.95rem; margin:0 0 18px; line-height:1.5; }14 .sub a { color:var(--accent); }15 .controls { display:flex; gap:10px; flex-wrap:wrap; align-items:center; margin-bottom:16px; }16 input[type=text] { flex:1; min-width:260px; background:var(--panel); border:1px solid var(--line); color:var(--fg); padding:12px 14px; border-radius:10px; font-size:1rem; }17 button { background:var(--accent); color:#1a1024; border:0; padding:12px 18px; border-radius:10px; font-size:1rem; font-weight:600; cursor:pointer; }18 button:disabled { opacity:.5; cursor:not-allowed; }19 .timer { font-family:ui-monospace,Menlo,monospace; color:var(--good); background:#11210f; border:1px solid #234; padding:8px 12px; border-radius:8px; display:none; }20 .grid { display:grid; grid-template-columns:1fr 1fr; gap:14px; }21 .card { background:var(--panel); border:1px solid var(--line); border-radius:14px; overflow:hidden; }22 .card h2 { font-size:.85rem; text-transform:uppercase; letter-spacing:.05em; color:#9aa0ad; margin:0; padding:10px 14px; border-bottom:1px solid var(--line); }23 .media { aspect-ratio:832/480; background:#000; display:flex; align-items:center; justify-content:center; }24 .media video, .media img { width:100%; height:100%; object-fit:cover; display:block; }25 .placeholder { color:#5a6070; font-size:.9rem; }26 .status { margin-top:14px; color:#9aa0ad; font-size:.9rem; min-height:1.2em; }27 @media (max-width:780px){ .grid{ grid-template-columns:1fr; } }28</style>29</head>30<body>31<div class="wrap">32 <h1>🌀 StreamDiffusionV2 · Realtime Webcam Diffusion</h1>33 <p class="sub">34 Live demo of <a href="https://streamdiffusionv2.github.io/" target="_blank">StreamDiffusionV2</a>35 (MLSys 2026 Best Paper) on Wan2.1-T2V-1.3B. It streams your webcam through a causal video-diffusion36 model with a <b>sink-token rolling KV cache</b> — built for <i>continuous</i> streaming, so it37 keeps flowing without the window-shift burst. Type a style prompt, click <b>Start</b> to grab ZeroGPU38 for ~60s. <b>Change the prompt anytime</b> — it updates the live stream.39 </p>40 41 <div class="controls">42 <input id="instruction" type="text" placeholder="style prompt · e.g. psychedelic neon dream · van gogh · cyberpunk city" />43 <button id="startBtn">▶ Start session</button>44 <span id="timer" class="timer">⏱ <span id="count">58</span>s</span>45 </div>46 47 <div class="grid">48 <div class="card">49 <h2>Your webcam</h2>50 <div class="media"><video id="cam" autoplay muted playsinline></video></div>51 </div>52 <div class="card">53 <h2>Edited (live)</h2>54 <div class="media"><img id="out" alt="" /><span id="outPh" class="placeholder">edited stream appears here</span></div>55 </div>56 </div>57 58 <div id="status" class="status"></div>59</div>60 61<canvas id="grab" width="640" height="360" style="display:none"></canvas>62 63<script type="module">64import { Client } from "https://cdn.jsdelivr.net/npm/@gradio/client/dist/index.min.js";65 66const camEl = document.getElementById("cam");67const outEl = document.getElementById("out");68const outPh = document.getElementById("outPh");69const startBtn = document.getElementById("startBtn");70const instr = document.getElementById("instruction");71const timerEl = document.getElementById("timer");72const countEl = document.getElementById("count");73const statusEl = document.getElementById("status");74const grab = document.getElementById("grab");75const gctx = grab.getContext("2d");76 77let client = null;78let stream = null;79let captureTimer = null;80let countdownTimer = null;81let running = false;82 83const FPS = 30; // webcam frames sent per second (backend drops backlog)84const SESSION_SECONDS = 58;85 86// --- low-latency player: StreamDiffusionV2 streams steadily (~14fps), so we do87// NOT jitter-buffer. Keep a tiny queue and aggressively drop the backlog so the88// preview always shows the most recent edit (minimal action->reaction delay).89let playQueue = [];90const MAX_QUEUE = 3; // ~0.2s worth; drop older frames past this91let lastShown = 0;92function playLoop(ts){93 if (playQueue.length > MAX_QUEUE) playQueue = playQueue.slice(-MAX_QUEUE);94 if (playQueue.length && ts - lastShown >= 1000 / 30){95 outEl.src = playQueue.shift();96 outPh.style.display = "none";97 lastShown = ts;98 }99 requestAnimationFrame(playLoop);100}101requestAnimationFrame(playLoop);102 103function setStatus(t){ statusEl.textContent = t; }104 105async function ensureClient(){106 if (!client) client = await Client.connect(window.location.origin);107 return client;108}109 110async function ensureCam(){111 if (stream) return;112 stream = await navigator.mediaDevices.getUserMedia({ video: { width: 832, height: 480 }, audio: false });113 camEl.srcObject = stream;114 await camEl.play().catch(()=>{});115}116 117async function sendInstruction(){118 try {119 await fetch("/instruction", {120 method:"POST", headers:{ "Content-Type":"application/json" },121 body: JSON.stringify({ instruction: instr.value || "" })122 });123 } catch(e){}124}125 126function startCapture(){127 captureTimer = setInterval(() => {128 if (!camEl.videoWidth) return;129 gctx.drawImage(camEl, 0, 0, grab.width, grab.height);130 grab.toBlob(async (blob) => {131 if (!blob) return;132 try { await fetch("/frame", { method:"POST", body: blob }); } catch(e){}133 }, "image/jpeg", 0.6);134 }, 1000 / FPS);135}136 137function stopAll(){138 running = false;139 if (captureTimer) { clearInterval(captureTimer); captureTimer = null; }140 if (countdownTimer) { clearInterval(countdownTimer); countdownTimer = null; }141 timerEl.style.display = "none";142 startBtn.disabled = false;143 startBtn.textContent = "▶ Start session";144}145 146function startCountdown(){147 let r = SESSION_SECONDS;148 countEl.textContent = r;149 timerEl.style.display = "inline-block";150 countdownTimer = setInterval(() => {151 r -= 1; countEl.textContent = Math.max(0, r);152 if (r <= 0) clearInterval(countdownTimer);153 }, 1000);154}155 156instr.addEventListener("change", () => { if (running) sendInstruction(); });157instr.addEventListener("input", () => { if (running) sendInstruction(); });158 159startBtn.addEventListener("click", async () => {160 if (running) return;161 startBtn.disabled = true;162 try {163 setStatus("Requesting webcam…");164 await ensureCam();165 setStatus("Connecting…");166 await ensureClient();167 running = true;168 playQueue = [];169 startBtn.textContent = "◌ Acquiring ZeroGPU…";170 await sendInstruction();171 setStatus("Queued for ZeroGPU — webcam streaming starts once the GPU is acquired…");172 173 const job = client.submit("/run_session", {});174 let frames = 0;175 for await (const msg of job) {176 if (msg.type !== "data" || !msg.data || msg.data[0] == null) continue;177 const payload = msg.data[0];178 if (payload === "__READY__") {179 // GPU is now allocated — only now start capturing & sending frames.180 startBtn.textContent = "● Live";181 await sendInstruction();182 startCapture();183 startCountdown();184 setStatus("ZeroGPU acquired — streaming your webcam through LiveEdit…");185 continue;186 }187 playQueue.push(payload); // jitter buffer paces actual display188 frames += 1;189 if (frames % 12 === 0) setStatus(`Streaming… ${frames} edited frames`);190 }191 setStatus(frames ? "Session ended. Click Start to run another ~60s session."192 : "Session ended before any frames were produced — try again.");193 } catch (e) {194 setStatus("Error: " + (e && e.message ? e.message : e));195 } finally {196 stopAll();197 }198});199</script>200</body>201</html>202