webml-community/YOLO26-WebGPU

1import { AutoModel, AutoProcessor, RawImage } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.8.1';2 3// DOM Elements4const video = document.getElementById('video');5const canvas = document.getElementById('canvas');6const ctx = canvas.getContext('2d');7const startBtn = document.getElementById('start-btn');8const btnIcon = document.getElementById('btn-icon');9const btnText = document.getElementById('btn-text');10const modelSelect = document.getElementById('model-select');11const toggleDetect = document.getElementById('toggle-detect');12const togglePose = document.getElementById('toggle-pose');13const thresholdInput = document.getElementById('threshold');14const thresholdValueEl = document.getElementById('threshold-value');15const fpsEl = document.getElementById('fps');16const loader = document.getElementById('loader');17const loaderText = document.getElementById('loader-text');18const statusDot = document.getElementById('status-dot');19const statusText = document.getElementById('status-text');20 21// State22let detectModel = null;23let poseModel = null;24let processor = null;25let isRunning = false;26let isProcessing = false;27let threshold = 0.5;28let enableDetect = true;29let enablePose = true;30let animationId = null;31 32// Offscreen canvas for frame capture33const offscreen = document.createElement('canvas');34const offscreenCtx = offscreen.getContext('2d');35 36// Constants37const COLORS = ['#6366f1', '#ec4899', '#14b8a6', '#f59e0b', '#8b5cf6', '#ef4444', '#10b981', '#3b82f6'];38const SKELETON = [39  [0, 1], [0, 2], [1, 3], [2, 4],40  [5, 6], [5, 7], [7, 9], [6, 8], [8, 10],41  [5, 11], [6, 12], [11, 12],42  [11, 13], [13, 15], [12, 14], [14, 16]43];44const POSE_THRESHOLD = 0.0001;45 46// UI Helpers47const setStatus = (text, type = 'default') => {48  statusText.textContent = text;49  statusDot.className = 'status-dot ' + type;50};51 52const showLoader = (text) => {53  loaderText.textContent = text;54  loader.classList.add('visible');55};56 57const hideLoader = () => loader.classList.remove('visible');58 59// Model Loading60async function loadModels(modelId) {61  try {62    if (isRunning) stopCamera(true);63    while (isProcessing) await new Promise(r => setTimeout(r, 50));64 65    if (detectModel) await detectModel.dispose();66    if (poseModel) await poseModel.dispose();67    detectModel = poseModel = processor = null;68    startBtn.disabled = true;69 70    const poseModelId = modelId.replace('-ONNX', '-pose-ONNX');71    const progressCallback = (label) => (info) => {72      if (info.status === 'progress' && info.file.endsWith('.onnx')) {73        showLoader(`${label} (${Math.round((info.loaded / info.total) * 100)}%)`);74      }75    };76 77    setStatus('Loading...', 'loading');78    showLoader('Loading detection model...');79    detectModel = await AutoModel.from_pretrained(modelId, {80      device: 'webgpu',81      dtype: 'fp16',82      progress_callback: progressCallback('Loading detection model')83    });84 85    showLoader('Loading pose model...');86    poseModel = await AutoModel.from_pretrained(poseModelId, {87      device: 'webgpu',88      dtype: 'fp32',89      progress_callback: progressCallback('Loading pose model')90    });91 92    showLoader('Loading processor...');93    processor = await AutoProcessor.from_pretrained(modelId);94 95    setStatus('Ready', 'ready');96    hideLoader();97    startBtn.disabled = false;98    startCamera();99  } catch (error) {100    console.error('Model loading failed:', error);101    setStatus('Error', 'error');102    showLoader('Failed: ' + error.message);103  }104}105 106// Camera Control107async function startCamera() {108  try {109    showLoader('Accessing camera...');110    const stream = await navigator.mediaDevices.getUserMedia({111      video: { facingMode: 'environment', width: { ideal: 640 }, height: { ideal: 640 } },112      audio: false113    });114 115    video.srcObject = stream;116    video.onloadedmetadata = () => {117      canvas.width = offscreen.width = video.videoWidth;118      canvas.height = offscreen.height = video.videoHeight;119 120      isRunning = true;121      btnIcon.innerHTML = '<svg width="16" height="16" viewBox="0 0 24 24" fill="currentColor"><rect x="4" y="4" width="16" height="16" rx="2"/></svg>';122      btnText.textContent = 'Stop Camera';123      startBtn.classList.add('running');124 125      hideLoader();126      setStatus('Running', 'running');127      loop();128    };129  } catch (error) {130    console.error('Camera error:', error);131    setStatus('Camera Error', 'error');132    showLoader('Camera access denied');133  }134}135 136function stopCamera(keepProcessingFlag = false) {137  if (animationId) cancelAnimationFrame(animationId);138  animationId = null;139 140  if (video.srcObject) {141    video.srcObject.getTracks().forEach(t => t.stop());142    video.srcObject = null;143  }144 145  isRunning = false;146  if (!keepProcessingFlag) isProcessing = false;147 148  btnIcon.innerHTML = '<svg width="16" height="16" viewBox="0 0 24 24" fill="currentColor"><path d="M6 4l15 8-15 8V4z"/></svg>';149  btnText.textContent = 'Start Camera';150  startBtn.classList.remove('running');151  canvas.width = canvas.width; // Clear canvas152  setStatus('Ready', 'ready');153  fpsEl.textContent = '0';154}155 156// Detection Loop157function loop() {158  if (!isRunning) return;159 160  if (detectModel && poseModel && processor && !isProcessing) {161    isProcessing = true;162    const startTime = performance.now();163    detect()164      .then(() => fpsEl.textContent = Math.round(1000 / (performance.now() - startTime)))165      .finally(() => isProcessing = false);166  }167 168  if (isRunning) animationId = requestAnimationFrame(loop);169}170 171async function detect() {172  offscreenCtx.drawImage(video, 0, 0);173  const image = RawImage.fromCanvas(offscreen);174  const inputs = await processor(image);175 176  const promises = [];177  if (enableDetect) promises.push(detectModel(inputs));178  if (enablePose) promises.push(poseModel(inputs));179 180  const results = await Promise.all(promises);181  let idx = 0;182  const detectOutput = enableDetect ? results[idx++] : null;183  const poseOutput = enablePose ? results[idx++] : null;184 185  const detections = [];186 187  if (detectOutput) {188    const scores = detectOutput.logits.sigmoid().data;189    const boxes = detectOutput.pred_boxes.data;190    const id2label = detectModel.config.id2label;191 192    for (let i = 0; i < 300; i++) {193      let maxScore = 0, maxClass = 0;194      for (let j = 0; j < 80; j++) {195        const score = scores[i * 80 + j];196        if (score > maxScore) { maxScore = score; maxClass = j; }197      }198      if (maxScore >= threshold) {199        const [cx, cy, w, h] = [boxes[i * 4], boxes[i * 4 + 1], boxes[i * 4 + 2], boxes[i * 4 + 3]];200        detections.push({201          type: 'object',202          box: [(cx - w / 2) * canvas.width, (cy - h / 2) * canvas.height, w * canvas.width, h * canvas.height],203          score: maxScore,204          classId: maxClass,205          label: id2label[maxClass] || `Class ${maxClass}`206        });207      }208    }209  }210 211  if (poseOutput) {212    const data = Object.values(poseOutput)[0].data;213    for (let i = 0; i < 300; i++) {214      const offset = i * 57;215      const score = data[offset + 4];216      if (score >= threshold) {217        const keypoints = [];218        for (let k = 0; k < 17; k++) {219          const kIdx = offset + 6 + k * 3;220          keypoints.push({ x: data[kIdx] * canvas.width, y: data[kIdx + 1] * canvas.height, c: data[kIdx + 2] });221        }222        detections.push({223          type: 'pose',224          box: [data[offset] * canvas.width, data[offset + 1] * canvas.height,225                (data[offset + 2] - data[offset]) * canvas.width, (data[offset + 3] - data[offset + 1]) * canvas.height],226          score,227          keypoints228        });229      }230    }231  }232 233  if (isRunning) draw(detections);234}235 236// Drawing237function draw(detections) {238  ctx.clearRect(0, 0, canvas.width, canvas.height);239 240  for (const det of detections.filter(d => d.type === 'object')) {241    const [x, y, w, h] = det.box;242    const color = COLORS[det.classId % COLORS.length];243    const label = `${det.label} ${Math.round(det.score * 100)}%`;244 245    ctx.strokeStyle = color;246    ctx.lineWidth = 2;247    ctx.strokeRect(x, y, w, h);248 249    ctx.font = 'bold 12px system-ui';250    const tw = ctx.measureText(label).width;251    ctx.fillStyle = color;252    ctx.fillRect(x, y > 18 ? y - 18 : y, tw + 8, 18);253    ctx.fillStyle = '#fff';254    ctx.fillText(label, x + 4, y > 18 ? y - 5 : y + 13);255  }256 257  for (const det of detections.filter(d => d.type === 'pose')) {258    ctx.lineWidth = 3;259    ctx.strokeStyle = '#22d3ee';260    for (const [i, j] of SKELETON) {261      const a = det.keypoints[i], b = det.keypoints[j];262      if (a?.c >= POSE_THRESHOLD && b?.c >= POSE_THRESHOLD) {263        ctx.beginPath();264        ctx.moveTo(a.x, a.y);265        ctx.lineTo(b.x, b.y);266        ctx.stroke();267      }268    }269 270    for (const kp of det.keypoints) {271      if (kp.c < POSE_THRESHOLD) continue;272      ctx.fillStyle = '#6366f1';273      ctx.beginPath();274      ctx.arc(kp.x, kp.y, 5, 0, Math.PI * 2);275      ctx.fill();276      ctx.strokeStyle = '#fff';277      ctx.lineWidth = 2;278      ctx.stroke();279    }280  }281}282 283// Event Listeners284startBtn.addEventListener('click', () => isRunning ? stopCamera() : startCamera());285thresholdInput.addEventListener('input', (e) => {286  threshold = e.target.value / 100;287  thresholdValueEl.textContent = `${e.target.value}%`;288});289toggleDetect.addEventListener('change', (e) => enableDetect = e.target.checked);290togglePose.addEventListener('change', (e) => enablePose = e.target.checked);291modelSelect.addEventListener('change', (e) => loadModels(e.target.value));292 293// Initialize294loadModels(modelSelect.value);295