Kannguyen/avatar-realtime-voice-chat-research

eyeBlinkLeft, eyeBlinkRight, eyeLookDownLeft, eyeLookDownRight,
eyeLookInLeft, eyeLookInRight, eyeLookOutLeft, eyeLookOutRight,
eyeLookUpLeft, eyeLookUpRight, eyeSquintLeft, eyeSquintRight,
eyeWideLeft, eyeWideRight, jawForward, jawLeft, jawRight, jawOpen,
mouthClose, mouthFunnel, mouthPucker, mouthLeft, mouthRight,
mouthSmileLeft, mouthSmileRight, mouthFrownLeft, mouthFrownRight,
mouthDimpleLeft, mouthDimpleRight, mouthStretchLeft, mouthStretchRight,
mouthRollLower, mouthRollUpper, mouthShrugLower, mouthShrugUpper,
mouthPressLeft, mouthPressRight, mouthLowerDownLeft, mouthLowerDownRight,
mouthUpperUpLeft, mouthUpperUpRight, browDownLeft, browDownRight,
browInnerUp, browOuterUpLeft, browOuterUpRight, cheekPuff,
cheekSquintLeft, cheekSquintRight, noseSneerLeft, noseSneerRight,
tongueOut

viseme_sil    - Im lặng
viseme_PP     - P, B, M
viseme_FF     - F, V
viseme_TH     - Th
viseme_DD     - D, T, N
viseme_kk     - K, G
viseme_CH     - Ch, J, Sh
viseme_SS     - S, Z
viseme_nn     - N, L
viseme_RR     - R
viseme_aa     - A
viseme_E      - E
viseme_I      - I
viseme_O      - O
viseme_U      - U

┌──────────────┐     WebRTC      ┌──────────────┐
│  Voice Chat  │ ──────────────► │ Remote Audio  │
│    Client    │                 │   Stream      │
└──────────────┘                 └──────┬───────┘
                                        │
                                        ▼
                                ┌───────────────┐
                                │  HeadAudio     │
                                │  (AudioWorklet)│
                                │  MFCC → Viseme │
                                └──────┬────────┘
                                       │
                                       ▼
                                ┌───────────────┐
                                │  TalkingHead   │
                                │  (Three.js)    │
                                │  3D Avatar     │
                                └───────────────┘

import { TalkingHead } from "@met4citizen/talkinghead";
import { HeadAudioNode } from "./modules/headaudio.mjs";

// 1. Khởi tạo avatar
const container = document.getElementById("avatar-container");
const head = new TalkingHead(container, {
  ttsEndpoint: "/api/tts",  // hoặc dùng ElevenLabs, Azure, etc.
  modelFPS: 30,
});

// Load avatar GLB
await head.showAvatar({
  url: "./models/avatar.glb",
  body: "F",            // "F" = female, "M" = male
  avatarMood: "neutral",
  lipsyncLang: "en",    // ngôn ngữ lip-sync
});

// 2. Setup audio worklet cho lip-sync từ audio stream
await head.audioCtx.audioWorklet.addModule("./modules/headworklet.mjs");
const headAudio = new HeadAudioNode(head.audioCtx);
await headAudio.loadModel("./dist/model-en-mixed.bin");

// 3. Kết nối WebRTC remote audio stream
const remoteStream = peerConnection.getRemoteStreams()[0]; // WebRTC stream
const source = head.audioCtx.createMediaStreamSource(remoteStream);
source.connect(headAudio);

// 4. Map viseme values → avatar blendshapes
headAudio.onvalue = (key, value) => {
  head.setMorphTarget(key, value);
  // key = "viseme_aa", value = 0.0 ~ 1.0
};

// 5. Animation loop
function animate(dt) {
  headAudio.update(dt);
  requestAnimationFrame(animate);
}
requestAnimationFrame(animate);

from datasets import load_dataset

dataset = load_dataset("Kannguyen/avatar-realtime-voice-chat-research")