GetSoloTech/solo-conversational-webgpu

1import { useEffect, useState, useRef } from "react";2import { Mic, PhoneOff, ChevronDown } from "lucide-react";3import { INPUT_SAMPLE_RATE } from "./constants";4 5import WORKLET from "./play-worklet.js";6 7export default function App() {8  const [callStartTime, setCallStartTime] = useState(null);9  const [callStarted, setCallStarted] = useState(false);10  const [playing, setPlaying] = useState(false);11 12  const [voice, setVoice] = useState("af_heart");13  const [voices, setVoices] = useState([]);14 15  const [isListening, setIsListening] = useState(false);16  const [isSpeaking, setIsSpeaking] = useState(false);17  const [listeningScale, setListeningScale] = useState(1);18  const [speakingScale, setSpeakingScale] = useState(1);19  const [ripples, setRipples] = useState([]);20 21  const [ready, setReady] = useState(false);22  const [error, setError] = useState(null);23  const [elapsedTime, setElapsedTime] = useState("00:00");24  const worker = useRef(null);25 26  const micStreamRef = useRef(null);27  const node = useRef(null);28 29  useEffect(() => {30    worker.current?.postMessage({31      type: "set_voice",32      voice,33    });34  }, [voice]);35 36  useEffect(() => {37    if (!callStarted) {38      // Reset worker state after call ends39      worker.current?.postMessage({40        type: "end_call",41      });42    }43  }, [callStarted]);44 45  useEffect(() => {46    if (callStarted && callStartTime) {47      const interval = setInterval(() => {48        const diff = Math.floor((Date.now() - callStartTime) / 1000);49        const minutes = String(Math.floor(diff / 60)).padStart(2, "0");50        const seconds = String(diff % 60).padStart(2, "0");51        setElapsedTime(`${minutes}:${seconds}`);52      }, 1000);53      return () => clearInterval(interval);54    } else {55      setElapsedTime("00:00");56    }57  }, [callStarted, callStartTime]);58 59  useEffect(() => {60    worker.current ??= new Worker(new URL("./worker.js", import.meta.url), {61      type: "module",62    });63 64    const onMessage = ({ data }) => {65      if (data.error) {66        return onError(data.error);67      }68 69      switch (data.type) {70        case "status":71          if (data.status === "recording_start") {72            setIsListening(true);73            setIsSpeaking(false);74          } else if (data.status === "recording_end") {75            setIsListening(false);76          } else if (data.status === "ready") {77            setVoices(data.voices);78            setReady(true);79          }80          break;81        case "output":82          if (!playing) {83            node.current?.port.postMessage(data.result.audio);84            setPlaying(true);85            setIsSpeaking(true);86            setIsListening(false);87          }88          break;89      }90    };91    const onError = (err) => setError(err.message);92 93    worker.current.addEventListener("message", onMessage);94    worker.current.addEventListener("error", onError);95 96    return () => {97      worker.current.removeEventListener("message", onMessage);98      worker.current.removeEventListener("error", onError);99    };100  }, []);101 102  useEffect(() => {103    if (!callStarted) return;104 105    let worklet;106    let inputAudioContext;107    let source;108    let ignore = false;109 110    let outputAudioContext;111    const audioStreamPromise = Promise.resolve(micStreamRef.current);112 113    audioStreamPromise114      .then(async (stream) => {115        if (ignore) return;116 117        inputAudioContext = new (window.AudioContext ||118          window.webkitAudioContext)({119          sampleRate: INPUT_SAMPLE_RATE,120        });121 122        const analyser = inputAudioContext.createAnalyser();123        analyser.fftSize = 256;124        source = inputAudioContext.createMediaStreamSource(stream);125        source.connect(analyser);126 127        const inputDataArray = new Uint8Array(analyser.frequencyBinCount);128 129        function calculateRMS(array) {130          let sum = 0;131          for (let i = 0; i < array.length; ++i) {132            const normalized = array[i] / 128 - 1;133            sum += normalized * normalized;134          }135          const rms = Math.sqrt(sum / array.length);136          return rms;137        }138 139        await inputAudioContext.audioWorklet.addModule(140          new URL("./vad-processor.js", import.meta.url),141        );142        worklet = new AudioWorkletNode(inputAudioContext, "vad-processor", {143          numberOfInputs: 1,144          numberOfOutputs: 0,145          channelCount: 1,146          channelCountMode: "explicit",147          channelInterpretation: "discrete",148        });149 150        source.connect(worklet);151        worklet.port.onmessage = (event) => {152          const { buffer } = event.data;153          worker.current?.postMessage({ type: "audio", buffer });154        };155 156        outputAudioContext = new AudioContext({157          sampleRate: 24000,158        });159        outputAudioContext.resume();160 161        const blob = new Blob([`(${WORKLET.toString()})()`], {162          type: "application/javascript",163        });164        const url = URL.createObjectURL(blob);165        await outputAudioContext.audioWorklet.addModule(url);166        URL.revokeObjectURL(url);167 168        node.current = new AudioWorkletNode(169          outputAudioContext,170          "buffered-audio-worklet-processor",171        );172 173        node.current.port.onmessage = (event) => {174          if (event.data.type === "playback_ended") {175            setPlaying(false);176            setIsSpeaking(false);177            worker.current?.postMessage({ type: "playback_ended" });178          }179        };180 181        const outputAnalyser = outputAudioContext.createAnalyser();182        outputAnalyser.fftSize = 256;183 184        node.current.connect(outputAnalyser);185        outputAnalyser.connect(outputAudioContext.destination);186 187        const outputDataArray = new Uint8Array(188          outputAnalyser.frequencyBinCount,189        );190 191        function updateVisualizers() {192          analyser.getByteTimeDomainData(inputDataArray);193          const rms = calculateRMS(inputDataArray);194          const targetScale = 1 + Math.min(1.25 * rms, 0.25);195          setListeningScale((prev) => prev + (targetScale - prev) * 0.25);196 197          outputAnalyser.getByteTimeDomainData(outputDataArray);198          const outputRMS = calculateRMS(outputDataArray);199          const targetOutputScale = 1 + Math.min(1.25 * outputRMS, 0.25);200          setSpeakingScale((prev) => prev + (targetOutputScale - prev) * 0.25);201 202          requestAnimationFrame(updateVisualizers);203        }204        updateVisualizers();205      })206      .catch((err) => {207        setError(err.message);208        console.error(err);209      });210 211    return () => {212      ignore = true;213      audioStreamPromise.then((s) => s.getTracks().forEach((t) => t.stop()));214      source?.disconnect();215      worklet?.disconnect();216      inputAudioContext?.close();217 218      outputAudioContext?.close();219    };220  }, [callStarted]);221 222  useEffect(() => {223    if (!callStarted) return;224    const interval = setInterval(() => {225      const id = Date.now();226      setRipples((prev) => [...prev, id]);227      setTimeout(() => {228        setRipples((prev) => prev.filter((r) => r !== id));229      }, 1500);230    }, 1000);231    return () => clearInterval(interval);232  }, [callStarted]);233 234  const handleStartCall = async () => {235    try {236      const stream = await navigator.mediaDevices.getUserMedia({237        audio: {238          channelCount: 1,239          echoCancellation: true,240          autoGainControl: true,241          noiseSuppression: true,242          sampleRate: INPUT_SAMPLE_RATE,243        },244      });245      micStreamRef.current = stream;246 247      setCallStartTime(Date.now());248      setCallStarted(true);249      worker.current?.postMessage({ type: "start_call" });250    } catch (err) {251      setError(err.message);252      console.error(err);253    }254  };255 256  return (257    <div className="h-screen min-h-[240px] flex items-center justify-center bg-gray-50 p-4 relative">258      <div className="h-full max-h-[320px] w-[640px] bg-white rounded-xl shadow-lg p-8 flex items-center justify-between space-x-16">259        <div className="text-green-700 w-[140px]">260          <div className="text-xl font-bold flex justify-between">261            {voices?.[voice]?.name}262            <span className="font-normal text-gray-500">{elapsedTime}</span>263          </div>264          <div className="text-base relative">265            <button266              type="button"267              disabled={!ready}268              className={`w-full flex items-center justify-between border border-gray-300 rounded-md transition-colors ${269                ready270                  ? "bg-transparent hover:border-gray-400"271                  : "bg-gray-100 opacity-50 cursor-not-allowed"272              }`}273            >274              <span className="px-2 py-1">Select voice</span>275              <ChevronDown className="absolute right-2" />276            </button>277            <select278              value={voice}279              onChange={(e) => setVoice(e.target.value)}280              className="absolute inset-0 opacity-0 cursor-pointer"281              disabled={!ready}282            >283              {Object.entries(voices).map(([key, v]) => (284                <option key={key} value={key}>285                  {`${v.name} (${286                    v.language === "en-us" ? "American" : v.language287                  } ${v.gender})`}288                </option>289              ))}290            </select>291          </div>292        </div>293 294        <div className="relative flex items-center justify-center w-32 h-32 flex-shrink-0 aspect-square">295          {callStarted &&296            ripples.map((id) => (297              <div298                key={id}299                className="absolute inset-0 rounded-full border-2 border-green-200 pointer-events-none"300                style={{ animation: "ripple 1.5s ease-out forwards" }}301              />302            ))}303          {/* Pulsing loader while initializing */}304          <div305            className={`absolute w-32 h-32 rounded-full ${306              error ? "bg-red-200" : "bg-green-200"307            } ${!ready ? "animate-ping opacity-75" : ""}`}308            style={{ animationDuration: "1.5s" }}309          />310          {/* Main rings */}311          <div312            className={`absolute w-32 h-32 rounded-full shadow-inner transition-transform duration-300 ease-out ${313              error ? "bg-red-300" : "bg-green-300"314            } ${!ready ? "opacity-0" : ""}`}315            style={{ transform: `scale(${speakingScale})` }}316          />317          <div318            className={`absolute w-32 h-32 rounded-full shadow-inner transition-transform duration-300 ease-out ${319              error ? "bg-red-200" : "bg-green-200"320            } ${!ready ? "opacity-0" : ""}`}321            style={{ transform: `scale(${listeningScale})` }}322          />323          {/* Center text: show error if present, else existing statuses */}324          <div325            className={`absolute z-10 text-lg text-center ${326              error ? "text-red-700" : "text-gray-700"327            }`}328          >329            {error ? (330              error331            ) : (332              <>333                {!ready && "Loading..."}334                {isListening && "Listening..."}335                {isSpeaking && "Speaking..."}336              </>337            )}338          </div>339        </div>340 341        <div className="space-y-4 w-[140px]">342          {callStarted ? (343            <button344              className="flex items-center space-x-2 px-4 py-2 bg-red-100 text-red-700 rounded-md hover:bg-red-200"345              onClick={() => {346                setCallStarted(false);347                setCallStartTime(null);348                setPlaying(false);349                setIsListening(false);350                setIsSpeaking(false);351              }}352            >353              <PhoneOff className="w-5 h-5" />354              <span>End call</span>355            </button>356          ) : (357            <button358              className={`flex items-center space-x-2 px-4 py-2 rounded-md ${359                ready360                  ? "bg-blue-100 text-blue-700 hover:bg-blue-200"361                  : "bg-blue-100 text-blue-700 opacity-50 cursor-not-allowed"362              }`}363              onClick={handleStartCall}364              disabled={!ready}365            >366              <span>Start call</span>367            </button>368          )}369        </div>370      </div>371 372      <div className="absolute bottom-4 text-sm">373        Built with{" "}374        <a375          href="https://github.com/huggingface/transformers.js"376          rel="noopener noreferrer"377          target="_blank"378          className="text-blue-600 hover:underline"379        >380          🤗 Transformers.js381        </a>382      </div>383    </div>384  );385}386