// Hook that runs SpeechRecognition on the local mic when live-captions are // enabled and a Room is connected. Each interim/final result is broadcast as // a `caption`-typed message via the LiveKit DataChannel so peers can render // it. Recognition stops cleanly when the call ends or the toggle flips off. import type { Room } from 'livekit-client'; import { useEffect, useRef } from 'react'; import { type LiveCaptionsSettings, getLiveCaptionsSettings, getSpeechRecognitionCtor, type SpeechRecognitionEventLike, type SpeechRecognitionLike, subscribeLiveCaptionsSettings, } from './liveCaptions'; interface Args { room: Room | null; /** True while we're connected and want captions to flow. */ active: boolean; /** Callback fired locally for our own captions so the overlay can show * them without going through the SFU round-trip. */ onLocalCaption: (text: string, final: boolean) => void; } export function useLiveCaptions({ room, active, onLocalCaption }: Args): void { const recognitionRef = useRef(null); const settingsRef = useRef(getLiveCaptionsSettings()); useEffect(() => { return subscribeLiveCaptionsSettings((s) => { settingsRef.current = s; }); }, []); useEffect(() => { const Ctor = getSpeechRecognitionCtor(); if (!Ctor) return; // unsupported runtime if (!active || !room) return; if (!getLiveCaptionsSettings().enabled) return; const send = (text: string, final: boolean) => { onLocalCaption(text, final); try { const payload = new TextEncoder().encode( JSON.stringify({ type: 'caption', captionText: text, captionFinal: final }), ); // Reliable channel — captions are infrequent enough to afford it, // and dropping interims looks worse than slight lag. void room.localParticipant.publishData(payload, { reliable: true }); } catch { /* ignore — best-effort */ } }; const start = () => { const r = new Ctor(); r.continuous = true; r.interimResults = true; const lang = settingsRef.current.lang ?? navigator.language ?? 'de-DE'; r.lang = lang; r.onresult = (e: SpeechRecognitionEventLike) => { // Pull whichever results arrived since last fire. Interim fires // many times per second; the final one is sticky and persists. for (let i = e.resultIndex; i < e.results.length; i++) { const result = e.results[i]; if (!result || result.length === 0) continue; const alt = result[0]; if (!alt) continue; const transcript = alt.transcript.trim(); if (!transcript) continue; send(transcript, result.isFinal); } }; r.onerror = () => { // Recoverable: stop + retry on next effect cycle. `not-allowed` and // `service-not-allowed` are permission-permanent — bail. try { r.stop(); } catch { /* ignore */ } }; r.onend = () => { // SpeechRecognition tends to auto-stop after silence — if we still // want captions, restart it. Guard against tear-down race. if (recognitionRef.current === r && getLiveCaptionsSettings().enabled) { try { r.start(); } catch { /* already running or browser refused */ } } }; try { r.start(); recognitionRef.current = r; } catch { // Some browsers throw when start() is called too soon after a // previous abort — wait a tick and retry. window.setTimeout(() => { try { r.start(); recognitionRef.current = r; } catch { /* give up */ } }, 250); } }; start(); const unsub = subscribeLiveCaptionsSettings((s) => { const cur = recognitionRef.current; if (!s.enabled && cur) { recognitionRef.current = null; try { cur.abort(); } catch { /* ignore */ } } else if (s.enabled && !cur) { start(); } }); return () => { unsub(); const cur = recognitionRef.current; recognitionRef.current = null; if (cur) { try { cur.abort(); } catch { /* ignore */ } } }; }, [active, room, onLocalCaption]); }