diff --git a/apps/desktop/src/context/CallContext.tsx b/apps/desktop/src/context/CallContext.tsx index 6b19a8c..ff853f3 100644 --- a/apps/desktop/src/context/CallContext.tsx +++ b/apps/desktop/src/context/CallContext.tsx @@ -1203,26 +1203,145 @@ export function CallProvider({ children }: { children: ReactNode }) { const fps = framerateOverride ?? ssParams.framerate; const sourceId = overrides?.sourceId ?? null; - // Native capture path — tried first when the user came in via our - // custom picker. xcap on the Rust side grabs video frames and, on - // Windows with "Mit System-Sound" on, the WASAPI loopback module - // grabs the render endpoint. Both stream over Tauri channels into - // tracks we publish directly to LiveKit — the OS picker never - // appears. If native audio fails on a platform that can't supply - // it (non-Windows v1), we continue with video-only and log; the - // user still gets their direct-video share. + // Ordered capture paths when our custom picker supplied a sourceId: + // 1. chromeMediaSource video — hardware-accelerated path that + // hands the frames directly to WebRTC, same mechanism the OS + // picker uses internally. Fast, smooth, what users expect. + // 2. xcap native (JPEG → canvas → captureStream) — fallback for + // WebView2 versions that reject the legacy chromeMediaSource + // constraint. Functional but CPU-heavy; the per-frame JPEG + // encode + base64 + createImageBitmap chain shows up as + // visible stutter on 1080p30. + // 3. OS picker (setScreenShareEnabled) — last resort when both + // native paths throw, and the only path when no sourceId was + // supplied. + // + // Audio (Windows only) always goes through the WASAPI loopback + // module — getUserMedia's chromeMediaSource audio constraint + // throws AbortError on Window captures, and pairing both into one + // call ended up blocking the video path entirely. Splitting them + // means the video path stays fast and the audio path stays + // reliable for any source type. if (!sourceId) { console.info( 'screen-share: no sourceId supplied by picker, OS picker will open', ); } + + // Publish system audio via WASAPI. Resolves to the audio publication + // + an ended-cleanup registration so whichever video path succeeds + // can wire the audio teardown onto its own video-ended handler. + const startWasapiAudio = async ( + LkTrack: typeof import('livekit-client').Track, + ): Promise<{ + stopAudio: () => Promise; + } | null> => { + if (!settings.includeSystemAudio) return null; + try { + const audioHandle = await startSystemAudioCapture(); + nativeAudioCaptureRef.current = audioHandle; + const audioMst = audioHandle.stream.getAudioTracks()[0]; + if (!audioMst) { + await audioHandle.stop(); + nativeAudioCaptureRef.current = null; + return null; + } + const audioPub = await lp.publishTrack(audioMst, { + source: LkTrack.Source.ScreenShareAudio, + }); + audioMst.addEventListener('ended', () => { + void (async () => { + try { + if (audioPub.track) await lp.unpublishTrack(audioPub.track); + } catch { + /* already unpublished */ + } + })(); + }); + const stopAudio = async () => { + try { + if (audioPub.track) await lp.unpublishTrack(audioPub.track); + } catch { + /* already unpublished */ + } + const active = nativeAudioCaptureRef.current; + if (active && active.captureId === audioHandle.captureId) { + nativeAudioCaptureRef.current = null; + await active.stop().catch(() => undefined); + } + }; + return { stopAudio }; + } catch (err: unknown) { + console.warn( + 'screen-share: native system-audio unavailable, sharing video only', + err instanceof Error ? err.message : err, + ); + if (nativeAudioCaptureRef.current) { + await nativeAudioCaptureRef.current.stop().catch(() => undefined); + nativeAudioCaptureRef.current = null; + } + return null; + } + }; + + // ----- Path 1: chromeMediaSource (fast) ----- if (sourceId) { try { - console.info('screen-share: trying native capture path', { - sourceId, - fps, - includeSystemAudio: settings.includeSystemAudio, + const { Track: LkTrack } = await import('livekit-client'); + const maxWidth = ssParams.dims?.width ?? 3840; + const maxHeight = ssParams.dims?.height ?? 2160; + // Cast chains: browsers expose the legacy constraint via + // `MediaTrackConstraints.mandatory` which isn't in lib.dom. + const videoConstraints = { + mandatory: { + chromeMediaSource: 'desktop', + chromeMediaSourceId: sourceId, + maxWidth, + maxHeight, + maxFrameRate: fps, + }, + } as unknown as MediaTrackConstraints; + const stream = await navigator.mediaDevices.getUserMedia({ + audio: false, + video: videoConstraints, }); + const videoMst = stream.getVideoTracks()[0]; + if (!videoMst) { + stream.getTracks().forEach((t) => t.stop()); + throw new Error('no video track from chromeMediaSource'); + } + const videoPub = await lp.publishTrack(videoMst, { + source: LkTrack.Source.ScreenShare, + videoCodec: 'vp9', + }); + const audioRes = await startWasapiAudio(LkTrack); + videoMst.addEventListener('ended', () => { + void (async () => { + try { + if (videoPub.track) await lp.unpublishTrack(videoPub.track); + } catch { + /* already unpublished */ + } + if (audioRes) await audioRes.stopAudio(); + setIsScreenSharing(false); + })(); + }); + console.info('screen-share: chromeMediaSource active', { + audio: audioRes != null, + }); + setIsScreenSharing(true); + return; + } catch (err: unknown) { + console.warn( + 'screen-share: chromeMediaSource failed, trying native xcap', + err instanceof Error ? err.message : err, + ); + } + } + + // ----- Path 2: xcap native (CPU fallback) ----- + if (sourceId) { + try { const { Track: LkTrack } = await import('livekit-client'); const maxWidth = ssParams.dims?.width ?? 1920; const maxHeight = ssParams.dims?.height ?? 1080; @@ -1243,54 +1362,7 @@ export function CallProvider({ children }: { children: ReactNode }) { source: LkTrack.Source.ScreenShare, videoCodec: 'vp9', }); - - // Optional native audio. Failure here is non-fatal — the video - // pipeline is already running and bailing out would be worse - // UX than shipping a silent share. The warning surfaces the - // platform gap so the user knows why their audio is missing. - let audioHandle: SystemAudioHandle | null = null; - if (settings.includeSystemAudio) { - try { - audioHandle = await startSystemAudioCapture(); - nativeAudioCaptureRef.current = audioHandle; - const audioMst = audioHandle.stream.getAudioTracks()[0]; - if (audioMst) { - const audioPub = await lp.publishTrack(audioMst, { - source: LkTrack.Source.ScreenShareAudio, - }); - audioMst.addEventListener('ended', () => { - void (async () => { - try { - if (audioPub.track) await lp.unpublishTrack(audioPub.track); - } catch { - /* already unpublished */ - } - const active = nativeAudioCaptureRef.current; - if (active && active.captureId === audioHandle!.captureId) { - nativeAudioCaptureRef.current = null; - await active.stop().catch(() => undefined); - } - })(); - }); - } - } catch (err: unknown) { - console.warn( - 'screen-share: native system-audio unavailable, sharing video only', - err instanceof Error ? err.message : err, - ); - if (nativeAudioCaptureRef.current) { - await nativeAudioCaptureRef.current.stop().catch(() => undefined); - nativeAudioCaptureRef.current = null; - } - audioHandle = null; - } - } - - // Canvas stream 'ended' fires on handle.stop() (we track.stop() - // each track) — chain unpublish + native teardown so one ended - // event cleans everything up regardless of who triggered it. - // Also tear down any paired audio capture so sound can't - // outlive the video share. + const audioRes = await startWasapiAudio(LkTrack); videoMst.addEventListener('ended', () => { void (async () => { try { @@ -1303,23 +1375,16 @@ export function CallProvider({ children }: { children: ReactNode }) { nativeCaptureRef.current = null; await active.stop().catch(() => undefined); } - const audioActive = nativeAudioCaptureRef.current; - if (audioActive) { - nativeAudioCaptureRef.current = null; - await audioActive.stop().catch(() => undefined); - } + if (audioRes) await audioRes.stopAudio(); setIsScreenSharing(false); })(); }); - console.info('screen-share: native capture active', { - audio: audioHandle != null, + console.info('screen-share: native xcap active', { + audio: audioRes != null, }); setIsScreenSharing(true); return; } catch (err: unknown) { - // Native path unavailable (non-Tauri runtime, source vanished, - // first-frame timeout). Clean up any partial handle and fall - // through to the getUserMedia / getDisplayMedia paths. if (nativeCaptureRef.current) { await nativeCaptureRef.current.stop().catch(() => undefined); nativeCaptureRef.current = null; @@ -1329,101 +1394,12 @@ export function CallProvider({ children }: { children: ReactNode }) { nativeAudioCaptureRef.current = null; } console.warn( - 'screen-share: native path failed, falling back', + 'screen-share: native xcap failed, falling back to OS picker', err instanceof Error ? err.message : err, ); } } - // Direct-publish path when our custom picker supplied a Chromium- - // format source id. Bypasses the OS picker so the user shares exactly - // the window/monitor they clicked in the grid. getUserMedia with the - // legacy chromeMediaSourceId constraint is not in the MediaStream - // spec but is honoured by Chromium / WebView2. If it throws we fall - // through to setScreenShareEnabled and let the OS picker run. - if (sourceId) { - try { - const { Track: LkTrack } = await import('livekit-client'); - const maxWidth = ssParams.dims?.width ?? 3840; - const maxHeight = ssParams.dims?.height ?? 2160; - // Cast chains: browsers expose the legacy constraint via - // `MediaTrackConstraints.mandatory` which isn't in lib.dom. - const videoConstraints = { - mandatory: { - chromeMediaSource: 'desktop', - chromeMediaSourceId: sourceId, - maxWidth, - maxHeight, - maxFrameRate: fps, - }, - } as unknown as MediaTrackConstraints; - const audioConstraints: MediaTrackConstraints | false = settings.includeSystemAudio - ? ({ - mandatory: { - chromeMediaSource: 'desktop', - chromeMediaSourceId: sourceId, - }, - } as unknown as MediaTrackConstraints) - : false; - const stream = await navigator.mediaDevices.getUserMedia({ - audio: audioConstraints, - video: videoConstraints, - }); - const videoMst = stream.getVideoTracks()[0]; - const audioMst = stream.getAudioTracks()[0]; - if (!videoMst) { - stream.getTracks().forEach((t) => t.stop()); - throw new Error('no video track from chromeMediaSource'); - } - // Pass raw MediaStreamTracks — `publishTrack` wraps them in the - // right Local*Track internally and the publishDefaults on the - // Room handle VP9 codec + screenShareEncoding caps. Passing the - // raw tracks also sidesteps a type incompatibility between - // livekit-client's Local*Track and our exactOptionalPropertyTypes - // setting. - const videoPub = await lp.publishTrack(videoMst, { - source: LkTrack.Source.ScreenShare, - videoCodec: 'vp9', - }); - // Stop the publish when the OS revokes capture (user hit the - // OS "Stop sharing" banner, or closed the window we were sharing). - videoMst.addEventListener('ended', () => { - void (async () => { - try { - if (videoPub.track) await lp.unpublishTrack(videoPub.track); - } catch { - /* already unpublished */ - } - setIsScreenSharing(false); - })(); - }); - if (audioMst) { - const audioPub = await lp.publishTrack(audioMst, { - source: LkTrack.Source.ScreenShareAudio, - }); - audioMst.addEventListener('ended', () => { - void (async () => { - try { - if (audioPub.track) await lp.unpublishTrack(audioPub.track); - } catch { - /* ignore */ - } - })(); - }); - } - setIsScreenSharing(true); - return; - } catch (err: unknown) { - // WebView2 / browser rejected the legacy constraint. Fall through - // to the normal OS picker path below so the user still gets a - // working share instead of a hard error. - console.warn( - 'direct screen-share via chromeMediaSourceId failed; falling back to getDisplayMedia', - err, - ); - } - } - try { await lp.setScreenShareEnabled(true, { // "Go live" mode — capture system audio alongside the screen when