byGalax/ChatApp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(call): screen-share hybrid — OS picker + WASAPI audio

Browse Source 
Strip the sourceId-gated chromeMediaSource + xcap native capture paths
from startScreenShare and collapse to a single setScreenShareEnabled
call. Neither of the bypassed paths produced smooth frames in WebView2:
chromeMediaSource: 'desktop' is an extension-only Chromium constraint
and throws outside extension origins, and the xcap JPEG-over-IPC
fallback couldn't sustain 30fps at 1080p on a single main-thread.
setScreenShareEnabled goes through Chromium's native getDisplayMedia
capture, which is the only path that gets HW-accelerated frames into
the WebRTC encoder from WebView2.

Audio continues via the WASAPI loopback module — getDisplayMedia can't
grab system sound in WebView2 without desktop-capture entitlements
Chromium reserves for extensions. The audio track's teardown chains to
the ScreenShare video track's 'ended' event so the Windows stop-share
overlay kills both sides in lockstep.

ScreenSourcePicker is now a quality + audio chooser only; the
thumbnail grid disappears because custom source IDs don't round-trip
through WebView2, and a custom picker in front of the OS picker just
means the user picks twice.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
byGalax
2026-04-23 17:59:41 +02:00
parent adbdfaa2aa
commit c9b885b91c
3 changed files with 149 additions and 589 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/desktop/src/context/CallContext.tsx
+78 -265
View File
@@ -75,11 +75,6 @@ import {
setAllPipelinesSinkId,
setPipelineGain,
} from '../lib/remoteAudioPipelines';
import {
type NativeCaptureHandle,
NativeCaptureUnavailable,
startNativeCapture,
} from '../lib/screenCapture';
import {
type SystemAudioHandle,
startSystemAudioCapture,
@@ -304,15 +299,10 @@ export function CallProvider({ children }: { children: ReactNode }) {
// to `connected` after a few seconds so the UI doesn't hang in "Verbinde…"
// indefinitely. The solo-timeout will then cleanly close if nobody arrives.
const joinFallbackTimerRef = useRef<number | null>(null);
// Active native screen-capture handle (Rust side). Set when
// startScreenShare takes the xcap path; cleared on stopScreenShare or
// on the canvas track's 'ended' event. Not kept in React state because
// it never feeds into a render.
const nativeCaptureRef = useRef<NativeCaptureHandle | null>(null);
// Matching handle for the Windows-only WASAPI system-audio capture.
// Lives in lockstep with the video handle above when the user picks
// "Mit System-Sound"; teardown is wired so that stopping either track
// also stops the other, so stale audio can't outlive the video share.
// Handle for the Windows-only WASAPI system-audio capture. Lives in
// lockstep with the ScreenShare video track published by
// setScreenShareEnabled; teardown is chained to the video track's
// `ended` event so stale audio can never outlive the video share.
const nativeAudioCaptureRef = useRef<SystemAudioHandle | null>(null);
const roomRef = useRef<Room | null>(null);
// Web Audio graph that mixes live mic + soundboard sources into a single
@@ -488,13 +478,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
/* ignore */
}
}
// Stop any lingering native screen-capture thread so we don't leak
// Rust threads when the call ends mid-share.
if (nativeCaptureRef.current) {
const h = nativeCaptureRef.current;
nativeCaptureRef.current = null;
await h.stop().catch(() => undefined);
}
if (nativeAudioCaptureRef.current) {
const h = nativeAudioCaptureRef.current;
nativeAudioCaptureRef.current = null;
@@ -729,6 +712,11 @@ export function CallProvider({ children }: { children: ReactNode }) {
r.on(RoomEvent.ParticipantConnected, () => {
setRemoteParticipants(Array.from(r.remoteParticipants.values()));
if (presenceRef.current !== 'dnd') void playJoinBeep();
// Rejoin während wir schon `connected` sind: markConnectedIfReady
// returnt früh und würde den Solo-Timer nicht cancellen. Hier
// unbedingt clearen, sonst kickt der 5-Minuten-Timer obwohl der
// andere Peer längst wieder im Raum ist.
clearSoloTimer();
markConnectedIfReady(r, conversationId, mediaKind, callId);
});
@@ -1173,11 +1161,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
preset: ScreenSharePreset;
displaySurface: DisplaySurfaceHint;
framerate: number | null;
/** Chromium-format source id from our custom picker. When set, we
* try to capture that exact source via `chromeMediaSourceId`
* instead of the OS-level getDisplayMedia picker. Falls back to
* getDisplayMedia if WebView2 rejects the constraint. */
sourceId: string | null;
}>,
) => {
const r = roomRef.current;
@@ -1185,8 +1168,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
const lp = r.localParticipant;
if (lp.isScreenShareEnabled) return;
// Persist the user's choice so subsequent shares use the same config
// without re-opening the picker unless they want to change something.
const settings = getScreenShareSettings();
const preset = overrides?.preset ?? settings.preset;
const displaySurface =
@@ -1201,220 +1182,17 @@ export function CallProvider({ children }: { children: ReactNode }) {
const ssParams = getPresetParams(preset);
const fps = framerateOverride ?? ssParams.framerate;
const sourceId = overrides?.sourceId ?? null;
// Ordered capture paths when our custom picker supplied a sourceId:
// 1. chromeMediaSource video — hardware-accelerated path that
// hands the frames directly to WebRTC, same mechanism the OS
// picker uses internally. Fast, smooth, what users expect.
// 2. xcap native (JPEG → canvas → captureStream) — fallback for
// WebView2 versions that reject the legacy chromeMediaSource
// constraint. Functional but CPU-heavy; the per-frame JPEG
// encode + base64 + createImageBitmap chain shows up as
// visible stutter on 1080p30.
// 3. OS picker (setScreenShareEnabled) — last resort when both
// native paths throw, and the only path when no sourceId was
// supplied.
//
// Audio (Windows only) always goes through the WASAPI loopback
// module — getUserMedia's chromeMediaSource audio constraint
// throws AbortError on Window captures, and pairing both into one
// call ended up blocking the video path entirely. Splitting them
// means the video path stays fast and the audio path stays
// reliable for any source type.
if (!sourceId) {
console.info(
'screen-share: no sourceId supplied by picker, OS picker will open',
);
}
// Publish system audio via WASAPI. Resolves to the audio publication
// + an ended-cleanup registration so whichever video path succeeds
// can wire the audio teardown onto its own video-ended handler.
const startWasapiAudio = async (
LkTrack: typeof import('livekit-client').Track,
): Promise<{
stopAudio: () => Promise<void>;
} | null> => {
if (!settings.includeSystemAudio) return null;
try {
const audioHandle = await startSystemAudioCapture();
nativeAudioCaptureRef.current = audioHandle;
const audioMst = audioHandle.stream.getAudioTracks()[0];
if (!audioMst) {
await audioHandle.stop();
nativeAudioCaptureRef.current = null;
return null;
}
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
});
audioMst.addEventListener('ended', () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
})();
});
const stopAudio = async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
const active = nativeAudioCaptureRef.current;
if (active && active.captureId === audioHandle.captureId) {
nativeAudioCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
};
return { stopAudio };
} catch (err: unknown) {
console.warn(
'screen-share: native system-audio unavailable, sharing video only',
err instanceof Error ? err.message : err,
);
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
return null;
}
};
// ----- Path 1: chromeMediaSource (fast) -----
if (sourceId) {
try {
const { Track: LkTrack } = await import('livekit-client');
const maxWidth = ssParams.dims?.width ?? 3840;
const maxHeight = ssParams.dims?.height ?? 2160;
// Cast chains: browsers expose the legacy constraint via
// `MediaTrackConstraints.mandatory` which isn't in lib.dom.
const videoConstraints = {
mandatory: {
chromeMediaSource: 'desktop',
chromeMediaSourceId: sourceId,
maxWidth,
maxHeight,
maxFrameRate: fps,
},
} as unknown as MediaTrackConstraints;
const stream = await navigator.mediaDevices.getUserMedia({
audio: false,
video: videoConstraints,
});
const videoMst = stream.getVideoTracks()[0];
if (!videoMst) {
stream.getTracks().forEach((t) => t.stop());
throw new Error('no video track from chromeMediaSource');
}
// H.264 lets Chromium's hardware encoder take over on Windows
// (Intel QuickSync / NVENC / AMD VCE) — VP9 is software-only in
// Chromium's WebRTC path and collapses under the L3T3_KEY
// scalability mode we set globally for camera. `contentHint` is
// what setScreenShareEnabled sets internally; pushing it by hand
// here tells the encoder to weight sharpness over smoothness,
// which is right for UI/text-heavy screen content.
videoMst.contentHint = 'detail';
const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare,
videoCodec: 'h264',
});
const audioRes = await startWasapiAudio(LkTrack);
videoMst.addEventListener('ended', () => {
void (async () => {
try {
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
} catch {
/* already unpublished */
}
if (audioRes) await audioRes.stopAudio();
setIsScreenSharing(false);
})();
});
console.info('screen-share: chromeMediaSource active', {
audio: audioRes != null,
});
setIsScreenSharing(true);
return;
} catch (err: unknown) {
console.warn(
'screen-share: chromeMediaSource failed, trying native xcap',
err instanceof Error ? err.message : err,
);
}
}
// ----- Path 2: xcap native (CPU fallback) -----
if (sourceId) {
try {
const { Track: LkTrack } = await import('livekit-client');
const maxWidth = ssParams.dims?.width ?? 1920;
const maxHeight = ssParams.dims?.height ?? 1080;
const handle = await startNativeCapture({
sourceId,
maxWidth,
maxHeight,
fps,
});
nativeCaptureRef.current = handle;
const videoMst = handle.stream.getVideoTracks()[0];
if (!videoMst) {
await handle.stop();
nativeCaptureRef.current = null;
throw new NativeCaptureUnavailable('canvas stream produced no video track');
}
videoMst.contentHint = 'detail';
const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare,
videoCodec: 'h264',
});
const audioRes = await startWasapiAudio(LkTrack);
videoMst.addEventListener('ended', () => {
void (async () => {
try {
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
} catch {
/* already unpublished */
}
const active = nativeCaptureRef.current;
if (active && active.captureId === handle.captureId) {
nativeCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
if (audioRes) await audioRes.stopAudio();
setIsScreenSharing(false);
})();
});
console.info('screen-share: native xcap active', {
audio: audioRes != null,
});
setIsScreenSharing(true);
return;
} catch (err: unknown) {
if (nativeCaptureRef.current) {
await nativeCaptureRef.current.stop().catch(() => undefined);
nativeCaptureRef.current = null;
}
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
console.warn(
'screen-share: native xcap failed, falling back to OS picker',
err instanceof Error ? err.message : err,
);
}
}
// Single capture path: setScreenShareEnabled → OS picker → Chromium's
// hardware-accelerated capture feeds WebRTC directly. In WebView2
// the extension-only `chromeMediaSource: 'desktop'` constraint and
// our xcap JPEG-over-IPC pipeline were both too slow for smooth
// 30fps, so the OS picker is the only route to the fast pipeline.
// Audio is handled separately below via WASAPI because
// getDisplayMedia can't grab system audio in WebView2.
try {
await lp.setScreenShareEnabled(true, {
// "Go live" mode — capture system audio alongside the screen when
// the user opted in. On hosts that can't fulfil the request the
// browser quietly drops it; peers just get video-only, no error.
audio: settings.includeSystemAudio,
audio: false,
...(ssParams.dims
? {
resolution: {
@@ -1430,9 +1208,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
frameRate: fps,
},
}),
// Hints the OS picker to pre-filter by source kind. `null` = no
// filter (show both). Cast because TS lib.dom doesn't know the
// field yet on all branches.
...(displaySurface
? ({ displaySurface } as { displaySurface: DisplaySurfaceHint })
: {}),
@@ -1441,24 +1216,67 @@ export function CallProvider({ children }: { children: ReactNode }) {
setIsScreenSharing(true);
} catch (err: unknown) {
console.error('setScreenShareEnabled failed', err);
return;
}
if (!settings.includeSystemAudio) return;
// WASAPI system-audio: publish as a separate ScreenShareAudio track.
// Chain its teardown onto the ScreenShare video track's `ended`
// event so the Windows "Stop sharing" overlay kills audio alongside
// video; the audio track's own `ended` handler covers the case
// where WASAPI dies by itself.
try {
const { Track: LkTrack } = await import('livekit-client');
const audioHandle = await startSystemAudioCapture();
nativeAudioCaptureRef.current = audioHandle;
const audioMst = audioHandle.stream.getAudioTracks()[0];
if (!audioMst) {
await audioHandle.stop();
nativeAudioCaptureRef.current = null;
return;
}
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
});
let videoMst: MediaStreamTrack | null = null;
for (const pub of lp.videoTrackPublications.values()) {
if (pub.source === LkTrack.Source.ScreenShare && pub.track) {
videoMst = pub.track.mediaStreamTrack;
break;
}
}
const teardown = () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
const active = nativeAudioCaptureRef.current;
if (active && active.captureId === audioHandle.captureId) {
nativeAudioCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
})();
};
audioMst.addEventListener('ended', teardown);
if (videoMst) videoMst.addEventListener('ended', teardown);
} catch (err: unknown) {
console.warn(
'screen-share: native system-audio unavailable, sharing video only',
err instanceof Error ? err.message : err,
);
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
}
},
[],
);
const stopScreenShare = useCallback(async () => {
// Native path first — stopping the handle kills the canvas track,
// which fires 'ended' on the MediaStreamTrack, which the start-handler
// already listens to for unpublishing and flipping isScreenSharing.
if (nativeCaptureRef.current) {
const h = nativeCaptureRef.current;
nativeCaptureRef.current = null;
try {
await h.stop();
} catch (err: unknown) {
console.warn('native capture stop failed', err);
}
}
if (nativeAudioCaptureRef.current) {
const h = nativeAudioCaptureRef.current;
nativeAudioCaptureRef.current = null;
@@ -1471,18 +1289,13 @@ export function CallProvider({ children }: { children: ReactNode }) {
const r = roomRef.current;
if (!r) return;
const lp = r.localParticipant;
// Unpublish any manually-published ScreenShare/ScreenShareAudio tracks
// (from the chromeMediaSourceId fallback path). setScreenShareEnabled
// only manages LK's own internally-captured tracks.
const toUnpublish: import('livekit-client').LocalTrackPublication[] = [];
for (const pub of lp.videoTrackPublications.values()) {
if (pub.source === Track.Source.ScreenShare && pub.track) toUnpublish.push(pub);
}
// Unpublish any ScreenShareAudio track we manually published alongside
// setScreenShareEnabled. LiveKit's setScreenShareEnabled(false) only
// drops the video track it captured itself.
for (const pub of lp.audioTrackPublications.values()) {
if (pub.source === Track.Source.ScreenShareAudio && pub.track) toUnpublish.push(pub);
}
for (const pub of toUnpublish) {
if (pub.track) await lp.unpublishTrack(pub.track).catch(() => undefined);
if (pub.source === Track.Source.ScreenShareAudio && pub.track) {
await lp.unpublishTrack(pub.track).catch(() => undefined);
}
}
if (lp.isScreenShareEnabled) {
try {