feat(call): native screen-capture pipeline + faster picker
Picker speed (Phase 1+2): - screen_sources.rs split into list_screen_sources (metadata only, returns in ~10ms) + capture_screen_source_thumbnail (single source, by id). ScreenSourcePicker now shows names + placeholders instantly and streams thumbnails in as each capture lands. Total wall-clock is bounded by the slowest source instead of the serial sum. - enumerate_screen_sources kept as a dead_code fallback so any rollout regression can switch the frontend back without code loss. Native capture (Phase 3): - New src-tauri/src/screen_capture.rs. start_screen_capture spawns a Rust thread per share that grabs frames via xcap, downscales to the user's quality preset, JPEG-encodes at Q72, and streams each frame through a Tauri Channel<FramePayload>. stop_screen_capture signals the stop flag and joins the worker. - Worker re-resolves the xcap handle inside the thread because xcap::Window holds a !Send HWND — passing the source id string across the thread boundary sidesteps that. - New lib/screenCapture.ts: decodes each frame into an ImageBitmap, draws to an offscreen canvas, exposes canvas.captureStream() as the MediaStream LiveKit publishes. Latest-wins frame queue drops stale frames when the JS side falls behind the Rust producer. 3s first- frame timeout so a silently-failing source (locked screen, DRM window) surfaces as a clean NativeCaptureUnavailable and we fall back to getDisplayMedia. - CallContext.startScreenShare takes the native path first when the picker provided a sourceId and system audio wasn't requested. The old chromeMediaSourceId attempt and final setScreenShareEnabled fallback stay in place for the audio case + non-Tauri runtimes. - stopScreenShare kills the native handle first, then unpublishes any manually-published ScreenShare/ScreenShareAudio tracks, then falls back to setScreenShareEnabled(false). disconnectRoom also stops the handle so we don't leak Rust threads across calls. Scope note: native path is video-only. System-audio capture needs WASAPI-loopback (Windows) or ScreenCaptureKit-audio (macOS); until those are wired, requesting audio in the picker falls through to the getDisplayMedia path and shows the OS picker for that one case. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
byGalax
@@ -75,6 +75,11 @@ import {
|
||||
setAllPipelinesSinkId,
|
||||
setPipelineGain,
|
||||
} from '../lib/remoteAudioPipelines';
|
||||
import {
|
||||
type NativeCaptureHandle,
|
||||
NativeCaptureUnavailable,
|
||||
startNativeCapture,
|
||||
} from '../lib/screenCapture';
|
||||
import {
|
||||
clearScreenShareVolumes,
|
||||
getScreenShareVolume,
|
||||
@@ -295,6 +300,11 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
// to `connected` after a few seconds so the UI doesn't hang in "Verbinde…"
|
||||
// indefinitely. The solo-timeout will then cleanly close if nobody arrives.
|
||||
const joinFallbackTimerRef = useRef<number | null>(null);
|
||||
// Active native screen-capture handle (Rust side). Set when
|
||||
// startScreenShare takes the xcap path; cleared on stopScreenShare or
|
||||
// on the canvas track's 'ended' event. Not kept in React state because
|
||||
// it never feeds into a render.
|
||||
const nativeCaptureRef = useRef<NativeCaptureHandle | null>(null);
|
||||
const roomRef = useRef<Room | null>(null);
|
||||
// Web Audio graph that mixes live mic + soundboard sources into a single
|
||||
// published track. Created per call in joinRoom, destroyed in disconnectRoom.
|
||||
@@ -469,6 +479,13 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
// Stop any lingering native screen-capture thread so we don't leak
|
||||
// Rust threads when the call ends mid-share.
|
||||
if (nativeCaptureRef.current) {
|
||||
const h = nativeCaptureRef.current;
|
||||
nativeCaptureRef.current = null;
|
||||
await h.stop().catch(() => undefined);
|
||||
}
|
||||
roomRef.current = null;
|
||||
setRoom(null);
|
||||
setRemoteParticipants([]);
|
||||
@@ -1172,6 +1189,69 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
const fps = framerateOverride ?? ssParams.framerate;
|
||||
const sourceId = overrides?.sourceId ?? null;
|
||||
|
||||
// Native capture path — tried first when the user came in via our
|
||||
// custom picker. xcap on the Rust side grabs frames, streams them as
|
||||
// JPEG over a Tauri channel, and we draw them onto a canvas whose
|
||||
// captureStream() becomes the MediaStream LiveKit publishes. This
|
||||
// completely skips the OS picker. Video-only (no system audio yet);
|
||||
// if the user asked for audio we fall through to the legacy paths
|
||||
// below so audio still works via getDisplayMedia.
|
||||
if (sourceId && !settings.includeSystemAudio) {
|
||||
try {
|
||||
const { Track: LkTrack } = await import('livekit-client');
|
||||
const maxWidth = ssParams.dims?.width ?? 1920;
|
||||
const maxHeight = ssParams.dims?.height ?? 1080;
|
||||
const handle = await startNativeCapture({
|
||||
sourceId,
|
||||
maxWidth,
|
||||
maxHeight,
|
||||
fps,
|
||||
});
|
||||
nativeCaptureRef.current = handle;
|
||||
const videoMst = handle.stream.getVideoTracks()[0];
|
||||
if (!videoMst) {
|
||||
await handle.stop();
|
||||
nativeCaptureRef.current = null;
|
||||
throw new NativeCaptureUnavailable('canvas stream produced no video track');
|
||||
}
|
||||
const videoPub = await lp.publishTrack(videoMst, {
|
||||
source: LkTrack.Source.ScreenShare,
|
||||
videoCodec: 'vp9',
|
||||
});
|
||||
// Canvas stream 'ended' fires on handle.stop() (we track.stop()
|
||||
// each track) — chain unpublish + native teardown so one ended
|
||||
// event cleans everything up regardless of who triggered it.
|
||||
videoMst.addEventListener('ended', () => {
|
||||
void (async () => {
|
||||
try {
|
||||
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
|
||||
} catch {
|
||||
/* already unpublished */
|
||||
}
|
||||
const active = nativeCaptureRef.current;
|
||||
if (active && active.captureId === handle.captureId) {
|
||||
nativeCaptureRef.current = null;
|
||||
await active.stop().catch(() => undefined);
|
||||
}
|
||||
setIsScreenSharing(false);
|
||||
})();
|
||||
});
|
||||
setIsScreenSharing(true);
|
||||
return;
|
||||
} catch (err: unknown) {
|
||||
// Native path unavailable (non-Tauri runtime, source vanished,
|
||||
// first-frame timeout). Clean up any partial handle and fall
|
||||
// through to the getUserMedia / getDisplayMedia paths.
|
||||
if (nativeCaptureRef.current) {
|
||||
await nativeCaptureRef.current.stop().catch(() => undefined);
|
||||
nativeCaptureRef.current = null;
|
||||
}
|
||||
if (!(err instanceof NativeCaptureUnavailable)) {
|
||||
console.warn('native screen-capture failed, falling back', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Direct-publish path when our custom picker supplied a Chromium-
|
||||
// format source id. Bypasses the OS picker so the user shares exactly
|
||||
// the window/monitor they clicked in the grid. getUserMedia with the
|
||||
@@ -1299,16 +1379,42 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
);
|
||||
|
||||
const stopScreenShare = useCallback(async () => {
|
||||
// Native path first — stopping the handle kills the canvas track,
|
||||
// which fires 'ended' on the MediaStreamTrack, which the start-handler
|
||||
// already listens to for unpublishing and flipping isScreenSharing.
|
||||
if (nativeCaptureRef.current) {
|
||||
const h = nativeCaptureRef.current;
|
||||
nativeCaptureRef.current = null;
|
||||
try {
|
||||
await h.stop();
|
||||
} catch (err: unknown) {
|
||||
console.warn('native capture stop failed', err);
|
||||
}
|
||||
}
|
||||
const r = roomRef.current;
|
||||
if (!r) return;
|
||||
const lp = r.localParticipant;
|
||||
if (!lp.isScreenShareEnabled) return;
|
||||
try {
|
||||
await lp.setScreenShareEnabled(false);
|
||||
setIsScreenSharing(false);
|
||||
} catch (err: unknown) {
|
||||
console.error('stopScreenShare failed', err);
|
||||
// Unpublish any manually-published ScreenShare/ScreenShareAudio tracks
|
||||
// (from the chromeMediaSourceId fallback path). setScreenShareEnabled
|
||||
// only manages LK's own internally-captured tracks.
|
||||
const toUnpublish: import('livekit-client').LocalTrackPublication[] = [];
|
||||
for (const pub of lp.videoTrackPublications.values()) {
|
||||
if (pub.source === Track.Source.ScreenShare && pub.track) toUnpublish.push(pub);
|
||||
}
|
||||
for (const pub of lp.audioTrackPublications.values()) {
|
||||
if (pub.source === Track.Source.ScreenShareAudio && pub.track) toUnpublish.push(pub);
|
||||
}
|
||||
for (const pub of toUnpublish) {
|
||||
if (pub.track) await lp.unpublishTrack(pub.track).catch(() => undefined);
|
||||
}
|
||||
if (lp.isScreenShareEnabled) {
|
||||
try {
|
||||
await lp.setScreenShareEnabled(false);
|
||||
} catch (err: unknown) {
|
||||
console.error('stopScreenShare failed', err);
|
||||
}
|
||||
}
|
||||
setIsScreenSharing(false);
|
||||
}, []);
|
||||
|
||||
// Legacy toggle kept for convenience elsewhere — opens/closes with the
|
||||
|
||||
Reference in New Issue
Block a user