|
|
|
@@ -1203,144 +1203,88 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
|
|
|
|
const fps = framerateOverride ?? ssParams.framerate;
|
|
|
|
|
const sourceId = overrides?.sourceId ?? null;
|
|
|
|
|
|
|
|
|
|
// Native capture path — tried first when the user came in via our
|
|
|
|
|
// custom picker. xcap on the Rust side grabs video frames and, on
|
|
|
|
|
// Windows with "Mit System-Sound" on, the WASAPI loopback module
|
|
|
|
|
// grabs the render endpoint. Both stream over Tauri channels into
|
|
|
|
|
// tracks we publish directly to LiveKit — the OS picker never
|
|
|
|
|
// appears. If native audio fails on a platform that can't supply
|
|
|
|
|
// it (non-Windows v1), we continue with video-only and log; the
|
|
|
|
|
// user still gets their direct-video share.
|
|
|
|
|
// Ordered capture paths when our custom picker supplied a sourceId:
|
|
|
|
|
// 1. chromeMediaSource video — hardware-accelerated path that
|
|
|
|
|
// hands the frames directly to WebRTC, same mechanism the OS
|
|
|
|
|
// picker uses internally. Fast, smooth, what users expect.
|
|
|
|
|
// 2. xcap native (JPEG → canvas → captureStream) — fallback for
|
|
|
|
|
// WebView2 versions that reject the legacy chromeMediaSource
|
|
|
|
|
// constraint. Functional but CPU-heavy; the per-frame JPEG
|
|
|
|
|
// encode + base64 + createImageBitmap chain shows up as
|
|
|
|
|
// visible stutter on 1080p30.
|
|
|
|
|
// 3. OS picker (setScreenShareEnabled) — last resort when both
|
|
|
|
|
// native paths throw, and the only path when no sourceId was
|
|
|
|
|
// supplied.
|
|
|
|
|
//
|
|
|
|
|
// Audio (Windows only) always goes through the WASAPI loopback
|
|
|
|
|
// module — getUserMedia's chromeMediaSource audio constraint
|
|
|
|
|
// throws AbortError on Window captures, and pairing both into one
|
|
|
|
|
// call ended up blocking the video path entirely. Splitting them
|
|
|
|
|
// means the video path stays fast and the audio path stays
|
|
|
|
|
// reliable for any source type.
|
|
|
|
|
if (!sourceId) {
|
|
|
|
|
console.info(
|
|
|
|
|
'screen-share: no sourceId supplied by picker, OS picker will open',
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if (sourceId) {
|
|
|
|
|
|
|
|
|
|
// Publish system audio via WASAPI. Resolves to the audio publication
|
|
|
|
|
// + an ended-cleanup registration so whichever video path succeeds
|
|
|
|
|
// can wire the audio teardown onto its own video-ended handler.
|
|
|
|
|
const startWasapiAudio = async (
|
|
|
|
|
LkTrack: typeof import('livekit-client').Track,
|
|
|
|
|
): Promise<{
|
|
|
|
|
stopAudio: () => Promise<void>;
|
|
|
|
|
} | null> => {
|
|
|
|
|
if (!settings.includeSystemAudio) return null;
|
|
|
|
|
try {
|
|
|
|
|
console.info('screen-share: trying native capture path', {
|
|
|
|
|
sourceId,
|
|
|
|
|
fps,
|
|
|
|
|
includeSystemAudio: settings.includeSystemAudio,
|
|
|
|
|
});
|
|
|
|
|
const { Track: LkTrack } = await import('livekit-client');
|
|
|
|
|
const maxWidth = ssParams.dims?.width ?? 1920;
|
|
|
|
|
const maxHeight = ssParams.dims?.height ?? 1080;
|
|
|
|
|
const handle = await startNativeCapture({
|
|
|
|
|
sourceId,
|
|
|
|
|
maxWidth,
|
|
|
|
|
maxHeight,
|
|
|
|
|
fps,
|
|
|
|
|
});
|
|
|
|
|
nativeCaptureRef.current = handle;
|
|
|
|
|
const videoMst = handle.stream.getVideoTracks()[0];
|
|
|
|
|
if (!videoMst) {
|
|
|
|
|
await handle.stop();
|
|
|
|
|
nativeCaptureRef.current = null;
|
|
|
|
|
throw new NativeCaptureUnavailable('canvas stream produced no video track');
|
|
|
|
|
const audioHandle = await startSystemAudioCapture();
|
|
|
|
|
nativeAudioCaptureRef.current = audioHandle;
|
|
|
|
|
const audioMst = audioHandle.stream.getAudioTracks()[0];
|
|
|
|
|
if (!audioMst) {
|
|
|
|
|
await audioHandle.stop();
|
|
|
|
|
nativeAudioCaptureRef.current = null;
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
const videoPub = await lp.publishTrack(videoMst, {
|
|
|
|
|
source: LkTrack.Source.ScreenShare,
|
|
|
|
|
videoCodec: 'vp9',
|
|
|
|
|
const audioPub = await lp.publishTrack(audioMst, {
|
|
|
|
|
source: LkTrack.Source.ScreenShareAudio,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Optional native audio. Failure here is non-fatal — the video
|
|
|
|
|
// pipeline is already running and bailing out would be worse
|
|
|
|
|
// UX than shipping a silent share. The warning surfaces the
|
|
|
|
|
// platform gap so the user knows why their audio is missing.
|
|
|
|
|
let audioHandle: SystemAudioHandle | null = null;
|
|
|
|
|
if (settings.includeSystemAudio) {
|
|
|
|
|
try {
|
|
|
|
|
audioHandle = await startSystemAudioCapture();
|
|
|
|
|
nativeAudioCaptureRef.current = audioHandle;
|
|
|
|
|
const audioMst = audioHandle.stream.getAudioTracks()[0];
|
|
|
|
|
if (audioMst) {
|
|
|
|
|
const audioPub = await lp.publishTrack(audioMst, {
|
|
|
|
|
source: LkTrack.Source.ScreenShareAudio,
|
|
|
|
|
});
|
|
|
|
|
audioMst.addEventListener('ended', () => {
|
|
|
|
|
void (async () => {
|
|
|
|
|
try {
|
|
|
|
|
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
|
|
|
|
|
} catch {
|
|
|
|
|
/* already unpublished */
|
|
|
|
|
}
|
|
|
|
|
const active = nativeAudioCaptureRef.current;
|
|
|
|
|
if (active && active.captureId === audioHandle!.captureId) {
|
|
|
|
|
nativeAudioCaptureRef.current = null;
|
|
|
|
|
await active.stop().catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
})();
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
} catch (err: unknown) {
|
|
|
|
|
console.warn(
|
|
|
|
|
'screen-share: native system-audio unavailable, sharing video only',
|
|
|
|
|
err instanceof Error ? err.message : err,
|
|
|
|
|
);
|
|
|
|
|
if (nativeAudioCaptureRef.current) {
|
|
|
|
|
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
|
|
|
|
|
nativeAudioCaptureRef.current = null;
|
|
|
|
|
}
|
|
|
|
|
audioHandle = null;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Canvas stream 'ended' fires on handle.stop() (we track.stop()
|
|
|
|
|
// each track) — chain unpublish + native teardown so one ended
|
|
|
|
|
// event cleans everything up regardless of who triggered it.
|
|
|
|
|
// Also tear down any paired audio capture so sound can't
|
|
|
|
|
// outlive the video share.
|
|
|
|
|
videoMst.addEventListener('ended', () => {
|
|
|
|
|
audioMst.addEventListener('ended', () => {
|
|
|
|
|
void (async () => {
|
|
|
|
|
try {
|
|
|
|
|
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
|
|
|
|
|
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
|
|
|
|
|
} catch {
|
|
|
|
|
/* already unpublished */
|
|
|
|
|
}
|
|
|
|
|
const active = nativeCaptureRef.current;
|
|
|
|
|
if (active && active.captureId === handle.captureId) {
|
|
|
|
|
nativeCaptureRef.current = null;
|
|
|
|
|
await active.stop().catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
const audioActive = nativeAudioCaptureRef.current;
|
|
|
|
|
if (audioActive) {
|
|
|
|
|
nativeAudioCaptureRef.current = null;
|
|
|
|
|
await audioActive.stop().catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
setIsScreenSharing(false);
|
|
|
|
|
})();
|
|
|
|
|
});
|
|
|
|
|
console.info('screen-share: native capture active', {
|
|
|
|
|
audio: audioHandle != null,
|
|
|
|
|
});
|
|
|
|
|
setIsScreenSharing(true);
|
|
|
|
|
return;
|
|
|
|
|
const stopAudio = async () => {
|
|
|
|
|
try {
|
|
|
|
|
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
|
|
|
|
|
} catch {
|
|
|
|
|
/* already unpublished */
|
|
|
|
|
}
|
|
|
|
|
const active = nativeAudioCaptureRef.current;
|
|
|
|
|
if (active && active.captureId === audioHandle.captureId) {
|
|
|
|
|
nativeAudioCaptureRef.current = null;
|
|
|
|
|
await active.stop().catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
return { stopAudio };
|
|
|
|
|
} catch (err: unknown) {
|
|
|
|
|
// Native path unavailable (non-Tauri runtime, source vanished,
|
|
|
|
|
// first-frame timeout). Clean up any partial handle and fall
|
|
|
|
|
// through to the getUserMedia / getDisplayMedia paths.
|
|
|
|
|
if (nativeCaptureRef.current) {
|
|
|
|
|
await nativeCaptureRef.current.stop().catch(() => undefined);
|
|
|
|
|
nativeCaptureRef.current = null;
|
|
|
|
|
}
|
|
|
|
|
console.warn(
|
|
|
|
|
'screen-share: native system-audio unavailable, sharing video only',
|
|
|
|
|
err instanceof Error ? err.message : err,
|
|
|
|
|
);
|
|
|
|
|
if (nativeAudioCaptureRef.current) {
|
|
|
|
|
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
|
|
|
|
|
nativeAudioCaptureRef.current = null;
|
|
|
|
|
}
|
|
|
|
|
console.warn(
|
|
|
|
|
'screen-share: native path failed, falling back',
|
|
|
|
|
err instanceof Error ? err.message : err,
|
|
|
|
|
);
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// Direct-publish path when our custom picker supplied a Chromium-
|
|
|
|
|
// format source id. Bypasses the OS picker so the user shares exactly
|
|
|
|
|
// the window/monitor they clicked in the grid. getUserMedia with the
|
|
|
|
|
// legacy chromeMediaSourceId constraint is not in the MediaStream
|
|
|
|
|
// spec but is honoured by Chromium / WebView2. If it throws we fall
|
|
|
|
|
// through to setScreenShareEnabled and let the OS picker run.
|
|
|
|
|
// ----- Path 1: chromeMediaSource (fast) -----
|
|
|
|
|
if (sourceId) {
|
|
|
|
|
try {
|
|
|
|
|
const { Track: LkTrack } = await import('livekit-client');
|
|
|
|
@@ -1357,36 +1301,28 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
|
|
|
|
maxFrameRate: fps,
|
|
|
|
|
},
|
|
|
|
|
} as unknown as MediaTrackConstraints;
|
|
|
|
|
const audioConstraints: MediaTrackConstraints | false = settings.includeSystemAudio
|
|
|
|
|
? ({
|
|
|
|
|
mandatory: {
|
|
|
|
|
chromeMediaSource: 'desktop',
|
|
|
|
|
chromeMediaSourceId: sourceId,
|
|
|
|
|
},
|
|
|
|
|
} as unknown as MediaTrackConstraints)
|
|
|
|
|
: false;
|
|
|
|
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
|
|
|
audio: audioConstraints,
|
|
|
|
|
audio: false,
|
|
|
|
|
video: videoConstraints,
|
|
|
|
|
});
|
|
|
|
|
const videoMst = stream.getVideoTracks()[0];
|
|
|
|
|
const audioMst = stream.getAudioTracks()[0];
|
|
|
|
|
if (!videoMst) {
|
|
|
|
|
stream.getTracks().forEach((t) => t.stop());
|
|
|
|
|
throw new Error('no video track from chromeMediaSource');
|
|
|
|
|
}
|
|
|
|
|
// Pass raw MediaStreamTracks — `publishTrack` wraps them in the
|
|
|
|
|
// right Local*Track internally and the publishDefaults on the
|
|
|
|
|
// Room handle VP9 codec + screenShareEncoding caps. Passing the
|
|
|
|
|
// raw tracks also sidesteps a type incompatibility between
|
|
|
|
|
// livekit-client's Local*Track and our exactOptionalPropertyTypes
|
|
|
|
|
// setting.
|
|
|
|
|
// H.264 lets Chromium's hardware encoder take over on Windows
|
|
|
|
|
// (Intel QuickSync / NVENC / AMD VCE) — VP9 is software-only in
|
|
|
|
|
// Chromium's WebRTC path and collapses under the L3T3_KEY
|
|
|
|
|
// scalability mode we set globally for camera. `contentHint` is
|
|
|
|
|
// what setScreenShareEnabled sets internally; pushing it by hand
|
|
|
|
|
// here tells the encoder to weight sharpness over smoothness,
|
|
|
|
|
// which is right for UI/text-heavy screen content.
|
|
|
|
|
videoMst.contentHint = 'detail';
|
|
|
|
|
const videoPub = await lp.publishTrack(videoMst, {
|
|
|
|
|
source: LkTrack.Source.ScreenShare,
|
|
|
|
|
videoCodec: 'vp9',
|
|
|
|
|
videoCodec: 'h264',
|
|
|
|
|
});
|
|
|
|
|
// Stop the publish when the OS revokes capture (user hit the
|
|
|
|
|
// OS "Stop sharing" banner, or closed the window we were sharing).
|
|
|
|
|
const audioRes = await startWasapiAudio(LkTrack);
|
|
|
|
|
videoMst.addEventListener('ended', () => {
|
|
|
|
|
void (async () => {
|
|
|
|
|
try {
|
|
|
|
@@ -1394,32 +1330,81 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
|
|
|
|
} catch {
|
|
|
|
|
/* already unpublished */
|
|
|
|
|
}
|
|
|
|
|
if (audioRes) await audioRes.stopAudio();
|
|
|
|
|
setIsScreenSharing(false);
|
|
|
|
|
})();
|
|
|
|
|
});
|
|
|
|
|
if (audioMst) {
|
|
|
|
|
const audioPub = await lp.publishTrack(audioMst, {
|
|
|
|
|
source: LkTrack.Source.ScreenShareAudio,
|
|
|
|
|
});
|
|
|
|
|
audioMst.addEventListener('ended', () => {
|
|
|
|
|
void (async () => {
|
|
|
|
|
try {
|
|
|
|
|
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
|
|
|
|
|
} catch {
|
|
|
|
|
/* ignore */
|
|
|
|
|
}
|
|
|
|
|
})();
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
console.info('screen-share: chromeMediaSource active', {
|
|
|
|
|
audio: audioRes != null,
|
|
|
|
|
});
|
|
|
|
|
setIsScreenSharing(true);
|
|
|
|
|
return;
|
|
|
|
|
} catch (err: unknown) {
|
|
|
|
|
// WebView2 / browser rejected the legacy constraint. Fall through
|
|
|
|
|
// to the normal OS picker path below so the user still gets a
|
|
|
|
|
// working share instead of a hard error.
|
|
|
|
|
console.warn(
|
|
|
|
|
'direct screen-share via chromeMediaSourceId failed; falling back to getDisplayMedia',
|
|
|
|
|
err,
|
|
|
|
|
'screen-share: chromeMediaSource failed, trying native xcap',
|
|
|
|
|
err instanceof Error ? err.message : err,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ----- Path 2: xcap native (CPU fallback) -----
|
|
|
|
|
if (sourceId) {
|
|
|
|
|
try {
|
|
|
|
|
const { Track: LkTrack } = await import('livekit-client');
|
|
|
|
|
const maxWidth = ssParams.dims?.width ?? 1920;
|
|
|
|
|
const maxHeight = ssParams.dims?.height ?? 1080;
|
|
|
|
|
const handle = await startNativeCapture({
|
|
|
|
|
sourceId,
|
|
|
|
|
maxWidth,
|
|
|
|
|
maxHeight,
|
|
|
|
|
fps,
|
|
|
|
|
});
|
|
|
|
|
nativeCaptureRef.current = handle;
|
|
|
|
|
const videoMst = handle.stream.getVideoTracks()[0];
|
|
|
|
|
if (!videoMst) {
|
|
|
|
|
await handle.stop();
|
|
|
|
|
nativeCaptureRef.current = null;
|
|
|
|
|
throw new NativeCaptureUnavailable('canvas stream produced no video track');
|
|
|
|
|
}
|
|
|
|
|
videoMst.contentHint = 'detail';
|
|
|
|
|
const videoPub = await lp.publishTrack(videoMst, {
|
|
|
|
|
source: LkTrack.Source.ScreenShare,
|
|
|
|
|
videoCodec: 'h264',
|
|
|
|
|
});
|
|
|
|
|
const audioRes = await startWasapiAudio(LkTrack);
|
|
|
|
|
videoMst.addEventListener('ended', () => {
|
|
|
|
|
void (async () => {
|
|
|
|
|
try {
|
|
|
|
|
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
|
|
|
|
|
} catch {
|
|
|
|
|
/* already unpublished */
|
|
|
|
|
}
|
|
|
|
|
const active = nativeCaptureRef.current;
|
|
|
|
|
if (active && active.captureId === handle.captureId) {
|
|
|
|
|
nativeCaptureRef.current = null;
|
|
|
|
|
await active.stop().catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
if (audioRes) await audioRes.stopAudio();
|
|
|
|
|
setIsScreenSharing(false);
|
|
|
|
|
})();
|
|
|
|
|
});
|
|
|
|
|
console.info('screen-share: native xcap active', {
|
|
|
|
|
audio: audioRes != null,
|
|
|
|
|
});
|
|
|
|
|
setIsScreenSharing(true);
|
|
|
|
|
return;
|
|
|
|
|
} catch (err: unknown) {
|
|
|
|
|
if (nativeCaptureRef.current) {
|
|
|
|
|
await nativeCaptureRef.current.stop().catch(() => undefined);
|
|
|
|
|
nativeCaptureRef.current = null;
|
|
|
|
|
}
|
|
|
|
|
if (nativeAudioCaptureRef.current) {
|
|
|
|
|
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
|
|
|
|
|
nativeAudioCaptureRef.current = null;
|
|
|
|
|
}
|
|
|
|
|
console.warn(
|
|
|
|
|
'screen-share: native xcap failed, falling back to OS picker',
|
|
|
|
|
err instanceof Error ? err.message : err,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|