byGalax/ChatApp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(call): screen-share hybrid — OS picker + WASAPI audio

Browse Source 
Strip the sourceId-gated chromeMediaSource + xcap native capture paths
from startScreenShare and collapse to a single setScreenShareEnabled
call. Neither of the bypassed paths produced smooth frames in WebView2:
chromeMediaSource: 'desktop' is an extension-only Chromium constraint
and throws outside extension origins, and the xcap JPEG-over-IPC
fallback couldn't sustain 30fps at 1080p on a single main-thread.
setScreenShareEnabled goes through Chromium's native getDisplayMedia
capture, which is the only path that gets HW-accelerated frames into
the WebRTC encoder from WebView2.

Audio continues via the WASAPI loopback module — getDisplayMedia can't
grab system sound in WebView2 without desktop-capture entitlements
Chromium reserves for extensions. The audio track's teardown chains to
the ScreenShare video track's 'ended' event so the Windows stop-share
overlay kills both sides in lockstep.

ScreenSourcePicker is now a quality + audio chooser only; the
thumbnail grid disappears because custom source IDs don't round-trip
through WebView2, and a custom picker in front of the OS picker just
means the user picks twice.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
byGalax
2026-04-23 17:59:41 +02:00
parent adbdfaa2aa
commit c9b885b91c
3 changed files with 149 additions and 589 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/desktop/src-tauri/Cargo.lock
Generated
+1 -1
View File
@@ -834,7 +834,7 @@ dependencies = [
[[package]] [[package]]
name = "chat-app-desktop" name = "chat-app-desktop"
version = "0.11.1" version = "0.11.2"
dependencies = [ dependencies = [
"base64 0.22.1", "base64 0.22.1",
"dryoc", "dryoc",
apps/desktop/src/components/ScreenSourcePicker.tsx
+70 -323
View File
@@ -1,4 +1,4 @@
import { memo, startTransition, useEffect, useRef, useState } from 'react'; import { useState } from 'react';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { import {
@@ -9,21 +9,14 @@ import {
type ScreenSharePreset, type ScreenSharePreset,
updateScreenShareSettings, updateScreenShareSettings,
} from '../lib/screenShareSettings'; } from '../lib/screenShareSettings';
import {
captureScreenSourceThumbnailBytes,
listScreenSources,
type ScreenSource,
} from '../lib/screenSources';
import { MonitorShareIcon, SpinnerIcon, XIcon } from './icons'; import { MonitorShareIcon, SpinnerIcon, XIcon } from './icons';
interface Props { interface Props {
open: boolean; open: boolean;
onClose: () => void; onClose: () => void;
/** Parent handles the actual share start. `sourceId` is null when the user /** Parent starts the actual share. The OS picker runs afterwards; the
* clicks "Teilen" without picking a specific source — fallback to the * dialog only gathers quality + audio settings. */
* OS-level getDisplayMedia picker. */
onStart: (opts: { onStart: (opts: {
sourceId: string | null;
preset: ScreenSharePreset; preset: ScreenSharePreset;
displaySurface: DisplaySurfaceHint; displaySurface: DisplaySurfaceHint;
framerate: number | null; framerate: number | null;
@@ -31,143 +24,32 @@ interface Props {
}) => Promise<void>; }) => Promise<void>;
} }
// Discord-style picker. Replaces the old form-field dialog with a thumbnail // Quality + audio chooser shown before "Bildschirm teilen" opens the OS
// grid sourced from the Rust `enumerate_screen_sources` command. Clicking a // source picker. We can't substitute sources in WebView2 — the
// thumbnail stashes its Chromium-format id; the parent then attempts a // `chromeMediaSource: 'desktop'` constraint is extension-only and
// `chromeMediaSourceId`-constrained getUserMedia call. If WebView2 ignores // ScreenCaptureStarting offers allow/deny, not source-injection — so a
// the constraint (it may), the fallback OS picker still runs — but at least // custom thumbnail grid would just double-pick (user picks here, then
// the user already saw + chose from a real preview first. // picks again in the OS dialog). Dropping the grid keeps the smooth
// Chromium-native capture pipeline and narrows the UX to the one
// decision that still matters at share-time: quality + audio.
export function ScreenSourcePicker({ open, onClose, onStart }: Props) { export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
const { t } = useTranslation(['app']); const { t } = useTranslation(['app']);
const initial = getScreenShareSettings(); const initial = getScreenShareSettings();
const [sources, setSources] = useState<ScreenSource[] | null>(null);
// Thumbnails are kept in a separate state from the source list so an
// arriving thumbnail never creates a new `ScreenSource` object for
// unrelated cards — memo compares `thumbnailUrl` by string identity,
// so only the one card whose URL changes rerenders.
const [thumbnailUrls, setThumbnailUrls] = useState<Record<string, string>>({});
// All blob URLs we've handed out this session. Revoked on picker close
// so the native buffers they point at don't leak across opens.
const blobUrlsRef = useRef<string[]>([]);
const [selectedId, setSelectedId] = useState<string | null>(null);
const [preset, setPreset] = useState<ScreenSharePreset>(initial.preset); const [preset, setPreset] = useState<ScreenSharePreset>(initial.preset);
const [includeAudio, setIncludeAudio] = useState<boolean>(initial.includeSystemAudio); const [includeAudio, setIncludeAudio] = useState<boolean>(initial.includeSystemAudio);
const [busy, setBusy] = useState(false); const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
// Two-phase load: (1) fast list returns names + placeholders so the grid
// paints instantly, (2) capture thumbnails in a bounded worker-pool
// using the binary-IPC variant. Arriving bytes are wrapped in a Blob
// and exposed via URL.createObjectURL — no base64 on either side,
// which is the single biggest main-thread win compared to the old
// JSON-of-base64 flow. Combined with rAF-batched state updates, the
// picker stays responsive even on 20+ source enumerations.
useEffect(() => {
if (!open) {
// Revoke blob URLs created during the last session so native
// buffers don't linger after close.
for (const url of blobUrlsRef.current) URL.revokeObjectURL(url);
blobUrlsRef.current = [];
setSources(null);
setThumbnailUrls({});
setSelectedId(null);
setError(null);
return;
}
let cancelled = false;
// Coalesce thumbnail arrivals within a single animation frame into
// one setState — cuts re-renders from O(N) to O(frames) during the
// initial fan-in and prevents consecutive 10-30 ms long tasks from
// stacking in one frame.
//
// Previously `batch` was aliased to `pendingUrls` and then we cleared
// pendingUrls via `delete` — which emptied batch too (same reference)
// and every flush ended up spreading nothing into the state. Clone
// first, then clear, so the batch keeps its entries.
let pendingUrls: Record<string, string> = {};
let rafScheduled = false;
const flush = () => {
rafScheduled = false;
const batch = pendingUrls;
if (Object.keys(batch).length === 0) return;
pendingUrls = {};
startTransition(() => {
setThumbnailUrls((prev) => ({ ...prev, ...batch }));
});
};
const queueUrl = (id: string, url: string) => {
pendingUrls[id] = url;
if (!rafScheduled) {
rafScheduled = true;
requestAnimationFrame(flush);
}
};
// Concurrency 2: Windows GDI BitBlt / PrintWindow contends for the
// desktop compositor, so 4+ parallel captures stutter the whole Tauri
// window. 2 in parallel keeps the compositor breathing.
const CONCURRENCY = 2;
void (async () => {
const list = await listScreenSources();
if (cancelled) return;
setSources(list);
const queue = [...list];
const pickOne = (src: typeof list[number]) => {
void (async () => {
const blob = await captureScreenSourceThumbnailBytes(src.id);
if (cancelled) {
// Edge case: picker closed while this request was in flight.
// blob may still exist; nothing holds a URL to it, so it GCs.
return;
}
if (blob) {
const url = URL.createObjectURL(blob);
blobUrlsRef.current.push(url);
queueUrl(src.id, url);
}
const nextSrc = queue.shift();
if (nextSrc) pickOne(nextSrc);
})();
};
for (let i = 0; i < Math.min(CONCURRENCY, queue.length); i++) {
const s = queue.shift();
if (s) pickOne(s);
}
})();
return () => {
cancelled = true;
};
}, [open]);
if (!open) return null; if (!open) return null;
const screens = sources?.filter((s) => s.kind === 'screen') ?? [];
const windows = sources?.filter((s) => s.kind === 'window') ?? [];
const hasAny = (sources?.length ?? 0) > 0;
async function handleStart(): Promise<void> { async function handleStart(): Promise<void> {
setBusy(true); setBusy(true);
setError(null); setError(null);
try { try {
// Persist user's audio + preset choice so subsequent shares start with
// the same prefs when they skip the picker. The picker itself stays
// as the entry for future starts (right-click on share button also
// opens it — see InCallPanel wiring).
updateScreenShareSettings({ preset, includeSystemAudio: includeAudio }); updateScreenShareSettings({ preset, includeSystemAudio: includeAudio });
// Infer a displaySurface hint from the selection so the fallback OS
// picker jumps to the right tab when our direct-publish path is
// rejected by WebView2.
const selected = sources?.find((s) => s.id === selectedId) ?? null;
const hint: DisplaySurfaceHint =
selected?.kind === 'screen'
? 'monitor'
: selected?.kind === 'window'
? 'window'
: null;
await onStart({ await onStart({
sourceId: selected?.id ?? null,
preset, preset,
displaySurface: hint, displaySurface: null,
framerate: null, framerate: null,
includeAudio, includeAudio,
}); });
@@ -189,7 +71,7 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
> >
<div <div
onClick={(e) => e.stopPropagation()} onClick={(e) => e.stopPropagation()}
className="flex max-h-[88vh] w-full max-w-[860px] flex-col overflow-hidden rounded-2xl border border-line bg-surface-3 shadow-xl" className="flex w-full max-w-[420px] flex-col overflow-hidden rounded-2xl border border-line bg-surface-3 shadow-xl"
> >
<header className="flex items-center justify-between border-b border-line px-5 py-3"> <header className="flex items-center justify-between border-b border-line px-5 py-3">
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
@@ -208,206 +90,71 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
</button> </button>
</header> </header>
<div className="flex-1 overflow-y-auto"> <div className="flex flex-col gap-4 px-5 py-5">
{sources === null ? ( <p className="text-xs text-fg-muted">
<div className="flex h-40 items-center justify-center gap-2 text-sm text-fg-muted"> {t('app:call.share_os_picker_hint', {
<SpinnerIcon className="h-4 w-4" /> defaultValue:
<span> 'Nach dem Klick auf „Teilen" wählst du im System-Dialog den Bildschirm oder das Fenster aus.',
{t('app:call.share_loading', { defaultValue: 'Lade Quellen…' })} })}
</span> </p>
</div>
) : !hasAny ? ( <label className="flex items-center justify-between gap-3 text-xs text-fg">
<div className="flex flex-col items-center gap-2 px-6 py-10 text-center text-sm text-fg-muted"> <span className="font-semibold uppercase tracking-wider text-fg-muted">
<MonitorShareIcon className="h-6 w-6 opacity-60" /> {t('app:call.share_quality', { defaultValue: 'Qualität' })}
<span> </span>
{t('app:call.share_no_sources', { <select
defaultValue: value={preset}
'Keine Quellen-Previews verfügbar. Klick auf „Teilen" öffnet den System-Picker.', onChange={(e) => setPreset(e.target.value as ScreenSharePreset)}
})} className="flex-1 rounded-md border border-line bg-surface-3 px-2 py-1 text-xs text-fg focus:border-accent focus:outline-none"
</span> >
</div> {PRESET_ORDER.map((p) => (
) : ( <option key={p} value={p}>
<div className="space-y-5 p-5"> {getPresetParams(p).label} · ≤ {getPresetParams(p).bitrateKbps} kbps
{screens.length > 0 && ( </option>
<SourceSection ))}
title={t('app:call.share_screens', { defaultValue: 'Bildschirme' })} </select>
sources={screens} </label>
thumbnailUrls={thumbnailUrls}
selectedId={selectedId} <label className="flex cursor-pointer items-center gap-2 text-xs text-fg">
onSelect={setSelectedId} <input
/> type="checkbox"
)} checked={includeAudio}
{windows.length > 0 && ( onChange={(e) => setIncludeAudio(e.target.checked)}
<SourceSection className="accent-accent"
title={t('app:call.share_windows', { defaultValue: 'Fenster' })} />
sources={windows} <span>
thumbnailUrls={thumbnailUrls} {t('app:call.share_system_audio', {
selectedId={selectedId} defaultValue: 'System-Sound mit übertragen',
onSelect={setSelectedId} })}
/> </span>
)} </label>
</div>
)}
</div>
<footer className="flex flex-col gap-3 border-t border-line bg-surface-2 px-5 py-3">
<div className="flex flex-wrap items-center justify-between gap-3">
<label className="flex cursor-pointer items-center gap-2 text-xs text-fg">
<span className="font-semibold uppercase tracking-wider text-fg-muted">
{t('app:call.share_quality', { defaultValue: 'Qualität' })}
</span>
<select
value={preset}
onChange={(e) => setPreset(e.target.value as ScreenSharePreset)}
className="rounded-md border border-line bg-surface-3 px-2 py-1 text-xs text-fg focus:border-accent focus:outline-none"
>
{PRESET_ORDER.map((p) => (
<option key={p} value={p}>
{getPresetParams(p).label} · ≤ {getPresetParams(p).bitrateKbps} kbps
</option>
))}
</select>
</label>
<label className="flex cursor-pointer items-center gap-2 text-xs text-fg">
<input
type="checkbox"
checked={includeAudio}
onChange={(e) => setIncludeAudio(e.target.checked)}
className="accent-accent"
/>
<span>
{t('app:call.share_system_audio', {
defaultValue: 'System-Sound mit übertragen',
})}
</span>
</label>
</div>
{error && ( {error && (
<p role="alert" className="text-xs text-rose-600 dark:text-rose-300"> <p role="alert" className="text-xs text-rose-600 dark:text-rose-300">
{error} {error}
</p> </p>
)} )}
<div className="flex items-center justify-end gap-2"> </div>
<button
type="button" <footer className="flex items-center justify-end gap-2 border-t border-line bg-surface-2 px-5 py-3">
onClick={onClose} <button
className="cursor-pointer rounded-md border border-line bg-surface-3 px-3 py-2 text-sm text-fg transition hover:bg-surface-2" type="button"
> onClick={onClose}
{t('app:friends.action_cancel', { defaultValue: 'Abbrechen' })} className="cursor-pointer rounded-md border border-line bg-surface-3 px-3 py-2 text-sm text-fg transition hover:bg-surface-2"
</button> >
<button {t('app:friends.action_cancel', { defaultValue: 'Abbrechen' })}
type="button" </button>
onClick={() => void handleStart()} <button
disabled={busy} type="button"
className="inline-flex cursor-pointer items-center gap-1.5 rounded-md bg-accent px-3 py-2 text-sm font-semibold text-accent-fg transition hover:brightness-110 disabled:cursor-not-allowed disabled:opacity-60" onClick={() => void handleStart()}
> disabled={busy}
{busy && <SpinnerIcon className="h-4 w-4" />} className="inline-flex cursor-pointer items-center gap-1.5 rounded-md bg-accent px-3 py-2 text-sm font-semibold text-accent-fg transition hover:brightness-110 disabled:cursor-not-allowed disabled:opacity-60"
<span> >
{selectedId {busy && <SpinnerIcon className="h-4 w-4" />}
? t('app:call.share_start', { defaultValue: 'Teilen' }) <span>{t('app:call.share_start', { defaultValue: 'Teilen' })}</span>
: t('app:call.share_pick_system', { </button>
defaultValue: 'Ohne Auswahl weiter',
})}
</span>
</button>
</div>
</footer> </footer>
</div> </div>
</div> </div>
); );
} }
function SourceSection({
title,
sources,
thumbnailUrls,
selectedId,
onSelect,
}: {
title: string;
sources: ScreenSource[];
thumbnailUrls: Record<string, string>;
selectedId: string | null;
onSelect: (id: string) => void;
}) {
return (
<section>
<h4 className="mb-2 text-xs font-semibold uppercase tracking-wider text-fg-muted">
{title}
</h4>
<div className="grid grid-cols-2 gap-2.5 sm:grid-cols-3">
{sources.map((src) => (
<SourceCard
key={src.id}
source={src}
thumbnailUrl={thumbnailUrls[src.id] ?? null}
selected={selectedId === src.id}
onSelect={onSelect}
/>
))}
</div>
</section>
);
}
// Memoized so a thumbnail arriving for card B doesn't re-render card A.
// Keeps re-render work proportional to the number of updates instead of
// "whole grid on every update" — which was the main reason scrolling felt
// frozen during the initial thumbnail fan-in.
//
// The parent passes `onSelect(id)` rather than an inline `onClick`-arrow
// so the callback reference stays stable across renders; otherwise
// React.memo would always see a fresh function prop and re-render every
// card on every parent update.
const SourceCard = memo(function SourceCard({
source,
thumbnailUrl,
selected,
onSelect,
}: {
source: ScreenSource;
thumbnailUrl: string | null;
selected: boolean;
onSelect: (id: string) => void;
}) {
return (
<button
type="button"
onClick={() => onSelect(source.id)}
aria-pressed={selected}
title={source.name}
className={
'group flex cursor-pointer flex-col overflow-hidden rounded-lg border transition focus:outline-none focus-visible:ring-2 focus-visible:ring-accent/50 ' +
(selected
? 'border-accent ring-2 ring-accent/30'
: 'border-line hover:border-accent/70')
}
>
<div className="relative aspect-video w-full overflow-hidden bg-black">
{thumbnailUrl ? (
// decoding="async" keeps image decode off the main-thread paint
// step; loading="lazy" means cards outside the viewport don't
// ask the browser to decode until the user scrolls to them. The
// URL is a blob: URL backed by the ArrayBuffer Rust sent over
// IPC — no base64 decode, no data-URL parse, just direct bytes
// into the decoder.
<img
src={thumbnailUrl}
alt=""
decoding="async"
loading="lazy"
className="h-full w-full object-contain transition group-hover:brightness-110"
draggable={false}
/>
) : (
<div className="flex h-full w-full items-center justify-center bg-gradient-to-br from-surface-2 to-surface-3 text-fg-muted">
<MonitorShareIcon className="h-6 w-6 opacity-50" />
</div>
)}
</div>
<div className="truncate px-2.5 py-1.5 text-left text-xs font-medium text-fg">
{source.name}
</div>
</button>
);
});
apps/desktop/src/context/CallContext.tsx
+78 -265
View File
@@ -75,11 +75,6 @@ import {
setAllPipelinesSinkId, setAllPipelinesSinkId,
setPipelineGain, setPipelineGain,
} from '../lib/remoteAudioPipelines'; } from '../lib/remoteAudioPipelines';
import {
type NativeCaptureHandle,
NativeCaptureUnavailable,
startNativeCapture,
} from '../lib/screenCapture';
import { import {
type SystemAudioHandle, type SystemAudioHandle,
startSystemAudioCapture, startSystemAudioCapture,
@@ -304,15 +299,10 @@ export function CallProvider({ children }: { children: ReactNode }) {
// to `connected` after a few seconds so the UI doesn't hang in "Verbinde…" // to `connected` after a few seconds so the UI doesn't hang in "Verbinde…"
// indefinitely. The solo-timeout will then cleanly close if nobody arrives. // indefinitely. The solo-timeout will then cleanly close if nobody arrives.
const joinFallbackTimerRef = useRef<number | null>(null); const joinFallbackTimerRef = useRef<number | null>(null);
// Active native screen-capture handle (Rust side). Set when // Handle for the Windows-only WASAPI system-audio capture. Lives in
// startScreenShare takes the xcap path; cleared on stopScreenShare or // lockstep with the ScreenShare video track published by
// on the canvas track's 'ended' event. Not kept in React state because // setScreenShareEnabled; teardown is chained to the video track's
// it never feeds into a render. // `ended` event so stale audio can never outlive the video share.
const nativeCaptureRef = useRef<NativeCaptureHandle | null>(null);
// Matching handle for the Windows-only WASAPI system-audio capture.
// Lives in lockstep with the video handle above when the user picks
// "Mit System-Sound"; teardown is wired so that stopping either track
// also stops the other, so stale audio can't outlive the video share.
const nativeAudioCaptureRef = useRef<SystemAudioHandle | null>(null); const nativeAudioCaptureRef = useRef<SystemAudioHandle | null>(null);
const roomRef = useRef<Room | null>(null); const roomRef = useRef<Room | null>(null);
// Web Audio graph that mixes live mic + soundboard sources into a single // Web Audio graph that mixes live mic + soundboard sources into a single
@@ -488,13 +478,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
/* ignore */ /* ignore */
} }
} }
// Stop any lingering native screen-capture thread so we don't leak
// Rust threads when the call ends mid-share.
if (nativeCaptureRef.current) {
const h = nativeCaptureRef.current;
nativeCaptureRef.current = null;
await h.stop().catch(() => undefined);
}
if (nativeAudioCaptureRef.current) { if (nativeAudioCaptureRef.current) {
const h = nativeAudioCaptureRef.current; const h = nativeAudioCaptureRef.current;
nativeAudioCaptureRef.current = null; nativeAudioCaptureRef.current = null;
@@ -729,6 +712,11 @@ export function CallProvider({ children }: { children: ReactNode }) {
r.on(RoomEvent.ParticipantConnected, () => { r.on(RoomEvent.ParticipantConnected, () => {
setRemoteParticipants(Array.from(r.remoteParticipants.values())); setRemoteParticipants(Array.from(r.remoteParticipants.values()));
if (presenceRef.current !== 'dnd') void playJoinBeep(); if (presenceRef.current !== 'dnd') void playJoinBeep();
// Rejoin während wir schon `connected` sind: markConnectedIfReady
// returnt früh und würde den Solo-Timer nicht cancellen. Hier
// unbedingt clearen, sonst kickt der 5-Minuten-Timer obwohl der
// andere Peer längst wieder im Raum ist.
clearSoloTimer();
markConnectedIfReady(r, conversationId, mediaKind, callId); markConnectedIfReady(r, conversationId, mediaKind, callId);
}); });
@@ -1173,11 +1161,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
preset: ScreenSharePreset; preset: ScreenSharePreset;
displaySurface: DisplaySurfaceHint; displaySurface: DisplaySurfaceHint;
framerate: number | null; framerate: number | null;
/** Chromium-format source id from our custom picker. When set, we
* try to capture that exact source via `chromeMediaSourceId`
* instead of the OS-level getDisplayMedia picker. Falls back to
* getDisplayMedia if WebView2 rejects the constraint. */
sourceId: string | null;
}>, }>,
) => { ) => {
const r = roomRef.current; const r = roomRef.current;
@@ -1185,8 +1168,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
const lp = r.localParticipant; const lp = r.localParticipant;
if (lp.isScreenShareEnabled) return; if (lp.isScreenShareEnabled) return;
// Persist the user's choice so subsequent shares use the same config
// without re-opening the picker unless they want to change something.
const settings = getScreenShareSettings(); const settings = getScreenShareSettings();
const preset = overrides?.preset ?? settings.preset; const preset = overrides?.preset ?? settings.preset;
const displaySurface = const displaySurface =
@@ -1201,220 +1182,17 @@ export function CallProvider({ children }: { children: ReactNode }) {
const ssParams = getPresetParams(preset); const ssParams = getPresetParams(preset);
const fps = framerateOverride ?? ssParams.framerate; const fps = framerateOverride ?? ssParams.framerate;
const sourceId = overrides?.sourceId ?? null;
// Ordered capture paths when our custom picker supplied a sourceId:
// 1. chromeMediaSource video — hardware-accelerated path that
// hands the frames directly to WebRTC, same mechanism the OS
// picker uses internally. Fast, smooth, what users expect.
// 2. xcap native (JPEG → canvas → captureStream) — fallback for
// WebView2 versions that reject the legacy chromeMediaSource
// constraint. Functional but CPU-heavy; the per-frame JPEG
// encode + base64 + createImageBitmap chain shows up as
// visible stutter on 1080p30.
// 3. OS picker (setScreenShareEnabled) — last resort when both
// native paths throw, and the only path when no sourceId was
// supplied.
//
// Audio (Windows only) always goes through the WASAPI loopback
// module — getUserMedia's chromeMediaSource audio constraint
// throws AbortError on Window captures, and pairing both into one
// call ended up blocking the video path entirely. Splitting them
// means the video path stays fast and the audio path stays
// reliable for any source type.
if (!sourceId) {
console.info(
'screen-share: no sourceId supplied by picker, OS picker will open',
);
}
// Publish system audio via WASAPI. Resolves to the audio publication
// + an ended-cleanup registration so whichever video path succeeds
// can wire the audio teardown onto its own video-ended handler.
const startWasapiAudio = async (
LkTrack: typeof import('livekit-client').Track,
): Promise<{
stopAudio: () => Promise<void>;
} | null> => {
if (!settings.includeSystemAudio) return null;
try {
const audioHandle = await startSystemAudioCapture();
nativeAudioCaptureRef.current = audioHandle;
const audioMst = audioHandle.stream.getAudioTracks()[0];
if (!audioMst) {
await audioHandle.stop();
nativeAudioCaptureRef.current = null;
return null;
}
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
});
audioMst.addEventListener('ended', () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
})();
});
const stopAudio = async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
const active = nativeAudioCaptureRef.current;
if (active && active.captureId === audioHandle.captureId) {
nativeAudioCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
};
return { stopAudio };
} catch (err: unknown) {
console.warn(
'screen-share: native system-audio unavailable, sharing video only',
err instanceof Error ? err.message : err,
);
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
return null;
}
};
// ----- Path 1: chromeMediaSource (fast) -----
if (sourceId) {
try {
const { Track: LkTrack } = await import('livekit-client');
const maxWidth = ssParams.dims?.width ?? 3840;
const maxHeight = ssParams.dims?.height ?? 2160;
// Cast chains: browsers expose the legacy constraint via
// `MediaTrackConstraints.mandatory` which isn't in lib.dom.
const videoConstraints = {
mandatory: {
chromeMediaSource: 'desktop',
chromeMediaSourceId: sourceId,
maxWidth,
maxHeight,
maxFrameRate: fps,
},
} as unknown as MediaTrackConstraints;
const stream = await navigator.mediaDevices.getUserMedia({
audio: false,
video: videoConstraints,
});
const videoMst = stream.getVideoTracks()[0];
if (!videoMst) {
stream.getTracks().forEach((t) => t.stop());
throw new Error('no video track from chromeMediaSource');
}
// H.264 lets Chromium's hardware encoder take over on Windows
// (Intel QuickSync / NVENC / AMD VCE) — VP9 is software-only in
// Chromium's WebRTC path and collapses under the L3T3_KEY
// scalability mode we set globally for camera. `contentHint` is
// what setScreenShareEnabled sets internally; pushing it by hand
// here tells the encoder to weight sharpness over smoothness,
// which is right for UI/text-heavy screen content.
videoMst.contentHint = 'detail';
const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare,
videoCodec: 'h264',
});
const audioRes = await startWasapiAudio(LkTrack);
videoMst.addEventListener('ended', () => {
void (async () => {
try {
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
} catch {
/* already unpublished */
}
if (audioRes) await audioRes.stopAudio();
setIsScreenSharing(false);
})();
});
console.info('screen-share: chromeMediaSource active', {
audio: audioRes != null,
});
setIsScreenSharing(true);
return;
} catch (err: unknown) {
console.warn(
'screen-share: chromeMediaSource failed, trying native xcap',
err instanceof Error ? err.message : err,
);
}
}
// ----- Path 2: xcap native (CPU fallback) -----
if (sourceId) {
try {
const { Track: LkTrack } = await import('livekit-client');
const maxWidth = ssParams.dims?.width ?? 1920;
const maxHeight = ssParams.dims?.height ?? 1080;
const handle = await startNativeCapture({
sourceId,
maxWidth,
maxHeight,
fps,
});
nativeCaptureRef.current = handle;
const videoMst = handle.stream.getVideoTracks()[0];
if (!videoMst) {
await handle.stop();
nativeCaptureRef.current = null;
throw new NativeCaptureUnavailable('canvas stream produced no video track');
}
videoMst.contentHint = 'detail';
const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare,
videoCodec: 'h264',
});
const audioRes = await startWasapiAudio(LkTrack);
videoMst.addEventListener('ended', () => {
void (async () => {
try {
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
} catch {
/* already unpublished */
}
const active = nativeCaptureRef.current;
if (active && active.captureId === handle.captureId) {
nativeCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
if (audioRes) await audioRes.stopAudio();
setIsScreenSharing(false);
})();
});
console.info('screen-share: native xcap active', {
audio: audioRes != null,
});
setIsScreenSharing(true);
return;
} catch (err: unknown) {
if (nativeCaptureRef.current) {
await nativeCaptureRef.current.stop().catch(() => undefined);
nativeCaptureRef.current = null;
}
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
console.warn(
'screen-share: native xcap failed, falling back to OS picker',
err instanceof Error ? err.message : err,
);
}
}
// Single capture path: setScreenShareEnabled → OS picker → Chromium's
// hardware-accelerated capture feeds WebRTC directly. In WebView2
// the extension-only `chromeMediaSource: 'desktop'` constraint and
// our xcap JPEG-over-IPC pipeline were both too slow for smooth
// 30fps, so the OS picker is the only route to the fast pipeline.
// Audio is handled separately below via WASAPI because
// getDisplayMedia can't grab system audio in WebView2.
try { try {
await lp.setScreenShareEnabled(true, { await lp.setScreenShareEnabled(true, {
// "Go live" mode — capture system audio alongside the screen when audio: false,
// the user opted in. On hosts that can't fulfil the request the
// browser quietly drops it; peers just get video-only, no error.
audio: settings.includeSystemAudio,
...(ssParams.dims ...(ssParams.dims
? { ? {
resolution: { resolution: {
@@ -1430,9 +1208,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
frameRate: fps, frameRate: fps,
}, },
}), }),
// Hints the OS picker to pre-filter by source kind. `null` = no
// filter (show both). Cast because TS lib.dom doesn't know the
// field yet on all branches.
...(displaySurface ...(displaySurface
? ({ displaySurface } as { displaySurface: DisplaySurfaceHint }) ? ({ displaySurface } as { displaySurface: DisplaySurfaceHint })
: {}), : {}),
@@ -1441,24 +1216,67 @@ export function CallProvider({ children }: { children: ReactNode }) {
setIsScreenSharing(true); setIsScreenSharing(true);
} catch (err: unknown) { } catch (err: unknown) {
console.error('setScreenShareEnabled failed', err); console.error('setScreenShareEnabled failed', err);
return;
}
if (!settings.includeSystemAudio) return;
// WASAPI system-audio: publish as a separate ScreenShareAudio track.
// Chain its teardown onto the ScreenShare video track's `ended`
// event so the Windows "Stop sharing" overlay kills audio alongside
// video; the audio track's own `ended` handler covers the case
// where WASAPI dies by itself.
try {
const { Track: LkTrack } = await import('livekit-client');
const audioHandle = await startSystemAudioCapture();
nativeAudioCaptureRef.current = audioHandle;
const audioMst = audioHandle.stream.getAudioTracks()[0];
if (!audioMst) {
await audioHandle.stop();
nativeAudioCaptureRef.current = null;
return;
}
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
});
let videoMst: MediaStreamTrack | null = null;
for (const pub of lp.videoTrackPublications.values()) {
if (pub.source === LkTrack.Source.ScreenShare && pub.track) {
videoMst = pub.track.mediaStreamTrack;
break;
}
}
const teardown = () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
const active = nativeAudioCaptureRef.current;
if (active && active.captureId === audioHandle.captureId) {
nativeAudioCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
})();
};
audioMst.addEventListener('ended', teardown);
if (videoMst) videoMst.addEventListener('ended', teardown);
} catch (err: unknown) {
console.warn(
'screen-share: native system-audio unavailable, sharing video only',
err instanceof Error ? err.message : err,
);
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
} }
}, },
[], [],
); );
const stopScreenShare = useCallback(async () => { const stopScreenShare = useCallback(async () => {
// Native path first — stopping the handle kills the canvas track,
// which fires 'ended' on the MediaStreamTrack, which the start-handler
// already listens to for unpublishing and flipping isScreenSharing.
if (nativeCaptureRef.current) {
const h = nativeCaptureRef.current;
nativeCaptureRef.current = null;
try {
await h.stop();
} catch (err: unknown) {
console.warn('native capture stop failed', err);
}
}
if (nativeAudioCaptureRef.current) { if (nativeAudioCaptureRef.current) {
const h = nativeAudioCaptureRef.current; const h = nativeAudioCaptureRef.current;
nativeAudioCaptureRef.current = null; nativeAudioCaptureRef.current = null;
@@ -1471,18 +1289,13 @@ export function CallProvider({ children }: { children: ReactNode }) {
const r = roomRef.current; const r = roomRef.current;
if (!r) return; if (!r) return;
const lp = r.localParticipant; const lp = r.localParticipant;
// Unpublish any manually-published ScreenShare/ScreenShareAudio tracks // Unpublish any ScreenShareAudio track we manually published alongside
// (from the chromeMediaSourceId fallback path). setScreenShareEnabled // setScreenShareEnabled. LiveKit's setScreenShareEnabled(false) only
// only manages LK's own internally-captured tracks. // drops the video track it captured itself.
const toUnpublish: import('livekit-client').LocalTrackPublication[] = [];
for (const pub of lp.videoTrackPublications.values()) {
if (pub.source === Track.Source.ScreenShare && pub.track) toUnpublish.push(pub);
}
for (const pub of lp.audioTrackPublications.values()) { for (const pub of lp.audioTrackPublications.values()) {
if (pub.source === Track.Source.ScreenShareAudio && pub.track) toUnpublish.push(pub); if (pub.source === Track.Source.ScreenShareAudio && pub.track) {
} await lp.unpublishTrack(pub.track).catch(() => undefined);
for (const pub of toUnpublish) { }
if (pub.track) await lp.unpublishTrack(pub.track).catch(() => undefined);
} }
if (lp.isScreenShareEnabled) { if (lp.isScreenShareEnabled) {
try { try {