perf(call): screen-share hybrid — OS picker + WASAPI audio
Strip the sourceId-gated chromeMediaSource + xcap native capture paths from startScreenShare and collapse to a single setScreenShareEnabled call. Neither of the bypassed paths produced smooth frames in WebView2: chromeMediaSource: 'desktop' is an extension-only Chromium constraint and throws outside extension origins, and the xcap JPEG-over-IPC fallback couldn't sustain 30fps at 1080p on a single main-thread. setScreenShareEnabled goes through Chromium's native getDisplayMedia capture, which is the only path that gets HW-accelerated frames into the WebRTC encoder from WebView2. Audio continues via the WASAPI loopback module — getDisplayMedia can't grab system sound in WebView2 without desktop-capture entitlements Chromium reserves for extensions. The audio track's teardown chains to the ScreenShare video track's 'ended' event so the Windows stop-share overlay kills both sides in lockstep. ScreenSourcePicker is now a quality + audio chooser only; the thumbnail grid disappears because custom source IDs don't round-trip through WebView2, and a custom picker in front of the OS picker just means the user picks twice. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
byGalax
Generated
+1
-1
@@ -834,7 +834,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "chat-app-desktop"
|
||||
version = "0.11.1"
|
||||
version = "0.11.2"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"dryoc",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { memo, startTransition, useEffect, useRef, useState } from 'react';
|
||||
import { useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
import {
|
||||
@@ -9,21 +9,14 @@ import {
|
||||
type ScreenSharePreset,
|
||||
updateScreenShareSettings,
|
||||
} from '../lib/screenShareSettings';
|
||||
import {
|
||||
captureScreenSourceThumbnailBytes,
|
||||
listScreenSources,
|
||||
type ScreenSource,
|
||||
} from '../lib/screenSources';
|
||||
import { MonitorShareIcon, SpinnerIcon, XIcon } from './icons';
|
||||
|
||||
interface Props {
|
||||
open: boolean;
|
||||
onClose: () => void;
|
||||
/** Parent handles the actual share start. `sourceId` is null when the user
|
||||
* clicks "Teilen" without picking a specific source — fallback to the
|
||||
* OS-level getDisplayMedia picker. */
|
||||
/** Parent starts the actual share. The OS picker runs afterwards; the
|
||||
* dialog only gathers quality + audio settings. */
|
||||
onStart: (opts: {
|
||||
sourceId: string | null;
|
||||
preset: ScreenSharePreset;
|
||||
displaySurface: DisplaySurfaceHint;
|
||||
framerate: number | null;
|
||||
@@ -31,143 +24,32 @@ interface Props {
|
||||
}) => Promise<void>;
|
||||
}
|
||||
|
||||
// Discord-style picker. Replaces the old form-field dialog with a thumbnail
|
||||
// grid sourced from the Rust `enumerate_screen_sources` command. Clicking a
|
||||
// thumbnail stashes its Chromium-format id; the parent then attempts a
|
||||
// `chromeMediaSourceId`-constrained getUserMedia call. If WebView2 ignores
|
||||
// the constraint (it may), the fallback OS picker still runs — but at least
|
||||
// the user already saw + chose from a real preview first.
|
||||
// Quality + audio chooser shown before "Bildschirm teilen" opens the OS
|
||||
// source picker. We can't substitute sources in WebView2 — the
|
||||
// `chromeMediaSource: 'desktop'` constraint is extension-only and
|
||||
// ScreenCaptureStarting offers allow/deny, not source-injection — so a
|
||||
// custom thumbnail grid would just double-pick (user picks here, then
|
||||
// picks again in the OS dialog). Dropping the grid keeps the smooth
|
||||
// Chromium-native capture pipeline and narrows the UX to the one
|
||||
// decision that still matters at share-time: quality + audio.
|
||||
export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
|
||||
const { t } = useTranslation(['app']);
|
||||
const initial = getScreenShareSettings();
|
||||
const [sources, setSources] = useState<ScreenSource[] | null>(null);
|
||||
// Thumbnails are kept in a separate state from the source list so an
|
||||
// arriving thumbnail never creates a new `ScreenSource` object for
|
||||
// unrelated cards — memo compares `thumbnailUrl` by string identity,
|
||||
// so only the one card whose URL changes rerenders.
|
||||
const [thumbnailUrls, setThumbnailUrls] = useState<Record<string, string>>({});
|
||||
// All blob URLs we've handed out this session. Revoked on picker close
|
||||
// so the native buffers they point at don't leak across opens.
|
||||
const blobUrlsRef = useRef<string[]>([]);
|
||||
const [selectedId, setSelectedId] = useState<string | null>(null);
|
||||
const [preset, setPreset] = useState<ScreenSharePreset>(initial.preset);
|
||||
const [includeAudio, setIncludeAudio] = useState<boolean>(initial.includeSystemAudio);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
// Two-phase load: (1) fast list returns names + placeholders so the grid
|
||||
// paints instantly, (2) capture thumbnails in a bounded worker-pool
|
||||
// using the binary-IPC variant. Arriving bytes are wrapped in a Blob
|
||||
// and exposed via URL.createObjectURL — no base64 on either side,
|
||||
// which is the single biggest main-thread win compared to the old
|
||||
// JSON-of-base64 flow. Combined with rAF-batched state updates, the
|
||||
// picker stays responsive even on 20+ source enumerations.
|
||||
useEffect(() => {
|
||||
if (!open) {
|
||||
// Revoke blob URLs created during the last session so native
|
||||
// buffers don't linger after close.
|
||||
for (const url of blobUrlsRef.current) URL.revokeObjectURL(url);
|
||||
blobUrlsRef.current = [];
|
||||
setSources(null);
|
||||
setThumbnailUrls({});
|
||||
setSelectedId(null);
|
||||
setError(null);
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
// Coalesce thumbnail arrivals within a single animation frame into
|
||||
// one setState — cuts re-renders from O(N) to O(frames) during the
|
||||
// initial fan-in and prevents consecutive 10-30 ms long tasks from
|
||||
// stacking in one frame.
|
||||
//
|
||||
// Previously `batch` was aliased to `pendingUrls` and then we cleared
|
||||
// pendingUrls via `delete` — which emptied batch too (same reference)
|
||||
// and every flush ended up spreading nothing into the state. Clone
|
||||
// first, then clear, so the batch keeps its entries.
|
||||
let pendingUrls: Record<string, string> = {};
|
||||
let rafScheduled = false;
|
||||
const flush = () => {
|
||||
rafScheduled = false;
|
||||
const batch = pendingUrls;
|
||||
if (Object.keys(batch).length === 0) return;
|
||||
pendingUrls = {};
|
||||
startTransition(() => {
|
||||
setThumbnailUrls((prev) => ({ ...prev, ...batch }));
|
||||
});
|
||||
};
|
||||
const queueUrl = (id: string, url: string) => {
|
||||
pendingUrls[id] = url;
|
||||
if (!rafScheduled) {
|
||||
rafScheduled = true;
|
||||
requestAnimationFrame(flush);
|
||||
}
|
||||
};
|
||||
|
||||
// Concurrency 2: Windows GDI BitBlt / PrintWindow contends for the
|
||||
// desktop compositor, so 4+ parallel captures stutter the whole Tauri
|
||||
// window. 2 in parallel keeps the compositor breathing.
|
||||
const CONCURRENCY = 2;
|
||||
void (async () => {
|
||||
const list = await listScreenSources();
|
||||
if (cancelled) return;
|
||||
setSources(list);
|
||||
const queue = [...list];
|
||||
const pickOne = (src: typeof list[number]) => {
|
||||
void (async () => {
|
||||
const blob = await captureScreenSourceThumbnailBytes(src.id);
|
||||
if (cancelled) {
|
||||
// Edge case: picker closed while this request was in flight.
|
||||
// blob may still exist; nothing holds a URL to it, so it GCs.
|
||||
return;
|
||||
}
|
||||
if (blob) {
|
||||
const url = URL.createObjectURL(blob);
|
||||
blobUrlsRef.current.push(url);
|
||||
queueUrl(src.id, url);
|
||||
}
|
||||
const nextSrc = queue.shift();
|
||||
if (nextSrc) pickOne(nextSrc);
|
||||
})();
|
||||
};
|
||||
for (let i = 0; i < Math.min(CONCURRENCY, queue.length); i++) {
|
||||
const s = queue.shift();
|
||||
if (s) pickOne(s);
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [open]);
|
||||
|
||||
if (!open) return null;
|
||||
|
||||
const screens = sources?.filter((s) => s.kind === 'screen') ?? [];
|
||||
const windows = sources?.filter((s) => s.kind === 'window') ?? [];
|
||||
const hasAny = (sources?.length ?? 0) > 0;
|
||||
|
||||
async function handleStart(): Promise<void> {
|
||||
setBusy(true);
|
||||
setError(null);
|
||||
try {
|
||||
// Persist user's audio + preset choice so subsequent shares start with
|
||||
// the same prefs when they skip the picker. The picker itself stays
|
||||
// as the entry for future starts (right-click on share button also
|
||||
// opens it — see InCallPanel wiring).
|
||||
updateScreenShareSettings({ preset, includeSystemAudio: includeAudio });
|
||||
// Infer a displaySurface hint from the selection so the fallback OS
|
||||
// picker jumps to the right tab when our direct-publish path is
|
||||
// rejected by WebView2.
|
||||
const selected = sources?.find((s) => s.id === selectedId) ?? null;
|
||||
const hint: DisplaySurfaceHint =
|
||||
selected?.kind === 'screen'
|
||||
? 'monitor'
|
||||
: selected?.kind === 'window'
|
||||
? 'window'
|
||||
: null;
|
||||
await onStart({
|
||||
sourceId: selected?.id ?? null,
|
||||
preset,
|
||||
displaySurface: hint,
|
||||
displaySurface: null,
|
||||
framerate: null,
|
||||
includeAudio,
|
||||
});
|
||||
@@ -189,7 +71,7 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
|
||||
>
|
||||
<div
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
className="flex max-h-[88vh] w-full max-w-[860px] flex-col overflow-hidden rounded-2xl border border-line bg-surface-3 shadow-xl"
|
||||
className="flex w-full max-w-[420px] flex-col overflow-hidden rounded-2xl border border-line bg-surface-3 shadow-xl"
|
||||
>
|
||||
<header className="flex items-center justify-between border-b border-line px-5 py-3">
|
||||
<div className="flex items-center gap-2">
|
||||
@@ -208,206 +90,71 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
|
||||
</button>
|
||||
</header>
|
||||
|
||||
<div className="flex-1 overflow-y-auto">
|
||||
{sources === null ? (
|
||||
<div className="flex h-40 items-center justify-center gap-2 text-sm text-fg-muted">
|
||||
<SpinnerIcon className="h-4 w-4" />
|
||||
<span>
|
||||
{t('app:call.share_loading', { defaultValue: 'Lade Quellen…' })}
|
||||
</span>
|
||||
</div>
|
||||
) : !hasAny ? (
|
||||
<div className="flex flex-col items-center gap-2 px-6 py-10 text-center text-sm text-fg-muted">
|
||||
<MonitorShareIcon className="h-6 w-6 opacity-60" />
|
||||
<span>
|
||||
{t('app:call.share_no_sources', {
|
||||
defaultValue:
|
||||
'Keine Quellen-Previews verfügbar. Klick auf „Teilen" öffnet den System-Picker.',
|
||||
})}
|
||||
</span>
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-5 p-5">
|
||||
{screens.length > 0 && (
|
||||
<SourceSection
|
||||
title={t('app:call.share_screens', { defaultValue: 'Bildschirme' })}
|
||||
sources={screens}
|
||||
thumbnailUrls={thumbnailUrls}
|
||||
selectedId={selectedId}
|
||||
onSelect={setSelectedId}
|
||||
/>
|
||||
)}
|
||||
{windows.length > 0 && (
|
||||
<SourceSection
|
||||
title={t('app:call.share_windows', { defaultValue: 'Fenster' })}
|
||||
sources={windows}
|
||||
thumbnailUrls={thumbnailUrls}
|
||||
selectedId={selectedId}
|
||||
onSelect={setSelectedId}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex flex-col gap-4 px-5 py-5">
|
||||
<p className="text-xs text-fg-muted">
|
||||
{t('app:call.share_os_picker_hint', {
|
||||
defaultValue:
|
||||
'Nach dem Klick auf „Teilen" wählst du im System-Dialog den Bildschirm oder das Fenster aus.',
|
||||
})}
|
||||
</p>
|
||||
|
||||
<label className="flex items-center justify-between gap-3 text-xs text-fg">
|
||||
<span className="font-semibold uppercase tracking-wider text-fg-muted">
|
||||
{t('app:call.share_quality', { defaultValue: 'Qualität' })}
|
||||
</span>
|
||||
<select
|
||||
value={preset}
|
||||
onChange={(e) => setPreset(e.target.value as ScreenSharePreset)}
|
||||
className="flex-1 rounded-md border border-line bg-surface-3 px-2 py-1 text-xs text-fg focus:border-accent focus:outline-none"
|
||||
>
|
||||
{PRESET_ORDER.map((p) => (
|
||||
<option key={p} value={p}>
|
||||
{getPresetParams(p).label} · ≤ {getPresetParams(p).bitrateKbps} kbps
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
|
||||
<label className="flex cursor-pointer items-center gap-2 text-xs text-fg">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={includeAudio}
|
||||
onChange={(e) => setIncludeAudio(e.target.checked)}
|
||||
className="accent-accent"
|
||||
/>
|
||||
<span>
|
||||
{t('app:call.share_system_audio', {
|
||||
defaultValue: 'System-Sound mit übertragen',
|
||||
})}
|
||||
</span>
|
||||
</label>
|
||||
|
||||
<footer className="flex flex-col gap-3 border-t border-line bg-surface-2 px-5 py-3">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
<label className="flex cursor-pointer items-center gap-2 text-xs text-fg">
|
||||
<span className="font-semibold uppercase tracking-wider text-fg-muted">
|
||||
{t('app:call.share_quality', { defaultValue: 'Qualität' })}
|
||||
</span>
|
||||
<select
|
||||
value={preset}
|
||||
onChange={(e) => setPreset(e.target.value as ScreenSharePreset)}
|
||||
className="rounded-md border border-line bg-surface-3 px-2 py-1 text-xs text-fg focus:border-accent focus:outline-none"
|
||||
>
|
||||
{PRESET_ORDER.map((p) => (
|
||||
<option key={p} value={p}>
|
||||
{getPresetParams(p).label} · ≤ {getPresetParams(p).bitrateKbps} kbps
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<label className="flex cursor-pointer items-center gap-2 text-xs text-fg">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={includeAudio}
|
||||
onChange={(e) => setIncludeAudio(e.target.checked)}
|
||||
className="accent-accent"
|
||||
/>
|
||||
<span>
|
||||
{t('app:call.share_system_audio', {
|
||||
defaultValue: 'System-Sound mit übertragen',
|
||||
})}
|
||||
</span>
|
||||
</label>
|
||||
</div>
|
||||
{error && (
|
||||
<p role="alert" className="text-xs text-rose-600 dark:text-rose-300">
|
||||
{error}
|
||||
</p>
|
||||
)}
|
||||
<div className="flex items-center justify-end gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClose}
|
||||
className="cursor-pointer rounded-md border border-line bg-surface-3 px-3 py-2 text-sm text-fg transition hover:bg-surface-2"
|
||||
>
|
||||
{t('app:friends.action_cancel', { defaultValue: 'Abbrechen' })}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void handleStart()}
|
||||
disabled={busy}
|
||||
className="inline-flex cursor-pointer items-center gap-1.5 rounded-md bg-accent px-3 py-2 text-sm font-semibold text-accent-fg transition hover:brightness-110 disabled:cursor-not-allowed disabled:opacity-60"
|
||||
>
|
||||
{busy && <SpinnerIcon className="h-4 w-4" />}
|
||||
<span>
|
||||
{selectedId
|
||||
? t('app:call.share_start', { defaultValue: 'Teilen' })
|
||||
: t('app:call.share_pick_system', {
|
||||
defaultValue: 'Ohne Auswahl weiter',
|
||||
})}
|
||||
</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<footer className="flex items-center justify-end gap-2 border-t border-line bg-surface-2 px-5 py-3">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClose}
|
||||
className="cursor-pointer rounded-md border border-line bg-surface-3 px-3 py-2 text-sm text-fg transition hover:bg-surface-2"
|
||||
>
|
||||
{t('app:friends.action_cancel', { defaultValue: 'Abbrechen' })}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void handleStart()}
|
||||
disabled={busy}
|
||||
className="inline-flex cursor-pointer items-center gap-1.5 rounded-md bg-accent px-3 py-2 text-sm font-semibold text-accent-fg transition hover:brightness-110 disabled:cursor-not-allowed disabled:opacity-60"
|
||||
>
|
||||
{busy && <SpinnerIcon className="h-4 w-4" />}
|
||||
<span>{t('app:call.share_start', { defaultValue: 'Teilen' })}</span>
|
||||
</button>
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SourceSection({
|
||||
title,
|
||||
sources,
|
||||
thumbnailUrls,
|
||||
selectedId,
|
||||
onSelect,
|
||||
}: {
|
||||
title: string;
|
||||
sources: ScreenSource[];
|
||||
thumbnailUrls: Record<string, string>;
|
||||
selectedId: string | null;
|
||||
onSelect: (id: string) => void;
|
||||
}) {
|
||||
return (
|
||||
<section>
|
||||
<h4 className="mb-2 text-xs font-semibold uppercase tracking-wider text-fg-muted">
|
||||
{title}
|
||||
</h4>
|
||||
<div className="grid grid-cols-2 gap-2.5 sm:grid-cols-3">
|
||||
{sources.map((src) => (
|
||||
<SourceCard
|
||||
key={src.id}
|
||||
source={src}
|
||||
thumbnailUrl={thumbnailUrls[src.id] ?? null}
|
||||
selected={selectedId === src.id}
|
||||
onSelect={onSelect}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
// Memoized so a thumbnail arriving for card B doesn't re-render card A.
|
||||
// Keeps re-render work proportional to the number of updates instead of
|
||||
// "whole grid on every update" — which was the main reason scrolling felt
|
||||
// frozen during the initial thumbnail fan-in.
|
||||
//
|
||||
// The parent passes `onSelect(id)` rather than an inline `onClick`-arrow
|
||||
// so the callback reference stays stable across renders; otherwise
|
||||
// React.memo would always see a fresh function prop and re-render every
|
||||
// card on every parent update.
|
||||
const SourceCard = memo(function SourceCard({
|
||||
source,
|
||||
thumbnailUrl,
|
||||
selected,
|
||||
onSelect,
|
||||
}: {
|
||||
source: ScreenSource;
|
||||
thumbnailUrl: string | null;
|
||||
selected: boolean;
|
||||
onSelect: (id: string) => void;
|
||||
}) {
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onSelect(source.id)}
|
||||
aria-pressed={selected}
|
||||
title={source.name}
|
||||
className={
|
||||
'group flex cursor-pointer flex-col overflow-hidden rounded-lg border transition focus:outline-none focus-visible:ring-2 focus-visible:ring-accent/50 ' +
|
||||
(selected
|
||||
? 'border-accent ring-2 ring-accent/30'
|
||||
: 'border-line hover:border-accent/70')
|
||||
}
|
||||
>
|
||||
<div className="relative aspect-video w-full overflow-hidden bg-black">
|
||||
{thumbnailUrl ? (
|
||||
// decoding="async" keeps image decode off the main-thread paint
|
||||
// step; loading="lazy" means cards outside the viewport don't
|
||||
// ask the browser to decode until the user scrolls to them. The
|
||||
// URL is a blob: URL backed by the ArrayBuffer Rust sent over
|
||||
// IPC — no base64 decode, no data-URL parse, just direct bytes
|
||||
// into the decoder.
|
||||
<img
|
||||
src={thumbnailUrl}
|
||||
alt=""
|
||||
decoding="async"
|
||||
loading="lazy"
|
||||
className="h-full w-full object-contain transition group-hover:brightness-110"
|
||||
draggable={false}
|
||||
/>
|
||||
) : (
|
||||
<div className="flex h-full w-full items-center justify-center bg-gradient-to-br from-surface-2 to-surface-3 text-fg-muted">
|
||||
<MonitorShareIcon className="h-6 w-6 opacity-50" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="truncate px-2.5 py-1.5 text-left text-xs font-medium text-fg">
|
||||
{source.name}
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
});
|
||||
|
||||
@@ -75,11 +75,6 @@ import {
|
||||
setAllPipelinesSinkId,
|
||||
setPipelineGain,
|
||||
} from '../lib/remoteAudioPipelines';
|
||||
import {
|
||||
type NativeCaptureHandle,
|
||||
NativeCaptureUnavailable,
|
||||
startNativeCapture,
|
||||
} from '../lib/screenCapture';
|
||||
import {
|
||||
type SystemAudioHandle,
|
||||
startSystemAudioCapture,
|
||||
@@ -304,15 +299,10 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
// to `connected` after a few seconds so the UI doesn't hang in "Verbinde…"
|
||||
// indefinitely. The solo-timeout will then cleanly close if nobody arrives.
|
||||
const joinFallbackTimerRef = useRef<number | null>(null);
|
||||
// Active native screen-capture handle (Rust side). Set when
|
||||
// startScreenShare takes the xcap path; cleared on stopScreenShare or
|
||||
// on the canvas track's 'ended' event. Not kept in React state because
|
||||
// it never feeds into a render.
|
||||
const nativeCaptureRef = useRef<NativeCaptureHandle | null>(null);
|
||||
// Matching handle for the Windows-only WASAPI system-audio capture.
|
||||
// Lives in lockstep with the video handle above when the user picks
|
||||
// "Mit System-Sound"; teardown is wired so that stopping either track
|
||||
// also stops the other, so stale audio can't outlive the video share.
|
||||
// Handle for the Windows-only WASAPI system-audio capture. Lives in
|
||||
// lockstep with the ScreenShare video track published by
|
||||
// setScreenShareEnabled; teardown is chained to the video track's
|
||||
// `ended` event so stale audio can never outlive the video share.
|
||||
const nativeAudioCaptureRef = useRef<SystemAudioHandle | null>(null);
|
||||
const roomRef = useRef<Room | null>(null);
|
||||
// Web Audio graph that mixes live mic + soundboard sources into a single
|
||||
@@ -488,13 +478,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
// Stop any lingering native screen-capture thread so we don't leak
|
||||
// Rust threads when the call ends mid-share.
|
||||
if (nativeCaptureRef.current) {
|
||||
const h = nativeCaptureRef.current;
|
||||
nativeCaptureRef.current = null;
|
||||
await h.stop().catch(() => undefined);
|
||||
}
|
||||
if (nativeAudioCaptureRef.current) {
|
||||
const h = nativeAudioCaptureRef.current;
|
||||
nativeAudioCaptureRef.current = null;
|
||||
@@ -729,6 +712,11 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
r.on(RoomEvent.ParticipantConnected, () => {
|
||||
setRemoteParticipants(Array.from(r.remoteParticipants.values()));
|
||||
if (presenceRef.current !== 'dnd') void playJoinBeep();
|
||||
// Rejoin während wir schon `connected` sind: markConnectedIfReady
|
||||
// returnt früh und würde den Solo-Timer nicht cancellen. Hier
|
||||
// unbedingt clearen, sonst kickt der 5-Minuten-Timer obwohl der
|
||||
// andere Peer längst wieder im Raum ist.
|
||||
clearSoloTimer();
|
||||
markConnectedIfReady(r, conversationId, mediaKind, callId);
|
||||
});
|
||||
|
||||
@@ -1173,11 +1161,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
preset: ScreenSharePreset;
|
||||
displaySurface: DisplaySurfaceHint;
|
||||
framerate: number | null;
|
||||
/** Chromium-format source id from our custom picker. When set, we
|
||||
* try to capture that exact source via `chromeMediaSourceId`
|
||||
* instead of the OS-level getDisplayMedia picker. Falls back to
|
||||
* getDisplayMedia if WebView2 rejects the constraint. */
|
||||
sourceId: string | null;
|
||||
}>,
|
||||
) => {
|
||||
const r = roomRef.current;
|
||||
@@ -1185,8 +1168,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
const lp = r.localParticipant;
|
||||
if (lp.isScreenShareEnabled) return;
|
||||
|
||||
// Persist the user's choice so subsequent shares use the same config
|
||||
// without re-opening the picker unless they want to change something.
|
||||
const settings = getScreenShareSettings();
|
||||
const preset = overrides?.preset ?? settings.preset;
|
||||
const displaySurface =
|
||||
@@ -1201,220 +1182,17 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
|
||||
const ssParams = getPresetParams(preset);
|
||||
const fps = framerateOverride ?? ssParams.framerate;
|
||||
const sourceId = overrides?.sourceId ?? null;
|
||||
|
||||
// Ordered capture paths when our custom picker supplied a sourceId:
|
||||
// 1. chromeMediaSource video — hardware-accelerated path that
|
||||
// hands the frames directly to WebRTC, same mechanism the OS
|
||||
// picker uses internally. Fast, smooth, what users expect.
|
||||
// 2. xcap native (JPEG → canvas → captureStream) — fallback for
|
||||
// WebView2 versions that reject the legacy chromeMediaSource
|
||||
// constraint. Functional but CPU-heavy; the per-frame JPEG
|
||||
// encode + base64 + createImageBitmap chain shows up as
|
||||
// visible stutter on 1080p30.
|
||||
// 3. OS picker (setScreenShareEnabled) — last resort when both
|
||||
// native paths throw, and the only path when no sourceId was
|
||||
// supplied.
|
||||
//
|
||||
// Audio (Windows only) always goes through the WASAPI loopback
|
||||
// module — getUserMedia's chromeMediaSource audio constraint
|
||||
// throws AbortError on Window captures, and pairing both into one
|
||||
// call ended up blocking the video path entirely. Splitting them
|
||||
// means the video path stays fast and the audio path stays
|
||||
// reliable for any source type.
|
||||
if (!sourceId) {
|
||||
console.info(
|
||||
'screen-share: no sourceId supplied by picker, OS picker will open',
|
||||
);
|
||||
}
|
||||
|
||||
// Publish system audio via WASAPI. Resolves to the audio publication
|
||||
// + an ended-cleanup registration so whichever video path succeeds
|
||||
// can wire the audio teardown onto its own video-ended handler.
|
||||
const startWasapiAudio = async (
|
||||
LkTrack: typeof import('livekit-client').Track,
|
||||
): Promise<{
|
||||
stopAudio: () => Promise<void>;
|
||||
} | null> => {
|
||||
if (!settings.includeSystemAudio) return null;
|
||||
try {
|
||||
const audioHandle = await startSystemAudioCapture();
|
||||
nativeAudioCaptureRef.current = audioHandle;
|
||||
const audioMst = audioHandle.stream.getAudioTracks()[0];
|
||||
if (!audioMst) {
|
||||
await audioHandle.stop();
|
||||
nativeAudioCaptureRef.current = null;
|
||||
return null;
|
||||
}
|
||||
const audioPub = await lp.publishTrack(audioMst, {
|
||||
source: LkTrack.Source.ScreenShareAudio,
|
||||
});
|
||||
audioMst.addEventListener('ended', () => {
|
||||
void (async () => {
|
||||
try {
|
||||
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
|
||||
} catch {
|
||||
/* already unpublished */
|
||||
}
|
||||
})();
|
||||
});
|
||||
const stopAudio = async () => {
|
||||
try {
|
||||
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
|
||||
} catch {
|
||||
/* already unpublished */
|
||||
}
|
||||
const active = nativeAudioCaptureRef.current;
|
||||
if (active && active.captureId === audioHandle.captureId) {
|
||||
nativeAudioCaptureRef.current = null;
|
||||
await active.stop().catch(() => undefined);
|
||||
}
|
||||
};
|
||||
return { stopAudio };
|
||||
} catch (err: unknown) {
|
||||
console.warn(
|
||||
'screen-share: native system-audio unavailable, sharing video only',
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
if (nativeAudioCaptureRef.current) {
|
||||
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
|
||||
nativeAudioCaptureRef.current = null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
// ----- Path 1: chromeMediaSource (fast) -----
|
||||
if (sourceId) {
|
||||
try {
|
||||
const { Track: LkTrack } = await import('livekit-client');
|
||||
const maxWidth = ssParams.dims?.width ?? 3840;
|
||||
const maxHeight = ssParams.dims?.height ?? 2160;
|
||||
// Cast chains: browsers expose the legacy constraint via
|
||||
// `MediaTrackConstraints.mandatory` which isn't in lib.dom.
|
||||
const videoConstraints = {
|
||||
mandatory: {
|
||||
chromeMediaSource: 'desktop',
|
||||
chromeMediaSourceId: sourceId,
|
||||
maxWidth,
|
||||
maxHeight,
|
||||
maxFrameRate: fps,
|
||||
},
|
||||
} as unknown as MediaTrackConstraints;
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: false,
|
||||
video: videoConstraints,
|
||||
});
|
||||
const videoMst = stream.getVideoTracks()[0];
|
||||
if (!videoMst) {
|
||||
stream.getTracks().forEach((t) => t.stop());
|
||||
throw new Error('no video track from chromeMediaSource');
|
||||
}
|
||||
// H.264 lets Chromium's hardware encoder take over on Windows
|
||||
// (Intel QuickSync / NVENC / AMD VCE) — VP9 is software-only in
|
||||
// Chromium's WebRTC path and collapses under the L3T3_KEY
|
||||
// scalability mode we set globally for camera. `contentHint` is
|
||||
// what setScreenShareEnabled sets internally; pushing it by hand
|
||||
// here tells the encoder to weight sharpness over smoothness,
|
||||
// which is right for UI/text-heavy screen content.
|
||||
videoMst.contentHint = 'detail';
|
||||
const videoPub = await lp.publishTrack(videoMst, {
|
||||
source: LkTrack.Source.ScreenShare,
|
||||
videoCodec: 'h264',
|
||||
});
|
||||
const audioRes = await startWasapiAudio(LkTrack);
|
||||
videoMst.addEventListener('ended', () => {
|
||||
void (async () => {
|
||||
try {
|
||||
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
|
||||
} catch {
|
||||
/* already unpublished */
|
||||
}
|
||||
if (audioRes) await audioRes.stopAudio();
|
||||
setIsScreenSharing(false);
|
||||
})();
|
||||
});
|
||||
console.info('screen-share: chromeMediaSource active', {
|
||||
audio: audioRes != null,
|
||||
});
|
||||
setIsScreenSharing(true);
|
||||
return;
|
||||
} catch (err: unknown) {
|
||||
console.warn(
|
||||
'screen-share: chromeMediaSource failed, trying native xcap',
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ----- Path 2: xcap native (CPU fallback) -----
|
||||
if (sourceId) {
|
||||
try {
|
||||
const { Track: LkTrack } = await import('livekit-client');
|
||||
const maxWidth = ssParams.dims?.width ?? 1920;
|
||||
const maxHeight = ssParams.dims?.height ?? 1080;
|
||||
const handle = await startNativeCapture({
|
||||
sourceId,
|
||||
maxWidth,
|
||||
maxHeight,
|
||||
fps,
|
||||
});
|
||||
nativeCaptureRef.current = handle;
|
||||
const videoMst = handle.stream.getVideoTracks()[0];
|
||||
if (!videoMst) {
|
||||
await handle.stop();
|
||||
nativeCaptureRef.current = null;
|
||||
throw new NativeCaptureUnavailable('canvas stream produced no video track');
|
||||
}
|
||||
videoMst.contentHint = 'detail';
|
||||
const videoPub = await lp.publishTrack(videoMst, {
|
||||
source: LkTrack.Source.ScreenShare,
|
||||
videoCodec: 'h264',
|
||||
});
|
||||
const audioRes = await startWasapiAudio(LkTrack);
|
||||
videoMst.addEventListener('ended', () => {
|
||||
void (async () => {
|
||||
try {
|
||||
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
|
||||
} catch {
|
||||
/* already unpublished */
|
||||
}
|
||||
const active = nativeCaptureRef.current;
|
||||
if (active && active.captureId === handle.captureId) {
|
||||
nativeCaptureRef.current = null;
|
||||
await active.stop().catch(() => undefined);
|
||||
}
|
||||
if (audioRes) await audioRes.stopAudio();
|
||||
setIsScreenSharing(false);
|
||||
})();
|
||||
});
|
||||
console.info('screen-share: native xcap active', {
|
||||
audio: audioRes != null,
|
||||
});
|
||||
setIsScreenSharing(true);
|
||||
return;
|
||||
} catch (err: unknown) {
|
||||
if (nativeCaptureRef.current) {
|
||||
await nativeCaptureRef.current.stop().catch(() => undefined);
|
||||
nativeCaptureRef.current = null;
|
||||
}
|
||||
if (nativeAudioCaptureRef.current) {
|
||||
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
|
||||
nativeAudioCaptureRef.current = null;
|
||||
}
|
||||
console.warn(
|
||||
'screen-share: native xcap failed, falling back to OS picker',
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Single capture path: setScreenShareEnabled → OS picker → Chromium's
|
||||
// hardware-accelerated capture feeds WebRTC directly. In WebView2
|
||||
// the extension-only `chromeMediaSource: 'desktop'` constraint and
|
||||
// our xcap JPEG-over-IPC pipeline were both too slow for smooth
|
||||
// 30fps, so the OS picker is the only route to the fast pipeline.
|
||||
// Audio is handled separately below via WASAPI because
|
||||
// getDisplayMedia can't grab system audio in WebView2.
|
||||
try {
|
||||
await lp.setScreenShareEnabled(true, {
|
||||
// "Go live" mode — capture system audio alongside the screen when
|
||||
// the user opted in. On hosts that can't fulfil the request the
|
||||
// browser quietly drops it; peers just get video-only, no error.
|
||||
audio: settings.includeSystemAudio,
|
||||
audio: false,
|
||||
...(ssParams.dims
|
||||
? {
|
||||
resolution: {
|
||||
@@ -1430,9 +1208,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
frameRate: fps,
|
||||
},
|
||||
}),
|
||||
// Hints the OS picker to pre-filter by source kind. `null` = no
|
||||
// filter (show both). Cast because TS lib.dom doesn't know the
|
||||
// field yet on all branches.
|
||||
...(displaySurface
|
||||
? ({ displaySurface } as { displaySurface: DisplaySurfaceHint })
|
||||
: {}),
|
||||
@@ -1441,24 +1216,67 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
setIsScreenSharing(true);
|
||||
} catch (err: unknown) {
|
||||
console.error('setScreenShareEnabled failed', err);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!settings.includeSystemAudio) return;
|
||||
|
||||
// WASAPI system-audio: publish as a separate ScreenShareAudio track.
|
||||
// Chain its teardown onto the ScreenShare video track's `ended`
|
||||
// event so the Windows "Stop sharing" overlay kills audio alongside
|
||||
// video; the audio track's own `ended` handler covers the case
|
||||
// where WASAPI dies by itself.
|
||||
try {
|
||||
const { Track: LkTrack } = await import('livekit-client');
|
||||
const audioHandle = await startSystemAudioCapture();
|
||||
nativeAudioCaptureRef.current = audioHandle;
|
||||
const audioMst = audioHandle.stream.getAudioTracks()[0];
|
||||
if (!audioMst) {
|
||||
await audioHandle.stop();
|
||||
nativeAudioCaptureRef.current = null;
|
||||
return;
|
||||
}
|
||||
const audioPub = await lp.publishTrack(audioMst, {
|
||||
source: LkTrack.Source.ScreenShareAudio,
|
||||
});
|
||||
let videoMst: MediaStreamTrack | null = null;
|
||||
for (const pub of lp.videoTrackPublications.values()) {
|
||||
if (pub.source === LkTrack.Source.ScreenShare && pub.track) {
|
||||
videoMst = pub.track.mediaStreamTrack;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const teardown = () => {
|
||||
void (async () => {
|
||||
try {
|
||||
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
|
||||
} catch {
|
||||
/* already unpublished */
|
||||
}
|
||||
const active = nativeAudioCaptureRef.current;
|
||||
if (active && active.captureId === audioHandle.captureId) {
|
||||
nativeAudioCaptureRef.current = null;
|
||||
await active.stop().catch(() => undefined);
|
||||
}
|
||||
})();
|
||||
};
|
||||
audioMst.addEventListener('ended', teardown);
|
||||
if (videoMst) videoMst.addEventListener('ended', teardown);
|
||||
} catch (err: unknown) {
|
||||
console.warn(
|
||||
'screen-share: native system-audio unavailable, sharing video only',
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
if (nativeAudioCaptureRef.current) {
|
||||
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
|
||||
nativeAudioCaptureRef.current = null;
|
||||
}
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const stopScreenShare = useCallback(async () => {
|
||||
// Native path first — stopping the handle kills the canvas track,
|
||||
// which fires 'ended' on the MediaStreamTrack, which the start-handler
|
||||
// already listens to for unpublishing and flipping isScreenSharing.
|
||||
if (nativeCaptureRef.current) {
|
||||
const h = nativeCaptureRef.current;
|
||||
nativeCaptureRef.current = null;
|
||||
try {
|
||||
await h.stop();
|
||||
} catch (err: unknown) {
|
||||
console.warn('native capture stop failed', err);
|
||||
}
|
||||
}
|
||||
if (nativeAudioCaptureRef.current) {
|
||||
const h = nativeAudioCaptureRef.current;
|
||||
nativeAudioCaptureRef.current = null;
|
||||
@@ -1471,18 +1289,13 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
const r = roomRef.current;
|
||||
if (!r) return;
|
||||
const lp = r.localParticipant;
|
||||
// Unpublish any manually-published ScreenShare/ScreenShareAudio tracks
|
||||
// (from the chromeMediaSourceId fallback path). setScreenShareEnabled
|
||||
// only manages LK's own internally-captured tracks.
|
||||
const toUnpublish: import('livekit-client').LocalTrackPublication[] = [];
|
||||
for (const pub of lp.videoTrackPublications.values()) {
|
||||
if (pub.source === Track.Source.ScreenShare && pub.track) toUnpublish.push(pub);
|
||||
}
|
||||
// Unpublish any ScreenShareAudio track we manually published alongside
|
||||
// setScreenShareEnabled. LiveKit's setScreenShareEnabled(false) only
|
||||
// drops the video track it captured itself.
|
||||
for (const pub of lp.audioTrackPublications.values()) {
|
||||
if (pub.source === Track.Source.ScreenShareAudio && pub.track) toUnpublish.push(pub);
|
||||
}
|
||||
for (const pub of toUnpublish) {
|
||||
if (pub.track) await lp.unpublishTrack(pub.track).catch(() => undefined);
|
||||
if (pub.source === Track.Source.ScreenShareAudio && pub.track) {
|
||||
await lp.unpublishTrack(pub.track).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
if (lp.isScreenShareEnabled) {
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user