Files
ChatApp/apps/desktop/src/components/ScreenSourcePicker.tsx
T
byGalax 665f450878 feat(call): native WASAPI system-audio for screen-share (Windows)
Hooks the custom screen-share picker up to a native WASAPI loopback
capture so "Mit System-Sound" no longer falls back to the OS picker on
Windows. Rust side opens the default render endpoint, channels 48 kHz
f32 stereo to an AudioWorklet, which feeds a MediaStreamDestination for
LiveKit to publish as ScreenShareAudio. Ring buffer sized for latency
(80 ms target, drop-to-target on overflow) and the AudioContext is
resumed eagerly so initial burstiness can't pile up.

Adds a temporary attachTrack:audio diagnostic log to confirm source
tagging matches between old and new clients.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:03:15 +02:00

414 lines
16 KiB
TypeScript

import { memo, startTransition, useEffect, useRef, useState } from 'react';
import { useTranslation } from 'react-i18next';
import {
type DisplaySurfaceHint,
getPresetParams,
getScreenShareSettings,
PRESET_ORDER,
type ScreenSharePreset,
updateScreenShareSettings,
} from '../lib/screenShareSettings';
import {
captureScreenSourceThumbnailBytes,
listScreenSources,
type ScreenSource,
} from '../lib/screenSources';
import { MonitorShareIcon, SpinnerIcon, XIcon } from './icons';
interface Props {
open: boolean;
onClose: () => void;
/** Parent handles the actual share start. `sourceId` is null when the user
* clicks "Teilen" without picking a specific source — fallback to the
* OS-level getDisplayMedia picker. */
onStart: (opts: {
sourceId: string | null;
preset: ScreenSharePreset;
displaySurface: DisplaySurfaceHint;
framerate: number | null;
includeAudio: boolean;
}) => Promise<void>;
}
// Discord-style picker. Replaces the old form-field dialog with a thumbnail
// grid sourced from the Rust `enumerate_screen_sources` command. Clicking a
// thumbnail stashes its Chromium-format id; the parent then attempts a
// `chromeMediaSourceId`-constrained getUserMedia call. If WebView2 ignores
// the constraint (it may), the fallback OS picker still runs — but at least
// the user already saw + chose from a real preview first.
export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
const { t } = useTranslation(['app']);
const initial = getScreenShareSettings();
const [sources, setSources] = useState<ScreenSource[] | null>(null);
// Thumbnails are kept in a separate state from the source list so an
// arriving thumbnail never creates a new `ScreenSource` object for
// unrelated cards — memo compares `thumbnailUrl` by string identity,
// so only the one card whose URL changes rerenders.
const [thumbnailUrls, setThumbnailUrls] = useState<Record<string, string>>({});
// All blob URLs we've handed out this session. Revoked on picker close
// so the native buffers they point at don't leak across opens.
const blobUrlsRef = useRef<string[]>([]);
const [selectedId, setSelectedId] = useState<string | null>(null);
const [preset, setPreset] = useState<ScreenSharePreset>(initial.preset);
const [includeAudio, setIncludeAudio] = useState<boolean>(initial.includeSystemAudio);
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
// Two-phase load: (1) fast list returns names + placeholders so the grid
// paints instantly, (2) capture thumbnails in a bounded worker-pool
// using the binary-IPC variant. Arriving bytes are wrapped in a Blob
// and exposed via URL.createObjectURL — no base64 on either side,
// which is the single biggest main-thread win compared to the old
// JSON-of-base64 flow. Combined with rAF-batched state updates, the
// picker stays responsive even on 20+ source enumerations.
useEffect(() => {
if (!open) {
// Revoke blob URLs created during the last session so native
// buffers don't linger after close.
for (const url of blobUrlsRef.current) URL.revokeObjectURL(url);
blobUrlsRef.current = [];
setSources(null);
setThumbnailUrls({});
setSelectedId(null);
setError(null);
return;
}
let cancelled = false;
// Coalesce thumbnail arrivals within a single animation frame into
// one setState — cuts re-renders from O(N) to O(frames) during the
// initial fan-in and prevents consecutive 10-30 ms long tasks from
// stacking in one frame.
//
// Previously `batch` was aliased to `pendingUrls` and then we cleared
// pendingUrls via `delete` — which emptied batch too (same reference)
// and every flush ended up spreading nothing into the state. Clone
// first, then clear, so the batch keeps its entries.
let pendingUrls: Record<string, string> = {};
let rafScheduled = false;
const flush = () => {
rafScheduled = false;
const batch = pendingUrls;
if (Object.keys(batch).length === 0) return;
pendingUrls = {};
startTransition(() => {
setThumbnailUrls((prev) => ({ ...prev, ...batch }));
});
};
const queueUrl = (id: string, url: string) => {
pendingUrls[id] = url;
if (!rafScheduled) {
rafScheduled = true;
requestAnimationFrame(flush);
}
};
// Concurrency 2: Windows GDI BitBlt / PrintWindow contends for the
// desktop compositor, so 4+ parallel captures stutter the whole Tauri
// window. 2 in parallel keeps the compositor breathing.
const CONCURRENCY = 2;
void (async () => {
const list = await listScreenSources();
if (cancelled) return;
setSources(list);
const queue = [...list];
const pickOne = (src: typeof list[number]) => {
void (async () => {
const blob = await captureScreenSourceThumbnailBytes(src.id);
if (cancelled) {
// Edge case: picker closed while this request was in flight.
// blob may still exist; nothing holds a URL to it, so it GCs.
return;
}
if (blob) {
const url = URL.createObjectURL(blob);
blobUrlsRef.current.push(url);
queueUrl(src.id, url);
}
const nextSrc = queue.shift();
if (nextSrc) pickOne(nextSrc);
})();
};
for (let i = 0; i < Math.min(CONCURRENCY, queue.length); i++) {
const s = queue.shift();
if (s) pickOne(s);
}
})();
return () => {
cancelled = true;
};
}, [open]);
if (!open) return null;
const screens = sources?.filter((s) => s.kind === 'screen') ?? [];
const windows = sources?.filter((s) => s.kind === 'window') ?? [];
const hasAny = (sources?.length ?? 0) > 0;
async function handleStart(): Promise<void> {
setBusy(true);
setError(null);
try {
// Persist user's audio + preset choice so subsequent shares start with
// the same prefs when they skip the picker. The picker itself stays
// as the entry for future starts (right-click on share button also
// opens it — see InCallPanel wiring).
updateScreenShareSettings({ preset, includeSystemAudio: includeAudio });
// Infer a displaySurface hint from the selection so the fallback OS
// picker jumps to the right tab when our direct-publish path is
// rejected by WebView2.
const selected = sources?.find((s) => s.id === selectedId) ?? null;
const hint: DisplaySurfaceHint =
selected?.kind === 'screen'
? 'monitor'
: selected?.kind === 'window'
? 'window'
: null;
await onStart({
sourceId: selected?.id ?? null,
preset,
displaySurface: hint,
framerate: null,
includeAudio,
});
onClose();
} catch (err: unknown) {
setError(err instanceof Error ? err.message : 'screenshare failed');
} finally {
setBusy(false);
}
}
return (
<div
role="dialog"
aria-modal="true"
aria-label={t('app:call.share_dialog_title', { defaultValue: 'Bildschirm teilen' })}
className="fixed inset-0 z-50 flex items-center justify-center bg-black/60 p-4"
onClick={onClose}
>
<div
onClick={(e) => e.stopPropagation()}
className="flex max-h-[88vh] w-full max-w-[860px] flex-col overflow-hidden rounded-2xl border border-line bg-surface-3 shadow-xl"
>
<header className="flex items-center justify-between border-b border-line px-5 py-3">
<div className="flex items-center gap-2">
<MonitorShareIcon className="h-4 w-4 text-accent" />
<h3 className="font-display text-sm font-semibold text-fg">
{t('app:call.share_dialog_title', { defaultValue: 'Bildschirm teilen' })}
</h3>
</div>
<button
type="button"
onClick={onClose}
aria-label={t('app:common.close', { defaultValue: 'Schließen' })}
className="flex h-8 w-8 cursor-pointer items-center justify-center rounded-lg text-fg-muted transition hover:bg-surface-2 hover:text-fg"
>
<XIcon className="h-4 w-4" />
</button>
</header>
<div className="flex-1 overflow-y-auto">
{sources === null ? (
<div className="flex h-40 items-center justify-center gap-2 text-sm text-fg-muted">
<SpinnerIcon className="h-4 w-4" />
<span>
{t('app:call.share_loading', { defaultValue: 'Lade Quellen…' })}
</span>
</div>
) : !hasAny ? (
<div className="flex flex-col items-center gap-2 px-6 py-10 text-center text-sm text-fg-muted">
<MonitorShareIcon className="h-6 w-6 opacity-60" />
<span>
{t('app:call.share_no_sources', {
defaultValue:
'Keine Quellen-Previews verfügbar. Klick auf „Teilen" öffnet den System-Picker.',
})}
</span>
</div>
) : (
<div className="space-y-5 p-5">
{screens.length > 0 && (
<SourceSection
title={t('app:call.share_screens', { defaultValue: 'Bildschirme' })}
sources={screens}
thumbnailUrls={thumbnailUrls}
selectedId={selectedId}
onSelect={setSelectedId}
/>
)}
{windows.length > 0 && (
<SourceSection
title={t('app:call.share_windows', { defaultValue: 'Fenster' })}
sources={windows}
thumbnailUrls={thumbnailUrls}
selectedId={selectedId}
onSelect={setSelectedId}
/>
)}
</div>
)}
</div>
<footer className="flex flex-col gap-3 border-t border-line bg-surface-2 px-5 py-3">
<div className="flex flex-wrap items-center justify-between gap-3">
<label className="flex cursor-pointer items-center gap-2 text-xs text-fg">
<span className="font-semibold uppercase tracking-wider text-fg-muted">
{t('app:call.share_quality', { defaultValue: 'Qualität' })}
</span>
<select
value={preset}
onChange={(e) => setPreset(e.target.value as ScreenSharePreset)}
className="rounded-md border border-line bg-surface-3 px-2 py-1 text-xs text-fg focus:border-accent focus:outline-none"
>
{PRESET_ORDER.map((p) => (
<option key={p} value={p}>
{getPresetParams(p).label} · {getPresetParams(p).bitrateKbps} kbps
</option>
))}
</select>
</label>
<label className="flex cursor-pointer items-center gap-2 text-xs text-fg">
<input
type="checkbox"
checked={includeAudio}
onChange={(e) => setIncludeAudio(e.target.checked)}
className="accent-accent"
/>
<span>
{t('app:call.share_system_audio', {
defaultValue: 'System-Sound mit übertragen',
})}
</span>
</label>
</div>
{error && (
<p role="alert" className="text-xs text-rose-600 dark:text-rose-300">
{error}
</p>
)}
<div className="flex items-center justify-end gap-2">
<button
type="button"
onClick={onClose}
className="cursor-pointer rounded-md border border-line bg-surface-3 px-3 py-2 text-sm text-fg transition hover:bg-surface-2"
>
{t('app:friends.action_cancel', { defaultValue: 'Abbrechen' })}
</button>
<button
type="button"
onClick={() => void handleStart()}
disabled={busy}
className="inline-flex cursor-pointer items-center gap-1.5 rounded-md bg-accent px-3 py-2 text-sm font-semibold text-accent-fg transition hover:brightness-110 disabled:cursor-not-allowed disabled:opacity-60"
>
{busy && <SpinnerIcon className="h-4 w-4" />}
<span>
{selectedId
? t('app:call.share_start', { defaultValue: 'Teilen' })
: t('app:call.share_pick_system', {
defaultValue: 'Ohne Auswahl weiter',
})}
</span>
</button>
</div>
</footer>
</div>
</div>
);
}
function SourceSection({
title,
sources,
thumbnailUrls,
selectedId,
onSelect,
}: {
title: string;
sources: ScreenSource[];
thumbnailUrls: Record<string, string>;
selectedId: string | null;
onSelect: (id: string) => void;
}) {
return (
<section>
<h4 className="mb-2 text-xs font-semibold uppercase tracking-wider text-fg-muted">
{title}
</h4>
<div className="grid grid-cols-2 gap-2.5 sm:grid-cols-3">
{sources.map((src) => (
<SourceCard
key={src.id}
source={src}
thumbnailUrl={thumbnailUrls[src.id] ?? null}
selected={selectedId === src.id}
onSelect={onSelect}
/>
))}
</div>
</section>
);
}
// Memoized so a thumbnail arriving for card B doesn't re-render card A.
// Keeps re-render work proportional to the number of updates instead of
// "whole grid on every update" — which was the main reason scrolling felt
// frozen during the initial thumbnail fan-in.
//
// The parent passes `onSelect(id)` rather than an inline `onClick`-arrow
// so the callback reference stays stable across renders; otherwise
// React.memo would always see a fresh function prop and re-render every
// card on every parent update.
const SourceCard = memo(function SourceCard({
source,
thumbnailUrl,
selected,
onSelect,
}: {
source: ScreenSource;
thumbnailUrl: string | null;
selected: boolean;
onSelect: (id: string) => void;
}) {
return (
<button
type="button"
onClick={() => onSelect(source.id)}
aria-pressed={selected}
title={source.name}
className={
'group flex cursor-pointer flex-col overflow-hidden rounded-lg border transition focus:outline-none focus-visible:ring-2 focus-visible:ring-accent/50 ' +
(selected
? 'border-accent ring-2 ring-accent/30'
: 'border-line hover:border-accent/70')
}
>
<div className="relative aspect-video w-full overflow-hidden bg-black">
{thumbnailUrl ? (
// decoding="async" keeps image decode off the main-thread paint
// step; loading="lazy" means cards outside the viewport don't
// ask the browser to decode until the user scrolls to them. The
// URL is a blob: URL backed by the ArrayBuffer Rust sent over
// IPC — no base64 decode, no data-URL parse, just direct bytes
// into the decoder.
<img
src={thumbnailUrl}
alt=""
decoding="async"
loading="lazy"
className="h-full w-full object-contain transition group-hover:brightness-110"
draggable={false}
/>
) : (
<div className="flex h-full w-full items-center justify-center bg-gradient-to-br from-surface-2 to-surface-3 text-fg-muted">
<MonitorShareIcon className="h-6 w-6 opacity-50" />
</div>
)}
</div>
<div className="truncate px-2.5 py-1.5 text-left text-xs font-medium text-fg">
{source.name}
</div>
</button>
);
});