Compare commits

...

5 Commits

Author SHA1 Message Date
byGalax 72d02e385f chore(desktop): release v0.11.3 2026-04-23 18:02:24 +02:00
byGalax c9b885b91c perf(call): screen-share hybrid — OS picker + WASAPI audio
Strip the sourceId-gated chromeMediaSource + xcap native capture paths
from startScreenShare and collapse to a single setScreenShareEnabled
call. Neither of the bypassed paths produced smooth frames in WebView2:
chromeMediaSource: 'desktop' is an extension-only Chromium constraint
and throws outside extension origins, and the xcap JPEG-over-IPC
fallback couldn't sustain 30fps at 1080p on a single main-thread.
setScreenShareEnabled goes through Chromium's native getDisplayMedia
capture, which is the only path that gets HW-accelerated frames into
the WebRTC encoder from WebView2.

Audio continues via the WASAPI loopback module — getDisplayMedia can't
grab system sound in WebView2 without desktop-capture entitlements
Chromium reserves for extensions. The audio track's teardown chains to
the ScreenShare video track's 'ended' event so the Windows stop-share
overlay kills both sides in lockstep.

ScreenSourcePicker is now a quality + audio chooser only; the
thumbnail grid disappears because custom source IDs don't round-trip
through WebView2, and a custom picker in front of the OS picker just
means the user picks twice.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-23 17:59:41 +02:00
byGalax adbdfaa2aa chore(desktop): release v0.11.2 2026-04-22 23:42:14 +02:00
byGalax 6f1e1a5f9a fix(call): raise xcap fps clamp 30 → 60
The fallback capture path was silently capping any 60fps preset to
30 because the hardcoded clamp never got updated when the 60fps presets
landed. Also syncs Cargo.lock that drifted against 0.11.0 metadata.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:40:05 +02:00
byGalax f9bbcdee47 perf(call): H.264 + contentHint + 30fps default for screen-share
The real cost in the "frame by frame" stutter wasn't the custom capture
path — it was LiveKit re-encoding via VP9 software with L3T3_KEY SVC
(three spatial × three temporal layers, all CPU). Switching the
per-publish codec to H.264 lets Chromium's hardware encoder take over
on Windows and sidesteps the SVC mode entirely (H.264 has no SVC).
Also pushes `contentHint = 'detail'` on the track — setScreenShareEnabled
does this internally, the manual publishTrack paths had been missing it,
which changes how the encoder allocates its frame budget for static UI
content.

Auto preset default framerate 60 → 30. 60fps desktop share burns three
full-res encodes per frame at sizes up to 4K; 30 is what getDisplayMedia
practically delivers anyway.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:37:22 +02:00
8 changed files with 154 additions and 585 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@chat-app/desktop",
"version": "0.11.1",
"version": "0.11.3",
"private": true,
"description": "Tauri v2 desktop client (Windows / macOS / Linux)",
"type": "module",
+1 -1
View File
@@ -834,7 +834,7 @@ dependencies = [
[[package]]
name = "chat-app-desktop"
version = "0.11.0"
version = "0.11.2"
dependencies = [
"base64 0.22.1",
"dryoc",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "chat-app-desktop"
version = "0.11.1"
version = "0.11.3"
description = "ChatApp desktop client"
authors = ["Dennis"]
edition = "2021"
+1 -1
View File
@@ -62,7 +62,7 @@ pub fn start_screen_capture(
fps: u32,
channel: Channel<FramePayload>,
) -> Result<u32, String> {
let clamped_fps = fps.clamp(5, 30);
let clamped_fps = fps.clamp(5, 60);
let clamped_w = max_width.max(320).min(3840);
let clamped_h = max_height.max(180).min(2160);
+1 -1
View File
@@ -1,7 +1,7 @@
{
"$schema": "https://schema.tauri.app/config/2",
"productName": "ChatApp",
"version": "0.11.1",
"version": "0.11.3",
"identifier": "com.meinname.chatapp",
"build": {
"beforeDevCommand": "pnpm vite:dev",
@@ -1,4 +1,4 @@
import { memo, startTransition, useEffect, useRef, useState } from 'react';
import { useState } from 'react';
import { useTranslation } from 'react-i18next';
import {
@@ -9,21 +9,14 @@ import {
type ScreenSharePreset,
updateScreenShareSettings,
} from '../lib/screenShareSettings';
import {
captureScreenSourceThumbnailBytes,
listScreenSources,
type ScreenSource,
} from '../lib/screenSources';
import { MonitorShareIcon, SpinnerIcon, XIcon } from './icons';
interface Props {
open: boolean;
onClose: () => void;
/** Parent handles the actual share start. `sourceId` is null when the user
* clicks "Teilen" without picking a specific source — fallback to the
* OS-level getDisplayMedia picker. */
/** Parent starts the actual share. The OS picker runs afterwards; the
* dialog only gathers quality + audio settings. */
onStart: (opts: {
sourceId: string | null;
preset: ScreenSharePreset;
displaySurface: DisplaySurfaceHint;
framerate: number | null;
@@ -31,143 +24,32 @@ interface Props {
}) => Promise<void>;
}
// Discord-style picker. Replaces the old form-field dialog with a thumbnail
// grid sourced from the Rust `enumerate_screen_sources` command. Clicking a
// thumbnail stashes its Chromium-format id; the parent then attempts a
// `chromeMediaSourceId`-constrained getUserMedia call. If WebView2 ignores
// the constraint (it may), the fallback OS picker still runs — but at least
// the user already saw + chose from a real preview first.
// Quality + audio chooser shown before "Bildschirm teilen" opens the OS
// source picker. We can't substitute sources in WebView2 — the
// `chromeMediaSource: 'desktop'` constraint is extension-only and
// ScreenCaptureStarting offers allow/deny, not source-injection — so a
// custom thumbnail grid would just double-pick (user picks here, then
// picks again in the OS dialog). Dropping the grid keeps the smooth
// Chromium-native capture pipeline and narrows the UX to the one
// decision that still matters at share-time: quality + audio.
export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
const { t } = useTranslation(['app']);
const initial = getScreenShareSettings();
const [sources, setSources] = useState<ScreenSource[] | null>(null);
// Thumbnails are kept in a separate state from the source list so an
// arriving thumbnail never creates a new `ScreenSource` object for
// unrelated cards — memo compares `thumbnailUrl` by string identity,
// so only the one card whose URL changes rerenders.
const [thumbnailUrls, setThumbnailUrls] = useState<Record<string, string>>({});
// All blob URLs we've handed out this session. Revoked on picker close
// so the native buffers they point at don't leak across opens.
const blobUrlsRef = useRef<string[]>([]);
const [selectedId, setSelectedId] = useState<string | null>(null);
const [preset, setPreset] = useState<ScreenSharePreset>(initial.preset);
const [includeAudio, setIncludeAudio] = useState<boolean>(initial.includeSystemAudio);
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
// Two-phase load: (1) fast list returns names + placeholders so the grid
// paints instantly, (2) capture thumbnails in a bounded worker-pool
// using the binary-IPC variant. Arriving bytes are wrapped in a Blob
// and exposed via URL.createObjectURL — no base64 on either side,
// which is the single biggest main-thread win compared to the old
// JSON-of-base64 flow. Combined with rAF-batched state updates, the
// picker stays responsive even on 20+ source enumerations.
useEffect(() => {
if (!open) {
// Revoke blob URLs created during the last session so native
// buffers don't linger after close.
for (const url of blobUrlsRef.current) URL.revokeObjectURL(url);
blobUrlsRef.current = [];
setSources(null);
setThumbnailUrls({});
setSelectedId(null);
setError(null);
return;
}
let cancelled = false;
// Coalesce thumbnail arrivals within a single animation frame into
// one setState — cuts re-renders from O(N) to O(frames) during the
// initial fan-in and prevents consecutive 10-30 ms long tasks from
// stacking in one frame.
//
// Previously `batch` was aliased to `pendingUrls` and then we cleared
// pendingUrls via `delete` — which emptied batch too (same reference)
// and every flush ended up spreading nothing into the state. Clone
// first, then clear, so the batch keeps its entries.
let pendingUrls: Record<string, string> = {};
let rafScheduled = false;
const flush = () => {
rafScheduled = false;
const batch = pendingUrls;
if (Object.keys(batch).length === 0) return;
pendingUrls = {};
startTransition(() => {
setThumbnailUrls((prev) => ({ ...prev, ...batch }));
});
};
const queueUrl = (id: string, url: string) => {
pendingUrls[id] = url;
if (!rafScheduled) {
rafScheduled = true;
requestAnimationFrame(flush);
}
};
// Concurrency 2: Windows GDI BitBlt / PrintWindow contends for the
// desktop compositor, so 4+ parallel captures stutter the whole Tauri
// window. 2 in parallel keeps the compositor breathing.
const CONCURRENCY = 2;
void (async () => {
const list = await listScreenSources();
if (cancelled) return;
setSources(list);
const queue = [...list];
const pickOne = (src: typeof list[number]) => {
void (async () => {
const blob = await captureScreenSourceThumbnailBytes(src.id);
if (cancelled) {
// Edge case: picker closed while this request was in flight.
// blob may still exist; nothing holds a URL to it, so it GCs.
return;
}
if (blob) {
const url = URL.createObjectURL(blob);
blobUrlsRef.current.push(url);
queueUrl(src.id, url);
}
const nextSrc = queue.shift();
if (nextSrc) pickOne(nextSrc);
})();
};
for (let i = 0; i < Math.min(CONCURRENCY, queue.length); i++) {
const s = queue.shift();
if (s) pickOne(s);
}
})();
return () => {
cancelled = true;
};
}, [open]);
if (!open) return null;
const screens = sources?.filter((s) => s.kind === 'screen') ?? [];
const windows = sources?.filter((s) => s.kind === 'window') ?? [];
const hasAny = (sources?.length ?? 0) > 0;
async function handleStart(): Promise<void> {
setBusy(true);
setError(null);
try {
// Persist user's audio + preset choice so subsequent shares start with
// the same prefs when they skip the picker. The picker itself stays
// as the entry for future starts (right-click on share button also
// opens it — see InCallPanel wiring).
updateScreenShareSettings({ preset, includeSystemAudio: includeAudio });
// Infer a displaySurface hint from the selection so the fallback OS
// picker jumps to the right tab when our direct-publish path is
// rejected by WebView2.
const selected = sources?.find((s) => s.id === selectedId) ?? null;
const hint: DisplaySurfaceHint =
selected?.kind === 'screen'
? 'monitor'
: selected?.kind === 'window'
? 'window'
: null;
await onStart({
sourceId: selected?.id ?? null,
preset,
displaySurface: hint,
displaySurface: null,
framerate: null,
includeAudio,
});
@@ -189,7 +71,7 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
>
<div
onClick={(e) => e.stopPropagation()}
className="flex max-h-[88vh] w-full max-w-[860px] flex-col overflow-hidden rounded-2xl border border-line bg-surface-3 shadow-xl"
className="flex w-full max-w-[420px] flex-col overflow-hidden rounded-2xl border border-line bg-surface-3 shadow-xl"
>
<header className="flex items-center justify-between border-b border-line px-5 py-3">
<div className="flex items-center gap-2">
@@ -208,206 +90,71 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
</button>
</header>
<div className="flex-1 overflow-y-auto">
{sources === null ? (
<div className="flex h-40 items-center justify-center gap-2 text-sm text-fg-muted">
<SpinnerIcon className="h-4 w-4" />
<span>
{t('app:call.share_loading', { defaultValue: 'Lade Quellen…' })}
</span>
</div>
) : !hasAny ? (
<div className="flex flex-col items-center gap-2 px-6 py-10 text-center text-sm text-fg-muted">
<MonitorShareIcon className="h-6 w-6 opacity-60" />
<span>
{t('app:call.share_no_sources', {
defaultValue:
'Keine Quellen-Previews verfügbar. Klick auf „Teilen" öffnet den System-Picker.',
})}
</span>
</div>
) : (
<div className="space-y-5 p-5">
{screens.length > 0 && (
<SourceSection
title={t('app:call.share_screens', { defaultValue: 'Bildschirme' })}
sources={screens}
thumbnailUrls={thumbnailUrls}
selectedId={selectedId}
onSelect={setSelectedId}
/>
)}
{windows.length > 0 && (
<SourceSection
title={t('app:call.share_windows', { defaultValue: 'Fenster' })}
sources={windows}
thumbnailUrls={thumbnailUrls}
selectedId={selectedId}
onSelect={setSelectedId}
/>
)}
</div>
)}
</div>
<div className="flex flex-col gap-4 px-5 py-5">
<p className="text-xs text-fg-muted">
{t('app:call.share_os_picker_hint', {
defaultValue:
'Nach dem Klick auf „Teilen" wählst du im System-Dialog den Bildschirm oder das Fenster aus.',
})}
</p>
<label className="flex items-center justify-between gap-3 text-xs text-fg">
<span className="font-semibold uppercase tracking-wider text-fg-muted">
{t('app:call.share_quality', { defaultValue: 'Qualität' })}
</span>
<select
value={preset}
onChange={(e) => setPreset(e.target.value as ScreenSharePreset)}
className="flex-1 rounded-md border border-line bg-surface-3 px-2 py-1 text-xs text-fg focus:border-accent focus:outline-none"
>
{PRESET_ORDER.map((p) => (
<option key={p} value={p}>
{getPresetParams(p).label} · {getPresetParams(p).bitrateKbps} kbps
</option>
))}
</select>
</label>
<label className="flex cursor-pointer items-center gap-2 text-xs text-fg">
<input
type="checkbox"
checked={includeAudio}
onChange={(e) => setIncludeAudio(e.target.checked)}
className="accent-accent"
/>
<span>
{t('app:call.share_system_audio', {
defaultValue: 'System-Sound mit übertragen',
})}
</span>
</label>
<footer className="flex flex-col gap-3 border-t border-line bg-surface-2 px-5 py-3">
<div className="flex flex-wrap items-center justify-between gap-3">
<label className="flex cursor-pointer items-center gap-2 text-xs text-fg">
<span className="font-semibold uppercase tracking-wider text-fg-muted">
{t('app:call.share_quality', { defaultValue: 'Qualität' })}
</span>
<select
value={preset}
onChange={(e) => setPreset(e.target.value as ScreenSharePreset)}
className="rounded-md border border-line bg-surface-3 px-2 py-1 text-xs text-fg focus:border-accent focus:outline-none"
>
{PRESET_ORDER.map((p) => (
<option key={p} value={p}>
{getPresetParams(p).label} · {getPresetParams(p).bitrateKbps} kbps
</option>
))}
</select>
</label>
<label className="flex cursor-pointer items-center gap-2 text-xs text-fg">
<input
type="checkbox"
checked={includeAudio}
onChange={(e) => setIncludeAudio(e.target.checked)}
className="accent-accent"
/>
<span>
{t('app:call.share_system_audio', {
defaultValue: 'System-Sound mit übertragen',
})}
</span>
</label>
</div>
{error && (
<p role="alert" className="text-xs text-rose-600 dark:text-rose-300">
{error}
</p>
)}
<div className="flex items-center justify-end gap-2">
<button
type="button"
onClick={onClose}
className="cursor-pointer rounded-md border border-line bg-surface-3 px-3 py-2 text-sm text-fg transition hover:bg-surface-2"
>
{t('app:friends.action_cancel', { defaultValue: 'Abbrechen' })}
</button>
<button
type="button"
onClick={() => void handleStart()}
disabled={busy}
className="inline-flex cursor-pointer items-center gap-1.5 rounded-md bg-accent px-3 py-2 text-sm font-semibold text-accent-fg transition hover:brightness-110 disabled:cursor-not-allowed disabled:opacity-60"
>
{busy && <SpinnerIcon className="h-4 w-4" />}
<span>
{selectedId
? t('app:call.share_start', { defaultValue: 'Teilen' })
: t('app:call.share_pick_system', {
defaultValue: 'Ohne Auswahl weiter',
})}
</span>
</button>
</div>
</div>
<footer className="flex items-center justify-end gap-2 border-t border-line bg-surface-2 px-5 py-3">
<button
type="button"
onClick={onClose}
className="cursor-pointer rounded-md border border-line bg-surface-3 px-3 py-2 text-sm text-fg transition hover:bg-surface-2"
>
{t('app:friends.action_cancel', { defaultValue: 'Abbrechen' })}
</button>
<button
type="button"
onClick={() => void handleStart()}
disabled={busy}
className="inline-flex cursor-pointer items-center gap-1.5 rounded-md bg-accent px-3 py-2 text-sm font-semibold text-accent-fg transition hover:brightness-110 disabled:cursor-not-allowed disabled:opacity-60"
>
{busy && <SpinnerIcon className="h-4 w-4" />}
<span>{t('app:call.share_start', { defaultValue: 'Teilen' })}</span>
</button>
</footer>
</div>
</div>
);
}
function SourceSection({
title,
sources,
thumbnailUrls,
selectedId,
onSelect,
}: {
title: string;
sources: ScreenSource[];
thumbnailUrls: Record<string, string>;
selectedId: string | null;
onSelect: (id: string) => void;
}) {
return (
<section>
<h4 className="mb-2 text-xs font-semibold uppercase tracking-wider text-fg-muted">
{title}
</h4>
<div className="grid grid-cols-2 gap-2.5 sm:grid-cols-3">
{sources.map((src) => (
<SourceCard
key={src.id}
source={src}
thumbnailUrl={thumbnailUrls[src.id] ?? null}
selected={selectedId === src.id}
onSelect={onSelect}
/>
))}
</div>
</section>
);
}
// Memoized so a thumbnail arriving for card B doesn't re-render card A.
// Keeps re-render work proportional to the number of updates instead of
// "whole grid on every update" — which was the main reason scrolling felt
// frozen during the initial thumbnail fan-in.
//
// The parent passes `onSelect(id)` rather than an inline `onClick`-arrow
// so the callback reference stays stable across renders; otherwise
// React.memo would always see a fresh function prop and re-render every
// card on every parent update.
const SourceCard = memo(function SourceCard({
source,
thumbnailUrl,
selected,
onSelect,
}: {
source: ScreenSource;
thumbnailUrl: string | null;
selected: boolean;
onSelect: (id: string) => void;
}) {
return (
<button
type="button"
onClick={() => onSelect(source.id)}
aria-pressed={selected}
title={source.name}
className={
'group flex cursor-pointer flex-col overflow-hidden rounded-lg border transition focus:outline-none focus-visible:ring-2 focus-visible:ring-accent/50 ' +
(selected
? 'border-accent ring-2 ring-accent/30'
: 'border-line hover:border-accent/70')
}
>
<div className="relative aspect-video w-full overflow-hidden bg-black">
{thumbnailUrl ? (
// decoding="async" keeps image decode off the main-thread paint
// step; loading="lazy" means cards outside the viewport don't
// ask the browser to decode until the user scrolls to them. The
// URL is a blob: URL backed by the ArrayBuffer Rust sent over
// IPC — no base64 decode, no data-URL parse, just direct bytes
// into the decoder.
<img
src={thumbnailUrl}
alt=""
decoding="async"
loading="lazy"
className="h-full w-full object-contain transition group-hover:brightness-110"
draggable={false}
/>
) : (
<div className="flex h-full w-full items-center justify-center bg-gradient-to-br from-surface-2 to-surface-3 text-fg-muted">
<MonitorShareIcon className="h-6 w-6 opacity-50" />
</div>
)}
</div>
<div className="truncate px-2.5 py-1.5 text-left text-xs font-medium text-fg">
{source.name}
</div>
</button>
);
});
+78 -256
View File
@@ -75,11 +75,6 @@ import {
setAllPipelinesSinkId,
setPipelineGain,
} from '../lib/remoteAudioPipelines';
import {
type NativeCaptureHandle,
NativeCaptureUnavailable,
startNativeCapture,
} from '../lib/screenCapture';
import {
type SystemAudioHandle,
startSystemAudioCapture,
@@ -304,15 +299,10 @@ export function CallProvider({ children }: { children: ReactNode }) {
// to `connected` after a few seconds so the UI doesn't hang in "Verbinde…"
// indefinitely. The solo-timeout will then cleanly close if nobody arrives.
const joinFallbackTimerRef = useRef<number | null>(null);
// Active native screen-capture handle (Rust side). Set when
// startScreenShare takes the xcap path; cleared on stopScreenShare or
// on the canvas track's 'ended' event. Not kept in React state because
// it never feeds into a render.
const nativeCaptureRef = useRef<NativeCaptureHandle | null>(null);
// Matching handle for the Windows-only WASAPI system-audio capture.
// Lives in lockstep with the video handle above when the user picks
// "Mit System-Sound"; teardown is wired so that stopping either track
// also stops the other, so stale audio can't outlive the video share.
// Handle for the Windows-only WASAPI system-audio capture. Lives in
// lockstep with the ScreenShare video track published by
// setScreenShareEnabled; teardown is chained to the video track's
// `ended` event so stale audio can never outlive the video share.
const nativeAudioCaptureRef = useRef<SystemAudioHandle | null>(null);
const roomRef = useRef<Room | null>(null);
// Web Audio graph that mixes live mic + soundboard sources into a single
@@ -488,13 +478,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
/* ignore */
}
}
// Stop any lingering native screen-capture thread so we don't leak
// Rust threads when the call ends mid-share.
if (nativeCaptureRef.current) {
const h = nativeCaptureRef.current;
nativeCaptureRef.current = null;
await h.stop().catch(() => undefined);
}
if (nativeAudioCaptureRef.current) {
const h = nativeAudioCaptureRef.current;
nativeAudioCaptureRef.current = null;
@@ -729,6 +712,11 @@ export function CallProvider({ children }: { children: ReactNode }) {
r.on(RoomEvent.ParticipantConnected, () => {
setRemoteParticipants(Array.from(r.remoteParticipants.values()));
if (presenceRef.current !== 'dnd') void playJoinBeep();
// Rejoin während wir schon `connected` sind: markConnectedIfReady
// returnt früh und würde den Solo-Timer nicht cancellen. Hier
// unbedingt clearen, sonst kickt der 5-Minuten-Timer obwohl der
// andere Peer längst wieder im Raum ist.
clearSoloTimer();
markConnectedIfReady(r, conversationId, mediaKind, callId);
});
@@ -1173,11 +1161,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
preset: ScreenSharePreset;
displaySurface: DisplaySurfaceHint;
framerate: number | null;
/** Chromium-format source id from our custom picker. When set, we
* try to capture that exact source via `chromeMediaSourceId`
* instead of the OS-level getDisplayMedia picker. Falls back to
* getDisplayMedia if WebView2 rejects the constraint. */
sourceId: string | null;
}>,
) => {
const r = roomRef.current;
@@ -1185,8 +1168,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
const lp = r.localParticipant;
if (lp.isScreenShareEnabled) return;
// Persist the user's choice so subsequent shares use the same config
// without re-opening the picker unless they want to change something.
const settings = getScreenShareSettings();
const preset = overrides?.preset ?? settings.preset;
const displaySurface =
@@ -1201,211 +1182,17 @@ export function CallProvider({ children }: { children: ReactNode }) {
const ssParams = getPresetParams(preset);
const fps = framerateOverride ?? ssParams.framerate;
const sourceId = overrides?.sourceId ?? null;
// Ordered capture paths when our custom picker supplied a sourceId:
// 1. chromeMediaSource video — hardware-accelerated path that
// hands the frames directly to WebRTC, same mechanism the OS
// picker uses internally. Fast, smooth, what users expect.
// 2. xcap native (JPEG → canvas → captureStream) — fallback for
// WebView2 versions that reject the legacy chromeMediaSource
// constraint. Functional but CPU-heavy; the per-frame JPEG
// encode + base64 + createImageBitmap chain shows up as
// visible stutter on 1080p30.
// 3. OS picker (setScreenShareEnabled) — last resort when both
// native paths throw, and the only path when no sourceId was
// supplied.
//
// Audio (Windows only) always goes through the WASAPI loopback
// module — getUserMedia's chromeMediaSource audio constraint
// throws AbortError on Window captures, and pairing both into one
// call ended up blocking the video path entirely. Splitting them
// means the video path stays fast and the audio path stays
// reliable for any source type.
if (!sourceId) {
console.info(
'screen-share: no sourceId supplied by picker, OS picker will open',
);
}
// Publish system audio via WASAPI. Resolves to the audio publication
// + an ended-cleanup registration so whichever video path succeeds
// can wire the audio teardown onto its own video-ended handler.
const startWasapiAudio = async (
LkTrack: typeof import('livekit-client').Track,
): Promise<{
stopAudio: () => Promise<void>;
} | null> => {
if (!settings.includeSystemAudio) return null;
try {
const audioHandle = await startSystemAudioCapture();
nativeAudioCaptureRef.current = audioHandle;
const audioMst = audioHandle.stream.getAudioTracks()[0];
if (!audioMst) {
await audioHandle.stop();
nativeAudioCaptureRef.current = null;
return null;
}
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
});
audioMst.addEventListener('ended', () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
})();
});
const stopAudio = async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
const active = nativeAudioCaptureRef.current;
if (active && active.captureId === audioHandle.captureId) {
nativeAudioCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
};
return { stopAudio };
} catch (err: unknown) {
console.warn(
'screen-share: native system-audio unavailable, sharing video only',
err instanceof Error ? err.message : err,
);
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
return null;
}
};
// ----- Path 1: chromeMediaSource (fast) -----
if (sourceId) {
try {
const { Track: LkTrack } = await import('livekit-client');
const maxWidth = ssParams.dims?.width ?? 3840;
const maxHeight = ssParams.dims?.height ?? 2160;
// Cast chains: browsers expose the legacy constraint via
// `MediaTrackConstraints.mandatory` which isn't in lib.dom.
const videoConstraints = {
mandatory: {
chromeMediaSource: 'desktop',
chromeMediaSourceId: sourceId,
maxWidth,
maxHeight,
maxFrameRate: fps,
},
} as unknown as MediaTrackConstraints;
const stream = await navigator.mediaDevices.getUserMedia({
audio: false,
video: videoConstraints,
});
const videoMst = stream.getVideoTracks()[0];
if (!videoMst) {
stream.getTracks().forEach((t) => t.stop());
throw new Error('no video track from chromeMediaSource');
}
const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare,
videoCodec: 'vp9',
});
const audioRes = await startWasapiAudio(LkTrack);
videoMst.addEventListener('ended', () => {
void (async () => {
try {
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
} catch {
/* already unpublished */
}
if (audioRes) await audioRes.stopAudio();
setIsScreenSharing(false);
})();
});
console.info('screen-share: chromeMediaSource active', {
audio: audioRes != null,
});
setIsScreenSharing(true);
return;
} catch (err: unknown) {
console.warn(
'screen-share: chromeMediaSource failed, trying native xcap',
err instanceof Error ? err.message : err,
);
}
}
// ----- Path 2: xcap native (CPU fallback) -----
if (sourceId) {
try {
const { Track: LkTrack } = await import('livekit-client');
const maxWidth = ssParams.dims?.width ?? 1920;
const maxHeight = ssParams.dims?.height ?? 1080;
const handle = await startNativeCapture({
sourceId,
maxWidth,
maxHeight,
fps,
});
nativeCaptureRef.current = handle;
const videoMst = handle.stream.getVideoTracks()[0];
if (!videoMst) {
await handle.stop();
nativeCaptureRef.current = null;
throw new NativeCaptureUnavailable('canvas stream produced no video track');
}
const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare,
videoCodec: 'vp9',
});
const audioRes = await startWasapiAudio(LkTrack);
videoMst.addEventListener('ended', () => {
void (async () => {
try {
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
} catch {
/* already unpublished */
}
const active = nativeCaptureRef.current;
if (active && active.captureId === handle.captureId) {
nativeCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
if (audioRes) await audioRes.stopAudio();
setIsScreenSharing(false);
})();
});
console.info('screen-share: native xcap active', {
audio: audioRes != null,
});
setIsScreenSharing(true);
return;
} catch (err: unknown) {
if (nativeCaptureRef.current) {
await nativeCaptureRef.current.stop().catch(() => undefined);
nativeCaptureRef.current = null;
}
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
console.warn(
'screen-share: native xcap failed, falling back to OS picker',
err instanceof Error ? err.message : err,
);
}
}
// Single capture path: setScreenShareEnabled → OS picker → Chromium's
// hardware-accelerated capture feeds WebRTC directly. In WebView2
// the extension-only `chromeMediaSource: 'desktop'` constraint and
// our xcap JPEG-over-IPC pipeline were both too slow for smooth
// 30fps, so the OS picker is the only route to the fast pipeline.
// Audio is handled separately below via WASAPI because
// getDisplayMedia can't grab system audio in WebView2.
try {
await lp.setScreenShareEnabled(true, {
// "Go live" mode — capture system audio alongside the screen when
// the user opted in. On hosts that can't fulfil the request the
// browser quietly drops it; peers just get video-only, no error.
audio: settings.includeSystemAudio,
audio: false,
...(ssParams.dims
? {
resolution: {
@@ -1421,9 +1208,6 @@ export function CallProvider({ children }: { children: ReactNode }) {
frameRate: fps,
},
}),
// Hints the OS picker to pre-filter by source kind. `null` = no
// filter (show both). Cast because TS lib.dom doesn't know the
// field yet on all branches.
...(displaySurface
? ({ displaySurface } as { displaySurface: DisplaySurfaceHint })
: {}),
@@ -1432,24 +1216,67 @@ export function CallProvider({ children }: { children: ReactNode }) {
setIsScreenSharing(true);
} catch (err: unknown) {
console.error('setScreenShareEnabled failed', err);
return;
}
if (!settings.includeSystemAudio) return;
// WASAPI system-audio: publish as a separate ScreenShareAudio track.
// Chain its teardown onto the ScreenShare video track's `ended`
// event so the Windows "Stop sharing" overlay kills audio alongside
// video; the audio track's own `ended` handler covers the case
// where WASAPI dies by itself.
try {
const { Track: LkTrack } = await import('livekit-client');
const audioHandle = await startSystemAudioCapture();
nativeAudioCaptureRef.current = audioHandle;
const audioMst = audioHandle.stream.getAudioTracks()[0];
if (!audioMst) {
await audioHandle.stop();
nativeAudioCaptureRef.current = null;
return;
}
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
});
let videoMst: MediaStreamTrack | null = null;
for (const pub of lp.videoTrackPublications.values()) {
if (pub.source === LkTrack.Source.ScreenShare && pub.track) {
videoMst = pub.track.mediaStreamTrack;
break;
}
}
const teardown = () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
const active = nativeAudioCaptureRef.current;
if (active && active.captureId === audioHandle.captureId) {
nativeAudioCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
})();
};
audioMst.addEventListener('ended', teardown);
if (videoMst) videoMst.addEventListener('ended', teardown);
} catch (err: unknown) {
console.warn(
'screen-share: native system-audio unavailable, sharing video only',
err instanceof Error ? err.message : err,
);
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
}
},
[],
);
const stopScreenShare = useCallback(async () => {
// Native path first — stopping the handle kills the canvas track,
// which fires 'ended' on the MediaStreamTrack, which the start-handler
// already listens to for unpublishing and flipping isScreenSharing.
if (nativeCaptureRef.current) {
const h = nativeCaptureRef.current;
nativeCaptureRef.current = null;
try {
await h.stop();
} catch (err: unknown) {
console.warn('native capture stop failed', err);
}
}
if (nativeAudioCaptureRef.current) {
const h = nativeAudioCaptureRef.current;
nativeAudioCaptureRef.current = null;
@@ -1462,18 +1289,13 @@ export function CallProvider({ children }: { children: ReactNode }) {
const r = roomRef.current;
if (!r) return;
const lp = r.localParticipant;
// Unpublish any manually-published ScreenShare/ScreenShareAudio tracks
// (from the chromeMediaSourceId fallback path). setScreenShareEnabled
// only manages LK's own internally-captured tracks.
const toUnpublish: import('livekit-client').LocalTrackPublication[] = [];
for (const pub of lp.videoTrackPublications.values()) {
if (pub.source === Track.Source.ScreenShare && pub.track) toUnpublish.push(pub);
}
// Unpublish any ScreenShareAudio track we manually published alongside
// setScreenShareEnabled. LiveKit's setScreenShareEnabled(false) only
// drops the video track it captured itself.
for (const pub of lp.audioTrackPublications.values()) {
if (pub.source === Track.Source.ScreenShareAudio && pub.track) toUnpublish.push(pub);
}
for (const pub of toUnpublish) {
if (pub.track) await lp.unpublishTrack(pub.track).catch(() => undefined);
if (pub.source === Track.Source.ScreenShareAudio && pub.track) {
await lp.unpublishTrack(pub.track).catch(() => undefined);
}
}
if (lp.isScreenShareEnabled) {
try {
+1 -1
View File
@@ -46,7 +46,7 @@ export interface PresetParams {
}
const PRESET_PARAMS: Record<ScreenSharePreset, PresetParams> = {
auto: { dims: null, framerate: 60, bitrateKbps: 8000, label: 'Auto (Original)' },
auto: { dims: null, framerate: 30, bitrateKbps: 8000, label: 'Auto (Original)' },
'720p30': {
dims: { width: 1280, height: 720 },
framerate: 30,