Compare commits

...

6 Commits

Author SHA1 Message Date
byGalax adbdfaa2aa chore(desktop): release v0.11.2 2026-04-22 23:42:14 +02:00
byGalax 6f1e1a5f9a fix(call): raise xcap fps clamp 30 → 60
The fallback capture path was silently capping any 60fps preset to
30 because the hardcoded clamp never got updated when the 60fps presets
landed. Also syncs Cargo.lock that drifted against 0.11.0 metadata.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:40:05 +02:00
byGalax f9bbcdee47 perf(call): H.264 + contentHint + 30fps default for screen-share
The real cost in the "frame by frame" stutter wasn't the custom capture
path — it was LiveKit re-encoding via VP9 software with L3T3_KEY SVC
(three spatial × three temporal layers, all CPU). Switching the
per-publish codec to H.264 lets Chromium's hardware encoder take over
on Windows and sidesteps the SVC mode entirely (H.264 has no SVC).
Also pushes `contentHint = 'detail'` on the track — setScreenShareEnabled
does this internally, the manual publishTrack paths had been missing it,
which changes how the encoder allocates its frame budget for static UI
content.

Auto preset default framerate 60 → 30. 60fps desktop share burns three
full-res encodes per frame at sizes up to 4K; 30 is what getDisplayMedia
practically delivers anyway.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:37:22 +02:00
byGalax ccbef6d959 chore(desktop): release v0.11.1 2026-04-22 23:17:51 +02:00
byGalax 73ddeecfca chore(desktop): sync Cargo.lock to v0.11.0
Release script bumped Cargo.toml but cargo only refreshes the lock on
the next build. Aligning them so the lock doesn't drift across tags.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:15:54 +02:00
byGalax 727aced411 perf(call): prefer chromeMediaSource video over xcap native capture
The xcap native path does JPEG-encode-in-Rust → base64 → IPC → atob →
createImageBitmap → canvas.drawImage → canvas.captureStream → VP9 per
frame, all CPU-bound and mostly on the main thread — at 1080p30 that
lands well past one render quantum, producing visible frame-by-frame
stutter. chromeMediaSource+getUserMedia hands the capture to Chromium's
native desktop-capture backend and directly into the PeerConnection, so
it's the same path the OS picker uses and has no per-frame JS cost.

Reorders the capture attempts so chromeMediaSource is tried first; xcap
stays around as a fallback for WebView2 versions that reject the legacy
constraint. System audio still goes through WASAPI in both paths, since
getUserMedia's chromeMediaSource audio constraint throws AbortError on
Window captures — splitting the streams is what makes both work.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:14:21 +02:00
7 changed files with 145 additions and 160 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@chat-app/desktop", "name": "@chat-app/desktop",
"version": "0.11.0", "version": "0.11.2",
"private": true, "private": true,
"description": "Tauri v2 desktop client (Windows / macOS / Linux)", "description": "Tauri v2 desktop client (Windows / macOS / Linux)",
"type": "module", "type": "module",
+1 -1
View File
@@ -834,7 +834,7 @@ dependencies = [
[[package]] [[package]]
name = "chat-app-desktop" name = "chat-app-desktop"
version = "0.10.2" version = "0.11.1"
dependencies = [ dependencies = [
"base64 0.22.1", "base64 0.22.1",
"dryoc", "dryoc",
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "chat-app-desktop" name = "chat-app-desktop"
version = "0.11.0" version = "0.11.2"
description = "ChatApp desktop client" description = "ChatApp desktop client"
authors = ["Dennis"] authors = ["Dennis"]
edition = "2021" edition = "2021"
+1 -1
View File
@@ -62,7 +62,7 @@ pub fn start_screen_capture(
fps: u32, fps: u32,
channel: Channel<FramePayload>, channel: Channel<FramePayload>,
) -> Result<u32, String> { ) -> Result<u32, String> {
let clamped_fps = fps.clamp(5, 30); let clamped_fps = fps.clamp(5, 60);
let clamped_w = max_width.max(320).min(3840); let clamped_w = max_width.max(320).min(3840);
let clamped_h = max_height.max(180).min(2160); let clamped_h = max_height.max(180).min(2160);
+1 -1
View File
@@ -1,7 +1,7 @@
{ {
"$schema": "https://schema.tauri.app/config/2", "$schema": "https://schema.tauri.app/config/2",
"productName": "ChatApp", "productName": "ChatApp",
"version": "0.11.0", "version": "0.11.2",
"identifier": "com.meinname.chatapp", "identifier": "com.meinname.chatapp",
"build": { "build": {
"beforeDevCommand": "pnpm vite:dev", "beforeDevCommand": "pnpm vite:dev",
+139 -154
View File
@@ -1203,144 +1203,88 @@ export function CallProvider({ children }: { children: ReactNode }) {
const fps = framerateOverride ?? ssParams.framerate; const fps = framerateOverride ?? ssParams.framerate;
const sourceId = overrides?.sourceId ?? null; const sourceId = overrides?.sourceId ?? null;
// Native capture path — tried first when the user came in via our // Ordered capture paths when our custom picker supplied a sourceId:
// custom picker. xcap on the Rust side grabs video frames and, on // 1. chromeMediaSource video — hardware-accelerated path that
// Windows with "Mit System-Sound" on, the WASAPI loopback module // hands the frames directly to WebRTC, same mechanism the OS
// grabs the render endpoint. Both stream over Tauri channels into // picker uses internally. Fast, smooth, what users expect.
// tracks we publish directly to LiveKit — the OS picker never // 2. xcap native (JPEG → canvas → captureStream) — fallback for
// appears. If native audio fails on a platform that can't supply // WebView2 versions that reject the legacy chromeMediaSource
// it (non-Windows v1), we continue with video-only and log; the // constraint. Functional but CPU-heavy; the per-frame JPEG
// user still gets their direct-video share. // encode + base64 + createImageBitmap chain shows up as
// visible stutter on 1080p30.
// 3. OS picker (setScreenShareEnabled) — last resort when both
// native paths throw, and the only path when no sourceId was
// supplied.
//
// Audio (Windows only) always goes through the WASAPI loopback
// module — getUserMedia's chromeMediaSource audio constraint
// throws AbortError on Window captures, and pairing both into one
// call ended up blocking the video path entirely. Splitting them
// means the video path stays fast and the audio path stays
// reliable for any source type.
if (!sourceId) { if (!sourceId) {
console.info( console.info(
'screen-share: no sourceId supplied by picker, OS picker will open', 'screen-share: no sourceId supplied by picker, OS picker will open',
); );
} }
if (sourceId) {
// Publish system audio via WASAPI. Resolves to the audio publication
// + an ended-cleanup registration so whichever video path succeeds
// can wire the audio teardown onto its own video-ended handler.
const startWasapiAudio = async (
LkTrack: typeof import('livekit-client').Track,
): Promise<{
stopAudio: () => Promise<void>;
} | null> => {
if (!settings.includeSystemAudio) return null;
try { try {
console.info('screen-share: trying native capture path', { const audioHandle = await startSystemAudioCapture();
sourceId, nativeAudioCaptureRef.current = audioHandle;
fps, const audioMst = audioHandle.stream.getAudioTracks()[0];
includeSystemAudio: settings.includeSystemAudio, if (!audioMst) {
}); await audioHandle.stop();
const { Track: LkTrack } = await import('livekit-client'); nativeAudioCaptureRef.current = null;
const maxWidth = ssParams.dims?.width ?? 1920; return null;
const maxHeight = ssParams.dims?.height ?? 1080;
const handle = await startNativeCapture({
sourceId,
maxWidth,
maxHeight,
fps,
});
nativeCaptureRef.current = handle;
const videoMst = handle.stream.getVideoTracks()[0];
if (!videoMst) {
await handle.stop();
nativeCaptureRef.current = null;
throw new NativeCaptureUnavailable('canvas stream produced no video track');
} }
const videoPub = await lp.publishTrack(videoMst, { const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShare, source: LkTrack.Source.ScreenShareAudio,
videoCodec: 'vp9',
}); });
audioMst.addEventListener('ended', () => {
// Optional native audio. Failure here is non-fatal — the video
// pipeline is already running and bailing out would be worse
// UX than shipping a silent share. The warning surfaces the
// platform gap so the user knows why their audio is missing.
let audioHandle: SystemAudioHandle | null = null;
if (settings.includeSystemAudio) {
try {
audioHandle = await startSystemAudioCapture();
nativeAudioCaptureRef.current = audioHandle;
const audioMst = audioHandle.stream.getAudioTracks()[0];
if (audioMst) {
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
});
audioMst.addEventListener('ended', () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
const active = nativeAudioCaptureRef.current;
if (active && active.captureId === audioHandle!.captureId) {
nativeAudioCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
})();
});
}
} catch (err: unknown) {
console.warn(
'screen-share: native system-audio unavailable, sharing video only',
err instanceof Error ? err.message : err,
);
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
audioHandle = null;
}
}
// Canvas stream 'ended' fires on handle.stop() (we track.stop()
// each track) — chain unpublish + native teardown so one ended
// event cleans everything up regardless of who triggered it.
// Also tear down any paired audio capture so sound can't
// outlive the video share.
videoMst.addEventListener('ended', () => {
void (async () => { void (async () => {
try { try {
if (videoPub.track) await lp.unpublishTrack(videoPub.track); if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch { } catch {
/* already unpublished */ /* already unpublished */
} }
const active = nativeCaptureRef.current;
if (active && active.captureId === handle.captureId) {
nativeCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
const audioActive = nativeAudioCaptureRef.current;
if (audioActive) {
nativeAudioCaptureRef.current = null;
await audioActive.stop().catch(() => undefined);
}
setIsScreenSharing(false);
})(); })();
}); });
console.info('screen-share: native capture active', { const stopAudio = async () => {
audio: audioHandle != null, try {
}); if (audioPub.track) await lp.unpublishTrack(audioPub.track);
setIsScreenSharing(true); } catch {
return; /* already unpublished */
}
const active = nativeAudioCaptureRef.current;
if (active && active.captureId === audioHandle.captureId) {
nativeAudioCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
};
return { stopAudio };
} catch (err: unknown) { } catch (err: unknown) {
// Native path unavailable (non-Tauri runtime, source vanished, console.warn(
// first-frame timeout). Clean up any partial handle and fall 'screen-share: native system-audio unavailable, sharing video only',
// through to the getUserMedia / getDisplayMedia paths. err instanceof Error ? err.message : err,
if (nativeCaptureRef.current) { );
await nativeCaptureRef.current.stop().catch(() => undefined);
nativeCaptureRef.current = null;
}
if (nativeAudioCaptureRef.current) { if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined); await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null; nativeAudioCaptureRef.current = null;
} }
console.warn( return null;
'screen-share: native path failed, falling back',
err instanceof Error ? err.message : err,
);
} }
} };
// Direct-publish path when our custom picker supplied a Chromium- // ----- Path 1: chromeMediaSource (fast) -----
// format source id. Bypasses the OS picker so the user shares exactly
// the window/monitor they clicked in the grid. getUserMedia with the
// legacy chromeMediaSourceId constraint is not in the MediaStream
// spec but is honoured by Chromium / WebView2. If it throws we fall
// through to setScreenShareEnabled and let the OS picker run.
if (sourceId) { if (sourceId) {
try { try {
const { Track: LkTrack } = await import('livekit-client'); const { Track: LkTrack } = await import('livekit-client');
@@ -1357,36 +1301,28 @@ export function CallProvider({ children }: { children: ReactNode }) {
maxFrameRate: fps, maxFrameRate: fps,
}, },
} as unknown as MediaTrackConstraints; } as unknown as MediaTrackConstraints;
const audioConstraints: MediaTrackConstraints | false = settings.includeSystemAudio
? ({
mandatory: {
chromeMediaSource: 'desktop',
chromeMediaSourceId: sourceId,
},
} as unknown as MediaTrackConstraints)
: false;
const stream = await navigator.mediaDevices.getUserMedia({ const stream = await navigator.mediaDevices.getUserMedia({
audio: audioConstraints, audio: false,
video: videoConstraints, video: videoConstraints,
}); });
const videoMst = stream.getVideoTracks()[0]; const videoMst = stream.getVideoTracks()[0];
const audioMst = stream.getAudioTracks()[0];
if (!videoMst) { if (!videoMst) {
stream.getTracks().forEach((t) => t.stop()); stream.getTracks().forEach((t) => t.stop());
throw new Error('no video track from chromeMediaSource'); throw new Error('no video track from chromeMediaSource');
} }
// Pass raw MediaStreamTracks — `publishTrack` wraps them in the // H.264 lets Chromium's hardware encoder take over on Windows
// right Local*Track internally and the publishDefaults on the // (Intel QuickSync / NVENC / AMD VCE) — VP9 is software-only in
// Room handle VP9 codec + screenShareEncoding caps. Passing the // Chromium's WebRTC path and collapses under the L3T3_KEY
// raw tracks also sidesteps a type incompatibility between // scalability mode we set globally for camera. `contentHint` is
// livekit-client's Local*Track and our exactOptionalPropertyTypes // what setScreenShareEnabled sets internally; pushing it by hand
// setting. // here tells the encoder to weight sharpness over smoothness,
// which is right for UI/text-heavy screen content.
videoMst.contentHint = 'detail';
const videoPub = await lp.publishTrack(videoMst, { const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare, source: LkTrack.Source.ScreenShare,
videoCodec: 'vp9', videoCodec: 'h264',
}); });
// Stop the publish when the OS revokes capture (user hit the const audioRes = await startWasapiAudio(LkTrack);
// OS "Stop sharing" banner, or closed the window we were sharing).
videoMst.addEventListener('ended', () => { videoMst.addEventListener('ended', () => {
void (async () => { void (async () => {
try { try {
@@ -1394,32 +1330,81 @@ export function CallProvider({ children }: { children: ReactNode }) {
} catch { } catch {
/* already unpublished */ /* already unpublished */
} }
if (audioRes) await audioRes.stopAudio();
setIsScreenSharing(false); setIsScreenSharing(false);
})(); })();
}); });
if (audioMst) { console.info('screen-share: chromeMediaSource active', {
const audioPub = await lp.publishTrack(audioMst, { audio: audioRes != null,
source: LkTrack.Source.ScreenShareAudio, });
});
audioMst.addEventListener('ended', () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* ignore */
}
})();
});
}
setIsScreenSharing(true); setIsScreenSharing(true);
return; return;
} catch (err: unknown) { } catch (err: unknown) {
// WebView2 / browser rejected the legacy constraint. Fall through
// to the normal OS picker path below so the user still gets a
// working share instead of a hard error.
console.warn( console.warn(
'direct screen-share via chromeMediaSourceId failed; falling back to getDisplayMedia', 'screen-share: chromeMediaSource failed, trying native xcap',
err, err instanceof Error ? err.message : err,
);
}
}
// ----- Path 2: xcap native (CPU fallback) -----
if (sourceId) {
try {
const { Track: LkTrack } = await import('livekit-client');
const maxWidth = ssParams.dims?.width ?? 1920;
const maxHeight = ssParams.dims?.height ?? 1080;
const handle = await startNativeCapture({
sourceId,
maxWidth,
maxHeight,
fps,
});
nativeCaptureRef.current = handle;
const videoMst = handle.stream.getVideoTracks()[0];
if (!videoMst) {
await handle.stop();
nativeCaptureRef.current = null;
throw new NativeCaptureUnavailable('canvas stream produced no video track');
}
videoMst.contentHint = 'detail';
const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare,
videoCodec: 'h264',
});
const audioRes = await startWasapiAudio(LkTrack);
videoMst.addEventListener('ended', () => {
void (async () => {
try {
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
} catch {
/* already unpublished */
}
const active = nativeCaptureRef.current;
if (active && active.captureId === handle.captureId) {
nativeCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
if (audioRes) await audioRes.stopAudio();
setIsScreenSharing(false);
})();
});
console.info('screen-share: native xcap active', {
audio: audioRes != null,
});
setIsScreenSharing(true);
return;
} catch (err: unknown) {
if (nativeCaptureRef.current) {
await nativeCaptureRef.current.stop().catch(() => undefined);
nativeCaptureRef.current = null;
}
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
console.warn(
'screen-share: native xcap failed, falling back to OS picker',
err instanceof Error ? err.message : err,
); );
} }
} }
+1 -1
View File
@@ -46,7 +46,7 @@ export interface PresetParams {
} }
const PRESET_PARAMS: Record<ScreenSharePreset, PresetParams> = { const PRESET_PARAMS: Record<ScreenSharePreset, PresetParams> = {
auto: { dims: null, framerate: 60, bitrateKbps: 8000, label: 'Auto (Original)' }, auto: { dims: null, framerate: 30, bitrateKbps: 8000, label: 'Auto (Original)' },
'720p30': { '720p30': {
dims: { width: 1280, height: 720 }, dims: { width: 1280, height: 720 },
framerate: 30, framerate: 30,