Compare commits

...

6 Commits

Author SHA1 Message Date
byGalax adbdfaa2aa chore(desktop): release v0.11.2 2026-04-22 23:42:14 +02:00
byGalax 6f1e1a5f9a fix(call): raise xcap fps clamp 30 → 60
The fallback capture path was silently capping any 60fps preset to
30 because the hardcoded clamp never got updated when the 60fps presets
landed. Also syncs Cargo.lock that drifted against 0.11.0 metadata.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:40:05 +02:00
byGalax f9bbcdee47 perf(call): H.264 + contentHint + 30fps default for screen-share
The real cost in the "frame by frame" stutter wasn't the custom capture
path — it was LiveKit re-encoding via VP9 software with L3T3_KEY SVC
(three spatial × three temporal layers, all CPU). Switching the
per-publish codec to H.264 lets Chromium's hardware encoder take over
on Windows and sidesteps the SVC mode entirely (H.264 has no SVC).
Also pushes `contentHint = 'detail'` on the track — setScreenShareEnabled
does this internally, the manual publishTrack paths had been missing it,
which changes how the encoder allocates its frame budget for static UI
content.

Auto preset default framerate 60 → 30. 60fps desktop share burns three
full-res encodes per frame at sizes up to 4K; 30 is what getDisplayMedia
practically delivers anyway.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:37:22 +02:00
byGalax ccbef6d959 chore(desktop): release v0.11.1 2026-04-22 23:17:51 +02:00
byGalax 73ddeecfca chore(desktop): sync Cargo.lock to v0.11.0
Release script bumped Cargo.toml but cargo only refreshes the lock on
the next build. Aligning them so the lock doesn't drift across tags.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:15:54 +02:00
byGalax 727aced411 perf(call): prefer chromeMediaSource video over xcap native capture
The xcap native path does JPEG-encode-in-Rust → base64 → IPC → atob →
createImageBitmap → canvas.drawImage → canvas.captureStream → VP9 per
frame, all CPU-bound and mostly on the main thread — at 1080p30 that
lands well past one render quantum, producing visible frame-by-frame
stutter. chromeMediaSource+getUserMedia hands the capture to Chromium's
native desktop-capture backend and directly into the PeerConnection, so
it's the same path the OS picker uses and has no per-frame JS cost.

Reorders the capture attempts so chromeMediaSource is tried first; xcap
stays around as a fallback for WebView2 versions that reject the legacy
constraint. System audio still goes through WASAPI in both paths, since
getUserMedia's chromeMediaSource audio constraint throws AbortError on
Window captures — splitting the streams is what makes both work.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:14:21 +02:00
7 changed files with 145 additions and 160 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@chat-app/desktop",
"version": "0.11.0",
"version": "0.11.2",
"private": true,
"description": "Tauri v2 desktop client (Windows / macOS / Linux)",
"type": "module",
+1 -1
View File
@@ -834,7 +834,7 @@ dependencies = [
[[package]]
name = "chat-app-desktop"
version = "0.10.2"
version = "0.11.1"
dependencies = [
"base64 0.22.1",
"dryoc",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "chat-app-desktop"
version = "0.11.0"
version = "0.11.2"
description = "ChatApp desktop client"
authors = ["Dennis"]
edition = "2021"
+1 -1
View File
@@ -62,7 +62,7 @@ pub fn start_screen_capture(
fps: u32,
channel: Channel<FramePayload>,
) -> Result<u32, String> {
let clamped_fps = fps.clamp(5, 30);
let clamped_fps = fps.clamp(5, 60);
let clamped_w = max_width.max(320).min(3840);
let clamped_h = max_height.max(180).min(2160);
+1 -1
View File
@@ -1,7 +1,7 @@
{
"$schema": "https://schema.tauri.app/config/2",
"productName": "ChatApp",
"version": "0.11.0",
"version": "0.11.2",
"identifier": "com.meinname.chatapp",
"build": {
"beforeDevCommand": "pnpm vite:dev",
+126 -141
View File
@@ -1203,58 +1203,49 @@ export function CallProvider({ children }: { children: ReactNode }) {
const fps = framerateOverride ?? ssParams.framerate;
const sourceId = overrides?.sourceId ?? null;
// Native capture path — tried first when the user came in via our
// custom picker. xcap on the Rust side grabs video frames and, on
// Windows with "Mit System-Sound" on, the WASAPI loopback module
// grabs the render endpoint. Both stream over Tauri channels into
// tracks we publish directly to LiveKit — the OS picker never
// appears. If native audio fails on a platform that can't supply
// it (non-Windows v1), we continue with video-only and log; the
// user still gets their direct-video share.
// Ordered capture paths when our custom picker supplied a sourceId:
// 1. chromeMediaSource video — hardware-accelerated path that
// hands the frames directly to WebRTC, same mechanism the OS
// picker uses internally. Fast, smooth, what users expect.
// 2. xcap native (JPEG → canvas → captureStream) — fallback for
// WebView2 versions that reject the legacy chromeMediaSource
// constraint. Functional but CPU-heavy; the per-frame JPEG
// encode + base64 + createImageBitmap chain shows up as
// visible stutter on 1080p30.
// 3. OS picker (setScreenShareEnabled) — last resort when both
// native paths throw, and the only path when no sourceId was
// supplied.
//
// Audio (Windows only) always goes through the WASAPI loopback
// module — getUserMedia's chromeMediaSource audio constraint
// throws AbortError on Window captures, and pairing both into one
// call ended up blocking the video path entirely. Splitting them
// means the video path stays fast and the audio path stays
// reliable for any source type.
if (!sourceId) {
console.info(
'screen-share: no sourceId supplied by picker, OS picker will open',
);
}
if (sourceId) {
try {
console.info('screen-share: trying native capture path', {
sourceId,
fps,
includeSystemAudio: settings.includeSystemAudio,
});
const { Track: LkTrack } = await import('livekit-client');
const maxWidth = ssParams.dims?.width ?? 1920;
const maxHeight = ssParams.dims?.height ?? 1080;
const handle = await startNativeCapture({
sourceId,
maxWidth,
maxHeight,
fps,
});
nativeCaptureRef.current = handle;
const videoMst = handle.stream.getVideoTracks()[0];
if (!videoMst) {
await handle.stop();
nativeCaptureRef.current = null;
throw new NativeCaptureUnavailable('canvas stream produced no video track');
}
const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare,
videoCodec: 'vp9',
});
// Optional native audio. Failure here is non-fatal — the video
// pipeline is already running and bailing out would be worse
// UX than shipping a silent share. The warning surfaces the
// platform gap so the user knows why their audio is missing.
let audioHandle: SystemAudioHandle | null = null;
if (settings.includeSystemAudio) {
// Publish system audio via WASAPI. Resolves to the audio publication
// + an ended-cleanup registration so whichever video path succeeds
// can wire the audio teardown onto its own video-ended handler.
const startWasapiAudio = async (
LkTrack: typeof import('livekit-client').Track,
): Promise<{
stopAudio: () => Promise<void>;
} | null> => {
if (!settings.includeSystemAudio) return null;
try {
audioHandle = await startSystemAudioCapture();
const audioHandle = await startSystemAudioCapture();
nativeAudioCaptureRef.current = audioHandle;
const audioMst = audioHandle.stream.getAudioTracks()[0];
if (audioMst) {
if (!audioMst) {
await audioHandle.stop();
nativeAudioCaptureRef.current = null;
return null;
}
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
});
@@ -1265,14 +1256,21 @@ export function CallProvider({ children }: { children: ReactNode }) {
} catch {
/* already unpublished */
}
})();
});
const stopAudio = async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
const active = nativeAudioCaptureRef.current;
if (active && active.captureId === audioHandle!.captureId) {
if (active && active.captureId === audioHandle.captureId) {
nativeAudioCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
})();
});
}
};
return { stopAudio };
} catch (err: unknown) {
console.warn(
'screen-share: native system-audio unavailable, sharing video only',
@@ -1282,65 +1280,11 @@ export function CallProvider({ children }: { children: ReactNode }) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
audioHandle = null;
}
return null;
}
};
// Canvas stream 'ended' fires on handle.stop() (we track.stop()
// each track) — chain unpublish + native teardown so one ended
// event cleans everything up regardless of who triggered it.
// Also tear down any paired audio capture so sound can't
// outlive the video share.
videoMst.addEventListener('ended', () => {
void (async () => {
try {
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
} catch {
/* already unpublished */
}
const active = nativeCaptureRef.current;
if (active && active.captureId === handle.captureId) {
nativeCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
const audioActive = nativeAudioCaptureRef.current;
if (audioActive) {
nativeAudioCaptureRef.current = null;
await audioActive.stop().catch(() => undefined);
}
setIsScreenSharing(false);
})();
});
console.info('screen-share: native capture active', {
audio: audioHandle != null,
});
setIsScreenSharing(true);
return;
} catch (err: unknown) {
// Native path unavailable (non-Tauri runtime, source vanished,
// first-frame timeout). Clean up any partial handle and fall
// through to the getUserMedia / getDisplayMedia paths.
if (nativeCaptureRef.current) {
await nativeCaptureRef.current.stop().catch(() => undefined);
nativeCaptureRef.current = null;
}
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
console.warn(
'screen-share: native path failed, falling back',
err instanceof Error ? err.message : err,
);
}
}
// Direct-publish path when our custom picker supplied a Chromium-
// format source id. Bypasses the OS picker so the user shares exactly
// the window/monitor they clicked in the grid. getUserMedia with the
// legacy chromeMediaSourceId constraint is not in the MediaStream
// spec but is honoured by Chromium / WebView2. If it throws we fall
// through to setScreenShareEnabled and let the OS picker run.
// ----- Path 1: chromeMediaSource (fast) -----
if (sourceId) {
try {
const { Track: LkTrack } = await import('livekit-client');
@@ -1357,36 +1301,28 @@ export function CallProvider({ children }: { children: ReactNode }) {
maxFrameRate: fps,
},
} as unknown as MediaTrackConstraints;
const audioConstraints: MediaTrackConstraints | false = settings.includeSystemAudio
? ({
mandatory: {
chromeMediaSource: 'desktop',
chromeMediaSourceId: sourceId,
},
} as unknown as MediaTrackConstraints)
: false;
const stream = await navigator.mediaDevices.getUserMedia({
audio: audioConstraints,
audio: false,
video: videoConstraints,
});
const videoMst = stream.getVideoTracks()[0];
const audioMst = stream.getAudioTracks()[0];
if (!videoMst) {
stream.getTracks().forEach((t) => t.stop());
throw new Error('no video track from chromeMediaSource');
}
// Pass raw MediaStreamTracks — `publishTrack` wraps them in the
// right Local*Track internally and the publishDefaults on the
// Room handle VP9 codec + screenShareEncoding caps. Passing the
// raw tracks also sidesteps a type incompatibility between
// livekit-client's Local*Track and our exactOptionalPropertyTypes
// setting.
// H.264 lets Chromium's hardware encoder take over on Windows
// (Intel QuickSync / NVENC / AMD VCE) — VP9 is software-only in
// Chromium's WebRTC path and collapses under the L3T3_KEY
// scalability mode we set globally for camera. `contentHint` is
// what setScreenShareEnabled sets internally; pushing it by hand
// here tells the encoder to weight sharpness over smoothness,
// which is right for UI/text-heavy screen content.
videoMst.contentHint = 'detail';
const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare,
videoCodec: 'vp9',
videoCodec: 'h264',
});
// Stop the publish when the OS revokes capture (user hit the
// OS "Stop sharing" banner, or closed the window we were sharing).
const audioRes = await startWasapiAudio(LkTrack);
videoMst.addEventListener('ended', () => {
void (async () => {
try {
@@ -1394,32 +1330,81 @@ export function CallProvider({ children }: { children: ReactNode }) {
} catch {
/* already unpublished */
}
if (audioRes) await audioRes.stopAudio();
setIsScreenSharing(false);
})();
});
if (audioMst) {
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
console.info('screen-share: chromeMediaSource active', {
audio: audioRes != null,
});
audioMst.addEventListener('ended', () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* ignore */
}
})();
});
}
setIsScreenSharing(true);
return;
} catch (err: unknown) {
// WebView2 / browser rejected the legacy constraint. Fall through
// to the normal OS picker path below so the user still gets a
// working share instead of a hard error.
console.warn(
'direct screen-share via chromeMediaSourceId failed; falling back to getDisplayMedia',
err,
'screen-share: chromeMediaSource failed, trying native xcap',
err instanceof Error ? err.message : err,
);
}
}
// ----- Path 2: xcap native (CPU fallback) -----
if (sourceId) {
try {
const { Track: LkTrack } = await import('livekit-client');
const maxWidth = ssParams.dims?.width ?? 1920;
const maxHeight = ssParams.dims?.height ?? 1080;
const handle = await startNativeCapture({
sourceId,
maxWidth,
maxHeight,
fps,
});
nativeCaptureRef.current = handle;
const videoMst = handle.stream.getVideoTracks()[0];
if (!videoMst) {
await handle.stop();
nativeCaptureRef.current = null;
throw new NativeCaptureUnavailable('canvas stream produced no video track');
}
videoMst.contentHint = 'detail';
const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare,
videoCodec: 'h264',
});
const audioRes = await startWasapiAudio(LkTrack);
videoMst.addEventListener('ended', () => {
void (async () => {
try {
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
} catch {
/* already unpublished */
}
const active = nativeCaptureRef.current;
if (active && active.captureId === handle.captureId) {
nativeCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
if (audioRes) await audioRes.stopAudio();
setIsScreenSharing(false);
})();
});
console.info('screen-share: native xcap active', {
audio: audioRes != null,
});
setIsScreenSharing(true);
return;
} catch (err: unknown) {
if (nativeCaptureRef.current) {
await nativeCaptureRef.current.stop().catch(() => undefined);
nativeCaptureRef.current = null;
}
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
console.warn(
'screen-share: native xcap failed, falling back to OS picker',
err instanceof Error ? err.message : err,
);
}
}
+1 -1
View File
@@ -46,7 +46,7 @@ export interface PresetParams {
}
const PRESET_PARAMS: Record<ScreenSharePreset, PresetParams> = {
auto: { dims: null, framerate: 60, bitrateKbps: 8000, label: 'Auto (Original)' },
auto: { dims: null, framerate: 30, bitrateKbps: 8000, label: 'Auto (Original)' },
'720p30': {
dims: { width: 1280, height: 720 },
framerate: 30,