Compare commits

..

3 Commits

Author SHA1 Message Date
byGalax ccbef6d959 chore(desktop): release v0.11.1 2026-04-22 23:17:51 +02:00
byGalax 73ddeecfca chore(desktop): sync Cargo.lock to v0.11.0
Release script bumped Cargo.toml but cargo only refreshes the lock on
the next build. Aligning them so the lock doesn't drift across tags.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:15:54 +02:00
byGalax 727aced411 perf(call): prefer chromeMediaSource video over xcap native capture
The xcap native path does JPEG-encode-in-Rust → base64 → IPC → atob →
createImageBitmap → canvas.drawImage → canvas.captureStream → VP9 per
frame, all CPU-bound and mostly on the main thread — at 1080p30 that
lands well past one render quantum, producing visible frame-by-frame
stutter. chromeMediaSource+getUserMedia hands the capture to Chromium's
native desktop-capture backend and directly into the PeerConnection, so
it's the same path the OS picker uses and has no per-frame JS cost.

Reorders the capture attempts so chromeMediaSource is tried first; xcap
stays around as a fallback for WebView2 versions that reject the legacy
constraint. System audio still goes through WASAPI in both paths, since
getUserMedia's chromeMediaSource audio constraint throws AbortError on
Window captures — splitting the streams is what makes both work.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:14:21 +02:00
5 changed files with 140 additions and 164 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@chat-app/desktop", "name": "@chat-app/desktop",
"version": "0.11.0", "version": "0.11.1",
"private": true, "private": true,
"description": "Tauri v2 desktop client (Windows / macOS / Linux)", "description": "Tauri v2 desktop client (Windows / macOS / Linux)",
"type": "module", "type": "module",
+1 -1
View File
@@ -834,7 +834,7 @@ dependencies = [
[[package]] [[package]]
name = "chat-app-desktop" name = "chat-app-desktop"
version = "0.10.2" version = "0.11.0"
dependencies = [ dependencies = [
"base64 0.22.1", "base64 0.22.1",
"dryoc", "dryoc",
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "chat-app-desktop" name = "chat-app-desktop"
version = "0.11.0" version = "0.11.1"
description = "ChatApp desktop client" description = "ChatApp desktop client"
authors = ["Dennis"] authors = ["Dennis"]
edition = "2021" edition = "2021"
+1 -1
View File
@@ -1,7 +1,7 @@
{ {
"$schema": "https://schema.tauri.app/config/2", "$schema": "https://schema.tauri.app/config/2",
"productName": "ChatApp", "productName": "ChatApp",
"version": "0.11.0", "version": "0.11.1",
"identifier": "com.meinname.chatapp", "identifier": "com.meinname.chatapp",
"build": { "build": {
"beforeDevCommand": "pnpm vite:dev", "beforeDevCommand": "pnpm vite:dev",
+136 -160
View File
@@ -1203,26 +1203,145 @@ export function CallProvider({ children }: { children: ReactNode }) {
const fps = framerateOverride ?? ssParams.framerate; const fps = framerateOverride ?? ssParams.framerate;
const sourceId = overrides?.sourceId ?? null; const sourceId = overrides?.sourceId ?? null;
// Native capture path — tried first when the user came in via our // Ordered capture paths when our custom picker supplied a sourceId:
// custom picker. xcap on the Rust side grabs video frames and, on // 1. chromeMediaSource video — hardware-accelerated path that
// Windows with "Mit System-Sound" on, the WASAPI loopback module // hands the frames directly to WebRTC, same mechanism the OS
// grabs the render endpoint. Both stream over Tauri channels into // picker uses internally. Fast, smooth, what users expect.
// tracks we publish directly to LiveKit — the OS picker never // 2. xcap native (JPEG → canvas → captureStream) — fallback for
// appears. If native audio fails on a platform that can't supply // WebView2 versions that reject the legacy chromeMediaSource
// it (non-Windows v1), we continue with video-only and log; the // constraint. Functional but CPU-heavy; the per-frame JPEG
// user still gets their direct-video share. // encode + base64 + createImageBitmap chain shows up as
// visible stutter on 1080p30.
// 3. OS picker (setScreenShareEnabled) — last resort when both
// native paths throw, and the only path when no sourceId was
// supplied.
//
// Audio (Windows only) always goes through the WASAPI loopback
// module — getUserMedia's chromeMediaSource audio constraint
// throws AbortError on Window captures, and pairing both into one
// call ended up blocking the video path entirely. Splitting them
// means the video path stays fast and the audio path stays
// reliable for any source type.
if (!sourceId) { if (!sourceId) {
console.info( console.info(
'screen-share: no sourceId supplied by picker, OS picker will open', 'screen-share: no sourceId supplied by picker, OS picker will open',
); );
} }
// Publish system audio via WASAPI. Resolves to the audio publication
// + an ended-cleanup registration so whichever video path succeeds
// can wire the audio teardown onto its own video-ended handler.
const startWasapiAudio = async (
LkTrack: typeof import('livekit-client').Track,
): Promise<{
stopAudio: () => Promise<void>;
} | null> => {
if (!settings.includeSystemAudio) return null;
try {
const audioHandle = await startSystemAudioCapture();
nativeAudioCaptureRef.current = audioHandle;
const audioMst = audioHandle.stream.getAudioTracks()[0];
if (!audioMst) {
await audioHandle.stop();
nativeAudioCaptureRef.current = null;
return null;
}
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
});
audioMst.addEventListener('ended', () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
})();
});
const stopAudio = async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
const active = nativeAudioCaptureRef.current;
if (active && active.captureId === audioHandle.captureId) {
nativeAudioCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
};
return { stopAudio };
} catch (err: unknown) {
console.warn(
'screen-share: native system-audio unavailable, sharing video only',
err instanceof Error ? err.message : err,
);
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
return null;
}
};
// ----- Path 1: chromeMediaSource (fast) -----
if (sourceId) { if (sourceId) {
try { try {
console.info('screen-share: trying native capture path', { const { Track: LkTrack } = await import('livekit-client');
sourceId, const maxWidth = ssParams.dims?.width ?? 3840;
fps, const maxHeight = ssParams.dims?.height ?? 2160;
includeSystemAudio: settings.includeSystemAudio, // Cast chains: browsers expose the legacy constraint via
// `MediaTrackConstraints.mandatory` which isn't in lib.dom.
const videoConstraints = {
mandatory: {
chromeMediaSource: 'desktop',
chromeMediaSourceId: sourceId,
maxWidth,
maxHeight,
maxFrameRate: fps,
},
} as unknown as MediaTrackConstraints;
const stream = await navigator.mediaDevices.getUserMedia({
audio: false,
video: videoConstraints,
}); });
const videoMst = stream.getVideoTracks()[0];
if (!videoMst) {
stream.getTracks().forEach((t) => t.stop());
throw new Error('no video track from chromeMediaSource');
}
const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare,
videoCodec: 'vp9',
});
const audioRes = await startWasapiAudio(LkTrack);
videoMst.addEventListener('ended', () => {
void (async () => {
try {
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
} catch {
/* already unpublished */
}
if (audioRes) await audioRes.stopAudio();
setIsScreenSharing(false);
})();
});
console.info('screen-share: chromeMediaSource active', {
audio: audioRes != null,
});
setIsScreenSharing(true);
return;
} catch (err: unknown) {
console.warn(
'screen-share: chromeMediaSource failed, trying native xcap',
err instanceof Error ? err.message : err,
);
}
}
// ----- Path 2: xcap native (CPU fallback) -----
if (sourceId) {
try {
const { Track: LkTrack } = await import('livekit-client'); const { Track: LkTrack } = await import('livekit-client');
const maxWidth = ssParams.dims?.width ?? 1920; const maxWidth = ssParams.dims?.width ?? 1920;
const maxHeight = ssParams.dims?.height ?? 1080; const maxHeight = ssParams.dims?.height ?? 1080;
@@ -1243,54 +1362,7 @@ export function CallProvider({ children }: { children: ReactNode }) {
source: LkTrack.Source.ScreenShare, source: LkTrack.Source.ScreenShare,
videoCodec: 'vp9', videoCodec: 'vp9',
}); });
const audioRes = await startWasapiAudio(LkTrack);
// Optional native audio. Failure here is non-fatal — the video
// pipeline is already running and bailing out would be worse
// UX than shipping a silent share. The warning surfaces the
// platform gap so the user knows why their audio is missing.
let audioHandle: SystemAudioHandle | null = null;
if (settings.includeSystemAudio) {
try {
audioHandle = await startSystemAudioCapture();
nativeAudioCaptureRef.current = audioHandle;
const audioMst = audioHandle.stream.getAudioTracks()[0];
if (audioMst) {
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
});
audioMst.addEventListener('ended', () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
const active = nativeAudioCaptureRef.current;
if (active && active.captureId === audioHandle!.captureId) {
nativeAudioCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
})();
});
}
} catch (err: unknown) {
console.warn(
'screen-share: native system-audio unavailable, sharing video only',
err instanceof Error ? err.message : err,
);
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
audioHandle = null;
}
}
// Canvas stream 'ended' fires on handle.stop() (we track.stop()
// each track) — chain unpublish + native teardown so one ended
// event cleans everything up regardless of who triggered it.
// Also tear down any paired audio capture so sound can't
// outlive the video share.
videoMst.addEventListener('ended', () => { videoMst.addEventListener('ended', () => {
void (async () => { void (async () => {
try { try {
@@ -1303,23 +1375,16 @@ export function CallProvider({ children }: { children: ReactNode }) {
nativeCaptureRef.current = null; nativeCaptureRef.current = null;
await active.stop().catch(() => undefined); await active.stop().catch(() => undefined);
} }
const audioActive = nativeAudioCaptureRef.current; if (audioRes) await audioRes.stopAudio();
if (audioActive) {
nativeAudioCaptureRef.current = null;
await audioActive.stop().catch(() => undefined);
}
setIsScreenSharing(false); setIsScreenSharing(false);
})(); })();
}); });
console.info('screen-share: native capture active', { console.info('screen-share: native xcap active', {
audio: audioHandle != null, audio: audioRes != null,
}); });
setIsScreenSharing(true); setIsScreenSharing(true);
return; return;
} catch (err: unknown) { } catch (err: unknown) {
// Native path unavailable (non-Tauri runtime, source vanished,
// first-frame timeout). Clean up any partial handle and fall
// through to the getUserMedia / getDisplayMedia paths.
if (nativeCaptureRef.current) { if (nativeCaptureRef.current) {
await nativeCaptureRef.current.stop().catch(() => undefined); await nativeCaptureRef.current.stop().catch(() => undefined);
nativeCaptureRef.current = null; nativeCaptureRef.current = null;
@@ -1329,101 +1394,12 @@ export function CallProvider({ children }: { children: ReactNode }) {
nativeAudioCaptureRef.current = null; nativeAudioCaptureRef.current = null;
} }
console.warn( console.warn(
'screen-share: native path failed, falling back', 'screen-share: native xcap failed, falling back to OS picker',
err instanceof Error ? err.message : err, err instanceof Error ? err.message : err,
); );
} }
} }
// Direct-publish path when our custom picker supplied a Chromium-
// format source id. Bypasses the OS picker so the user shares exactly
// the window/monitor they clicked in the grid. getUserMedia with the
// legacy chromeMediaSourceId constraint is not in the MediaStream
// spec but is honoured by Chromium / WebView2. If it throws we fall
// through to setScreenShareEnabled and let the OS picker run.
if (sourceId) {
try {
const { Track: LkTrack } = await import('livekit-client');
const maxWidth = ssParams.dims?.width ?? 3840;
const maxHeight = ssParams.dims?.height ?? 2160;
// Cast chains: browsers expose the legacy constraint via
// `MediaTrackConstraints.mandatory` which isn't in lib.dom.
const videoConstraints = {
mandatory: {
chromeMediaSource: 'desktop',
chromeMediaSourceId: sourceId,
maxWidth,
maxHeight,
maxFrameRate: fps,
},
} as unknown as MediaTrackConstraints;
const audioConstraints: MediaTrackConstraints | false = settings.includeSystemAudio
? ({
mandatory: {
chromeMediaSource: 'desktop',
chromeMediaSourceId: sourceId,
},
} as unknown as MediaTrackConstraints)
: false;
const stream = await navigator.mediaDevices.getUserMedia({
audio: audioConstraints,
video: videoConstraints,
});
const videoMst = stream.getVideoTracks()[0];
const audioMst = stream.getAudioTracks()[0];
if (!videoMst) {
stream.getTracks().forEach((t) => t.stop());
throw new Error('no video track from chromeMediaSource');
}
// Pass raw MediaStreamTracks — `publishTrack` wraps them in the
// right Local*Track internally and the publishDefaults on the
// Room handle VP9 codec + screenShareEncoding caps. Passing the
// raw tracks also sidesteps a type incompatibility between
// livekit-client's Local*Track and our exactOptionalPropertyTypes
// setting.
const videoPub = await lp.publishTrack(videoMst, {
source: LkTrack.Source.ScreenShare,
videoCodec: 'vp9',
});
// Stop the publish when the OS revokes capture (user hit the
// OS "Stop sharing" banner, or closed the window we were sharing).
videoMst.addEventListener('ended', () => {
void (async () => {
try {
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
} catch {
/* already unpublished */
}
setIsScreenSharing(false);
})();
});
if (audioMst) {
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
});
audioMst.addEventListener('ended', () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* ignore */
}
})();
});
}
setIsScreenSharing(true);
return;
} catch (err: unknown) {
// WebView2 / browser rejected the legacy constraint. Fall through
// to the normal OS picker path below so the user still gets a
// working share instead of a hard error.
console.warn(
'direct screen-share via chromeMediaSourceId failed; falling back to getDisplayMedia',
err,
);
}
}
try { try {
await lp.setScreenShareEnabled(true, { await lp.setScreenShareEnabled(true, {
// "Go live" mode — capture system audio alongside the screen when // "Go live" mode — capture system audio alongside the screen when