Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| adbdfaa2aa | |||
| 6f1e1a5f9a | |||
| f9bbcdee47 | |||
| ccbef6d959 | |||
| 73ddeecfca | |||
| 727aced411 |
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@chat-app/desktop",
|
"name": "@chat-app/desktop",
|
||||||
"version": "0.11.0",
|
"version": "0.11.2",
|
||||||
"private": true,
|
"private": true,
|
||||||
"description": "Tauri v2 desktop client (Windows / macOS / Linux)",
|
"description": "Tauri v2 desktop client (Windows / macOS / Linux)",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
|
|||||||
Generated
+1
-1
@@ -834,7 +834,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "chat-app-desktop"
|
name = "chat-app-desktop"
|
||||||
version = "0.10.2"
|
version = "0.11.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
"dryoc",
|
"dryoc",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "chat-app-desktop"
|
name = "chat-app-desktop"
|
||||||
version = "0.11.0"
|
version = "0.11.2"
|
||||||
description = "ChatApp desktop client"
|
description = "ChatApp desktop client"
|
||||||
authors = ["Dennis"]
|
authors = ["Dennis"]
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ pub fn start_screen_capture(
|
|||||||
fps: u32,
|
fps: u32,
|
||||||
channel: Channel<FramePayload>,
|
channel: Channel<FramePayload>,
|
||||||
) -> Result<u32, String> {
|
) -> Result<u32, String> {
|
||||||
let clamped_fps = fps.clamp(5, 30);
|
let clamped_fps = fps.clamp(5, 60);
|
||||||
let clamped_w = max_width.max(320).min(3840);
|
let clamped_w = max_width.max(320).min(3840);
|
||||||
let clamped_h = max_height.max(180).min(2160);
|
let clamped_h = max_height.max(180).min(2160);
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://schema.tauri.app/config/2",
|
"$schema": "https://schema.tauri.app/config/2",
|
||||||
"productName": "ChatApp",
|
"productName": "ChatApp",
|
||||||
"version": "0.11.0",
|
"version": "0.11.2",
|
||||||
"identifier": "com.meinname.chatapp",
|
"identifier": "com.meinname.chatapp",
|
||||||
"build": {
|
"build": {
|
||||||
"beforeDevCommand": "pnpm vite:dev",
|
"beforeDevCommand": "pnpm vite:dev",
|
||||||
|
|||||||
@@ -1203,144 +1203,88 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
|||||||
const fps = framerateOverride ?? ssParams.framerate;
|
const fps = framerateOverride ?? ssParams.framerate;
|
||||||
const sourceId = overrides?.sourceId ?? null;
|
const sourceId = overrides?.sourceId ?? null;
|
||||||
|
|
||||||
// Native capture path — tried first when the user came in via our
|
// Ordered capture paths when our custom picker supplied a sourceId:
|
||||||
// custom picker. xcap on the Rust side grabs video frames and, on
|
// 1. chromeMediaSource video — hardware-accelerated path that
|
||||||
// Windows with "Mit System-Sound" on, the WASAPI loopback module
|
// hands the frames directly to WebRTC, same mechanism the OS
|
||||||
// grabs the render endpoint. Both stream over Tauri channels into
|
// picker uses internally. Fast, smooth, what users expect.
|
||||||
// tracks we publish directly to LiveKit — the OS picker never
|
// 2. xcap native (JPEG → canvas → captureStream) — fallback for
|
||||||
// appears. If native audio fails on a platform that can't supply
|
// WebView2 versions that reject the legacy chromeMediaSource
|
||||||
// it (non-Windows v1), we continue with video-only and log; the
|
// constraint. Functional but CPU-heavy; the per-frame JPEG
|
||||||
// user still gets their direct-video share.
|
// encode + base64 + createImageBitmap chain shows up as
|
||||||
|
// visible stutter on 1080p30.
|
||||||
|
// 3. OS picker (setScreenShareEnabled) — last resort when both
|
||||||
|
// native paths throw, and the only path when no sourceId was
|
||||||
|
// supplied.
|
||||||
|
//
|
||||||
|
// Audio (Windows only) always goes through the WASAPI loopback
|
||||||
|
// module — getUserMedia's chromeMediaSource audio constraint
|
||||||
|
// throws AbortError on Window captures, and pairing both into one
|
||||||
|
// call ended up blocking the video path entirely. Splitting them
|
||||||
|
// means the video path stays fast and the audio path stays
|
||||||
|
// reliable for any source type.
|
||||||
if (!sourceId) {
|
if (!sourceId) {
|
||||||
console.info(
|
console.info(
|
||||||
'screen-share: no sourceId supplied by picker, OS picker will open',
|
'screen-share: no sourceId supplied by picker, OS picker will open',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (sourceId) {
|
|
||||||
|
// Publish system audio via WASAPI. Resolves to the audio publication
|
||||||
|
// + an ended-cleanup registration so whichever video path succeeds
|
||||||
|
// can wire the audio teardown onto its own video-ended handler.
|
||||||
|
const startWasapiAudio = async (
|
||||||
|
LkTrack: typeof import('livekit-client').Track,
|
||||||
|
): Promise<{
|
||||||
|
stopAudio: () => Promise<void>;
|
||||||
|
} | null> => {
|
||||||
|
if (!settings.includeSystemAudio) return null;
|
||||||
try {
|
try {
|
||||||
console.info('screen-share: trying native capture path', {
|
const audioHandle = await startSystemAudioCapture();
|
||||||
sourceId,
|
nativeAudioCaptureRef.current = audioHandle;
|
||||||
fps,
|
const audioMst = audioHandle.stream.getAudioTracks()[0];
|
||||||
includeSystemAudio: settings.includeSystemAudio,
|
if (!audioMst) {
|
||||||
});
|
await audioHandle.stop();
|
||||||
const { Track: LkTrack } = await import('livekit-client');
|
nativeAudioCaptureRef.current = null;
|
||||||
const maxWidth = ssParams.dims?.width ?? 1920;
|
return null;
|
||||||
const maxHeight = ssParams.dims?.height ?? 1080;
|
|
||||||
const handle = await startNativeCapture({
|
|
||||||
sourceId,
|
|
||||||
maxWidth,
|
|
||||||
maxHeight,
|
|
||||||
fps,
|
|
||||||
});
|
|
||||||
nativeCaptureRef.current = handle;
|
|
||||||
const videoMst = handle.stream.getVideoTracks()[0];
|
|
||||||
if (!videoMst) {
|
|
||||||
await handle.stop();
|
|
||||||
nativeCaptureRef.current = null;
|
|
||||||
throw new NativeCaptureUnavailable('canvas stream produced no video track');
|
|
||||||
}
|
}
|
||||||
const videoPub = await lp.publishTrack(videoMst, {
|
const audioPub = await lp.publishTrack(audioMst, {
|
||||||
source: LkTrack.Source.ScreenShare,
|
source: LkTrack.Source.ScreenShareAudio,
|
||||||
videoCodec: 'vp9',
|
|
||||||
});
|
});
|
||||||
|
audioMst.addEventListener('ended', () => {
|
||||||
// Optional native audio. Failure here is non-fatal — the video
|
|
||||||
// pipeline is already running and bailing out would be worse
|
|
||||||
// UX than shipping a silent share. The warning surfaces the
|
|
||||||
// platform gap so the user knows why their audio is missing.
|
|
||||||
let audioHandle: SystemAudioHandle | null = null;
|
|
||||||
if (settings.includeSystemAudio) {
|
|
||||||
try {
|
|
||||||
audioHandle = await startSystemAudioCapture();
|
|
||||||
nativeAudioCaptureRef.current = audioHandle;
|
|
||||||
const audioMst = audioHandle.stream.getAudioTracks()[0];
|
|
||||||
if (audioMst) {
|
|
||||||
const audioPub = await lp.publishTrack(audioMst, {
|
|
||||||
source: LkTrack.Source.ScreenShareAudio,
|
|
||||||
});
|
|
||||||
audioMst.addEventListener('ended', () => {
|
|
||||||
void (async () => {
|
|
||||||
try {
|
|
||||||
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
|
|
||||||
} catch {
|
|
||||||
/* already unpublished */
|
|
||||||
}
|
|
||||||
const active = nativeAudioCaptureRef.current;
|
|
||||||
if (active && active.captureId === audioHandle!.captureId) {
|
|
||||||
nativeAudioCaptureRef.current = null;
|
|
||||||
await active.stop().catch(() => undefined);
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} catch (err: unknown) {
|
|
||||||
console.warn(
|
|
||||||
'screen-share: native system-audio unavailable, sharing video only',
|
|
||||||
err instanceof Error ? err.message : err,
|
|
||||||
);
|
|
||||||
if (nativeAudioCaptureRef.current) {
|
|
||||||
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
|
|
||||||
nativeAudioCaptureRef.current = null;
|
|
||||||
}
|
|
||||||
audioHandle = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Canvas stream 'ended' fires on handle.stop() (we track.stop()
|
|
||||||
// each track) — chain unpublish + native teardown so one ended
|
|
||||||
// event cleans everything up regardless of who triggered it.
|
|
||||||
// Also tear down any paired audio capture so sound can't
|
|
||||||
// outlive the video share.
|
|
||||||
videoMst.addEventListener('ended', () => {
|
|
||||||
void (async () => {
|
void (async () => {
|
||||||
try {
|
try {
|
||||||
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
|
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
|
||||||
} catch {
|
} catch {
|
||||||
/* already unpublished */
|
/* already unpublished */
|
||||||
}
|
}
|
||||||
const active = nativeCaptureRef.current;
|
|
||||||
if (active && active.captureId === handle.captureId) {
|
|
||||||
nativeCaptureRef.current = null;
|
|
||||||
await active.stop().catch(() => undefined);
|
|
||||||
}
|
|
||||||
const audioActive = nativeAudioCaptureRef.current;
|
|
||||||
if (audioActive) {
|
|
||||||
nativeAudioCaptureRef.current = null;
|
|
||||||
await audioActive.stop().catch(() => undefined);
|
|
||||||
}
|
|
||||||
setIsScreenSharing(false);
|
|
||||||
})();
|
})();
|
||||||
});
|
});
|
||||||
console.info('screen-share: native capture active', {
|
const stopAudio = async () => {
|
||||||
audio: audioHandle != null,
|
try {
|
||||||
});
|
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
|
||||||
setIsScreenSharing(true);
|
} catch {
|
||||||
return;
|
/* already unpublished */
|
||||||
|
}
|
||||||
|
const active = nativeAudioCaptureRef.current;
|
||||||
|
if (active && active.captureId === audioHandle.captureId) {
|
||||||
|
nativeAudioCaptureRef.current = null;
|
||||||
|
await active.stop().catch(() => undefined);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return { stopAudio };
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
// Native path unavailable (non-Tauri runtime, source vanished,
|
console.warn(
|
||||||
// first-frame timeout). Clean up any partial handle and fall
|
'screen-share: native system-audio unavailable, sharing video only',
|
||||||
// through to the getUserMedia / getDisplayMedia paths.
|
err instanceof Error ? err.message : err,
|
||||||
if (nativeCaptureRef.current) {
|
);
|
||||||
await nativeCaptureRef.current.stop().catch(() => undefined);
|
|
||||||
nativeCaptureRef.current = null;
|
|
||||||
}
|
|
||||||
if (nativeAudioCaptureRef.current) {
|
if (nativeAudioCaptureRef.current) {
|
||||||
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
|
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
|
||||||
nativeAudioCaptureRef.current = null;
|
nativeAudioCaptureRef.current = null;
|
||||||
}
|
}
|
||||||
console.warn(
|
return null;
|
||||||
'screen-share: native path failed, falling back',
|
|
||||||
err instanceof Error ? err.message : err,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
|
|
||||||
// Direct-publish path when our custom picker supplied a Chromium-
|
// ----- Path 1: chromeMediaSource (fast) -----
|
||||||
// format source id. Bypasses the OS picker so the user shares exactly
|
|
||||||
// the window/monitor they clicked in the grid. getUserMedia with the
|
|
||||||
// legacy chromeMediaSourceId constraint is not in the MediaStream
|
|
||||||
// spec but is honoured by Chromium / WebView2. If it throws we fall
|
|
||||||
// through to setScreenShareEnabled and let the OS picker run.
|
|
||||||
if (sourceId) {
|
if (sourceId) {
|
||||||
try {
|
try {
|
||||||
const { Track: LkTrack } = await import('livekit-client');
|
const { Track: LkTrack } = await import('livekit-client');
|
||||||
@@ -1357,36 +1301,28 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
|||||||
maxFrameRate: fps,
|
maxFrameRate: fps,
|
||||||
},
|
},
|
||||||
} as unknown as MediaTrackConstraints;
|
} as unknown as MediaTrackConstraints;
|
||||||
const audioConstraints: MediaTrackConstraints | false = settings.includeSystemAudio
|
|
||||||
? ({
|
|
||||||
mandatory: {
|
|
||||||
chromeMediaSource: 'desktop',
|
|
||||||
chromeMediaSourceId: sourceId,
|
|
||||||
},
|
|
||||||
} as unknown as MediaTrackConstraints)
|
|
||||||
: false;
|
|
||||||
const stream = await navigator.mediaDevices.getUserMedia({
|
const stream = await navigator.mediaDevices.getUserMedia({
|
||||||
audio: audioConstraints,
|
audio: false,
|
||||||
video: videoConstraints,
|
video: videoConstraints,
|
||||||
});
|
});
|
||||||
const videoMst = stream.getVideoTracks()[0];
|
const videoMst = stream.getVideoTracks()[0];
|
||||||
const audioMst = stream.getAudioTracks()[0];
|
|
||||||
if (!videoMst) {
|
if (!videoMst) {
|
||||||
stream.getTracks().forEach((t) => t.stop());
|
stream.getTracks().forEach((t) => t.stop());
|
||||||
throw new Error('no video track from chromeMediaSource');
|
throw new Error('no video track from chromeMediaSource');
|
||||||
}
|
}
|
||||||
// Pass raw MediaStreamTracks — `publishTrack` wraps them in the
|
// H.264 lets Chromium's hardware encoder take over on Windows
|
||||||
// right Local*Track internally and the publishDefaults on the
|
// (Intel QuickSync / NVENC / AMD VCE) — VP9 is software-only in
|
||||||
// Room handle VP9 codec + screenShareEncoding caps. Passing the
|
// Chromium's WebRTC path and collapses under the L3T3_KEY
|
||||||
// raw tracks also sidesteps a type incompatibility between
|
// scalability mode we set globally for camera. `contentHint` is
|
||||||
// livekit-client's Local*Track and our exactOptionalPropertyTypes
|
// what setScreenShareEnabled sets internally; pushing it by hand
|
||||||
// setting.
|
// here tells the encoder to weight sharpness over smoothness,
|
||||||
|
// which is right for UI/text-heavy screen content.
|
||||||
|
videoMst.contentHint = 'detail';
|
||||||
const videoPub = await lp.publishTrack(videoMst, {
|
const videoPub = await lp.publishTrack(videoMst, {
|
||||||
source: LkTrack.Source.ScreenShare,
|
source: LkTrack.Source.ScreenShare,
|
||||||
videoCodec: 'vp9',
|
videoCodec: 'h264',
|
||||||
});
|
});
|
||||||
// Stop the publish when the OS revokes capture (user hit the
|
const audioRes = await startWasapiAudio(LkTrack);
|
||||||
// OS "Stop sharing" banner, or closed the window we were sharing).
|
|
||||||
videoMst.addEventListener('ended', () => {
|
videoMst.addEventListener('ended', () => {
|
||||||
void (async () => {
|
void (async () => {
|
||||||
try {
|
try {
|
||||||
@@ -1394,32 +1330,81 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
|||||||
} catch {
|
} catch {
|
||||||
/* already unpublished */
|
/* already unpublished */
|
||||||
}
|
}
|
||||||
|
if (audioRes) await audioRes.stopAudio();
|
||||||
setIsScreenSharing(false);
|
setIsScreenSharing(false);
|
||||||
})();
|
})();
|
||||||
});
|
});
|
||||||
if (audioMst) {
|
console.info('screen-share: chromeMediaSource active', {
|
||||||
const audioPub = await lp.publishTrack(audioMst, {
|
audio: audioRes != null,
|
||||||
source: LkTrack.Source.ScreenShareAudio,
|
});
|
||||||
});
|
|
||||||
audioMst.addEventListener('ended', () => {
|
|
||||||
void (async () => {
|
|
||||||
try {
|
|
||||||
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
|
|
||||||
} catch {
|
|
||||||
/* ignore */
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
setIsScreenSharing(true);
|
setIsScreenSharing(true);
|
||||||
return;
|
return;
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
// WebView2 / browser rejected the legacy constraint. Fall through
|
|
||||||
// to the normal OS picker path below so the user still gets a
|
|
||||||
// working share instead of a hard error.
|
|
||||||
console.warn(
|
console.warn(
|
||||||
'direct screen-share via chromeMediaSourceId failed; falling back to getDisplayMedia',
|
'screen-share: chromeMediaSource failed, trying native xcap',
|
||||||
err,
|
err instanceof Error ? err.message : err,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----- Path 2: xcap native (CPU fallback) -----
|
||||||
|
if (sourceId) {
|
||||||
|
try {
|
||||||
|
const { Track: LkTrack } = await import('livekit-client');
|
||||||
|
const maxWidth = ssParams.dims?.width ?? 1920;
|
||||||
|
const maxHeight = ssParams.dims?.height ?? 1080;
|
||||||
|
const handle = await startNativeCapture({
|
||||||
|
sourceId,
|
||||||
|
maxWidth,
|
||||||
|
maxHeight,
|
||||||
|
fps,
|
||||||
|
});
|
||||||
|
nativeCaptureRef.current = handle;
|
||||||
|
const videoMst = handle.stream.getVideoTracks()[0];
|
||||||
|
if (!videoMst) {
|
||||||
|
await handle.stop();
|
||||||
|
nativeCaptureRef.current = null;
|
||||||
|
throw new NativeCaptureUnavailable('canvas stream produced no video track');
|
||||||
|
}
|
||||||
|
videoMst.contentHint = 'detail';
|
||||||
|
const videoPub = await lp.publishTrack(videoMst, {
|
||||||
|
source: LkTrack.Source.ScreenShare,
|
||||||
|
videoCodec: 'h264',
|
||||||
|
});
|
||||||
|
const audioRes = await startWasapiAudio(LkTrack);
|
||||||
|
videoMst.addEventListener('ended', () => {
|
||||||
|
void (async () => {
|
||||||
|
try {
|
||||||
|
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
|
||||||
|
} catch {
|
||||||
|
/* already unpublished */
|
||||||
|
}
|
||||||
|
const active = nativeCaptureRef.current;
|
||||||
|
if (active && active.captureId === handle.captureId) {
|
||||||
|
nativeCaptureRef.current = null;
|
||||||
|
await active.stop().catch(() => undefined);
|
||||||
|
}
|
||||||
|
if (audioRes) await audioRes.stopAudio();
|
||||||
|
setIsScreenSharing(false);
|
||||||
|
})();
|
||||||
|
});
|
||||||
|
console.info('screen-share: native xcap active', {
|
||||||
|
audio: audioRes != null,
|
||||||
|
});
|
||||||
|
setIsScreenSharing(true);
|
||||||
|
return;
|
||||||
|
} catch (err: unknown) {
|
||||||
|
if (nativeCaptureRef.current) {
|
||||||
|
await nativeCaptureRef.current.stop().catch(() => undefined);
|
||||||
|
nativeCaptureRef.current = null;
|
||||||
|
}
|
||||||
|
if (nativeAudioCaptureRef.current) {
|
||||||
|
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
|
||||||
|
nativeAudioCaptureRef.current = null;
|
||||||
|
}
|
||||||
|
console.warn(
|
||||||
|
'screen-share: native xcap failed, falling back to OS picker',
|
||||||
|
err instanceof Error ? err.message : err,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ export interface PresetParams {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const PRESET_PARAMS: Record<ScreenSharePreset, PresetParams> = {
|
const PRESET_PARAMS: Record<ScreenSharePreset, PresetParams> = {
|
||||||
auto: { dims: null, framerate: 60, bitrateKbps: 8000, label: 'Auto (Original)' },
|
auto: { dims: null, framerate: 30, bitrateKbps: 8000, label: 'Auto (Original)' },
|
||||||
'720p30': {
|
'720p30': {
|
||||||
dims: { width: 1280, height: 720 },
|
dims: { width: 1280, height: 720 },
|
||||||
framerate: 30,
|
framerate: 30,
|
||||||
|
|||||||
Reference in New Issue
Block a user