byGalax/ChatApp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(call): native WASAPI system-audio for screen-share (Windows)

Browse Source 
Hooks the custom screen-share picker up to a native WASAPI loopback
capture so "Mit System-Sound" no longer falls back to the OS picker on
Windows. Rust side opens the default render endpoint, channels 48 kHz
f32 stereo to an AudioWorklet, which feeds a MediaStreamDestination for
LiveKit to publish as ScreenShareAudio. Ring buffer sized for latency
(80 ms target, drop-to-target on overflow) and the AudioContext is
resumed eagerly so initial burstiness can't pile up.

Adds a temporary attachTrack:audio diagnostic log to confirm source
tagging matches between old and new clients.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
byGalax
2026-04-22 23:03:15 +02:00
parent e2e8217b86
commit 665f450878
7 changed files with 827 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/desktop/src/components/ScreenSourcePicker.tsx
-8
View File
@@ -282,14 +282,6 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
</span>
</label>
</div>
{includeAudio && selectedId && (
<p className="rounded-md border border-amber-500/30 bg-amber-500/10 px-3 py-2 text-[11px] text-amber-700 dark:text-amber-300">
{t('app:call.share_audio_uses_os_picker', {
defaultValue:
'Mit System-Sound fragt der Browser noch einmal nach der Quelle — Video-Direktpfad geht nur ohne Audio.',
})}
</p>
)}
{error && (
<p role="alert" className="text-xs text-rose-600 dark:text-rose-300">
{error}
apps/desktop/src/context/CallContext.tsx
+107 -13
View File
@@ -80,6 +80,10 @@ import {
NativeCaptureUnavailable,
startNativeCapture,
} from '../lib/screenCapture';
import {
type SystemAudioHandle,
startSystemAudioCapture,
} from '../lib/screenAudio';
import {
clearScreenShareVolumes,
getScreenShareVolume,
@@ -305,6 +309,11 @@ export function CallProvider({ children }: { children: ReactNode }) {
// on the canvas track's 'ended' event. Not kept in React state because
// it never feeds into a render.
const nativeCaptureRef = useRef<NativeCaptureHandle | null>(null);
// Matching handle for the Windows-only WASAPI system-audio capture.
// Lives in lockstep with the video handle above when the user picks
// "Mit System-Sound"; teardown is wired so that stopping either track
// also stops the other, so stale audio can't outlive the video share.
const nativeAudioCaptureRef = useRef<SystemAudioHandle | null>(null);
const roomRef = useRef<Room | null>(null);
// Web Audio graph that mixes live mic + soundboard sources into a single
// published track. Created per call in joinRoom, destroyed in disconnectRoom.
@@ -486,6 +495,11 @@ export function CallProvider({ children }: { children: ReactNode }) {
nativeCaptureRef.current = null;
await h.stop().catch(() => undefined);
}
if (nativeAudioCaptureRef.current) {
const h = nativeAudioCaptureRef.current;
nativeAudioCaptureRef.current = null;
await h.stop().catch(() => undefined);
}
roomRef.current = null;
setRoom(null);
setRemoteParticipants([]);
@@ -1190,24 +1204,25 @@ export function CallProvider({ children }: { children: ReactNode }) {
const sourceId = overrides?.sourceId ?? null;
// Native capture path — tried first when the user came in via our
// custom picker. xcap on the Rust side grabs frames, streams them as
// JPEG over a Tauri channel, and we draw them onto a canvas whose
// captureStream() becomes the MediaStream LiveKit publishes. This
// completely skips the OS picker. Video-only (no system audio yet);
// if the user asked for audio we fall through to the legacy paths
// below so audio still works via getDisplayMedia.
// custom picker. xcap on the Rust side grabs video frames and, on
// Windows with "Mit System-Sound" on, the WASAPI loopback module
// grabs the render endpoint. Both stream over Tauri channels into
// tracks we publish directly to LiveKit — the OS picker never
// appears. If native audio fails on a platform that can't supply
// it (non-Windows v1), we continue with video-only and log; the
// user still gets their direct-video share.
if (!sourceId) {
console.info(
'screen-share: no sourceId supplied by picker, OS picker will open',
);
} else if (settings.includeSystemAudio) {
console.info(
'screen-share: system audio requested — native path unavailable (needs WASAPI/ScreenCaptureKit), OS picker will open',
);
}
if (sourceId && !settings.includeSystemAudio) {
if (sourceId) {
try {
console.info('screen-share: trying native capture path', { sourceId, fps });
console.info('screen-share: trying native capture path', {
sourceId,
fps,
includeSystemAudio: settings.includeSystemAudio,
});
const { Track: LkTrack } = await import('livekit-client');
const maxWidth = ssParams.dims?.width ?? 1920;
const maxHeight = ssParams.dims?.height ?? 1080;
@@ -1228,9 +1243,54 @@ export function CallProvider({ children }: { children: ReactNode }) {
source: LkTrack.Source.ScreenShare,
videoCodec: 'vp9',
});
// Optional native audio. Failure here is non-fatal — the video
// pipeline is already running and bailing out would be worse
// UX than shipping a silent share. The warning surfaces the
// platform gap so the user knows why their audio is missing.
let audioHandle: SystemAudioHandle | null = null;
if (settings.includeSystemAudio) {
try {
audioHandle = await startSystemAudioCapture();
nativeAudioCaptureRef.current = audioHandle;
const audioMst = audioHandle.stream.getAudioTracks()[0];
if (audioMst) {
const audioPub = await lp.publishTrack(audioMst, {
source: LkTrack.Source.ScreenShareAudio,
});
audioMst.addEventListener('ended', () => {
void (async () => {
try {
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
} catch {
/* already unpublished */
}
const active = nativeAudioCaptureRef.current;
if (active && active.captureId === audioHandle!.captureId) {
nativeAudioCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
})();
});
}
} catch (err: unknown) {
console.warn(
'screen-share: native system-audio unavailable, sharing video only',
err instanceof Error ? err.message : err,
);
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
audioHandle = null;
}
}
// Canvas stream 'ended' fires on handle.stop() (we track.stop()
// each track) — chain unpublish + native teardown so one ended
// event cleans everything up regardless of who triggered it.
// Also tear down any paired audio capture so sound can't
// outlive the video share.
videoMst.addEventListener('ended', () => {
void (async () => {
try {
@@ -1243,10 +1303,17 @@ export function CallProvider({ children }: { children: ReactNode }) {
nativeCaptureRef.current = null;
await active.stop().catch(() => undefined);
}
const audioActive = nativeAudioCaptureRef.current;
if (audioActive) {
nativeAudioCaptureRef.current = null;
await audioActive.stop().catch(() => undefined);
}
setIsScreenSharing(false);
})();
});
console.info('screen-share: native capture active');
console.info('screen-share: native capture active', {
audio: audioHandle != null,
});
setIsScreenSharing(true);
return;
} catch (err: unknown) {
@@ -1257,6 +1324,10 @@ export function CallProvider({ children }: { children: ReactNode }) {
await nativeCaptureRef.current.stop().catch(() => undefined);
nativeCaptureRef.current = null;
}
if (nativeAudioCaptureRef.current) {
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
nativeAudioCaptureRef.current = null;
}
console.warn(
'screen-share: native path failed, falling back',
err instanceof Error ? err.message : err,
@@ -1403,6 +1474,15 @@ export function CallProvider({ children }: { children: ReactNode }) {
console.warn('native capture stop failed', err);
}
}
if (nativeAudioCaptureRef.current) {
const h = nativeAudioCaptureRef.current;
nativeAudioCaptureRef.current = null;
try {
await h.stop();
} catch (err: unknown) {
console.warn('native audio capture stop failed', err);
}
}
const r = roomRef.current;
if (!r) return;
const lp = r.localParticipant;
@@ -2284,6 +2364,20 @@ function attachTrack(
if (isScreenShareAudio) {
audio.setAttribute('data-track-source', 'screenshare');
}
// Diagnostic — surfaces source-tag mismatches between SDK versions.
// If a remote participant publishes system-audio but the tag never
// reaches us, `isScreenShareAudio` flips false and the watching
// gate is bypassed; seeing this in the console tells us whether
// the unwanted playback is a gating bug or a tagging mismatch.
console.info('attachTrack:audio', {
participant: participant.identity,
trackSource: track.source,
pubSource: publication.source,
isScreenShareAudio,
watching: participant.identity
? watchingShareUserIdsMirror.has(participant.identity)
: null,
});
if (participant.identity) {
audio.setAttribute('data-participant', participant.identity);
}
apps/desktop/src/lib/screenAudio.ts
+260
View File
@@ -0,0 +1,260 @@
// Frontend side of the native system-audio pipeline. Pairs with the Rust
// `screen_audio` module: it opens a Tauri Channel, receives interleaved
// f32 stereo samples at 48kHz (base64-encoded), and surfaces them as a
// real `MediaStream` that LiveKit can publish as a `ScreenShareAudio`
// track. An AudioWorklet does the heavy lifting so the render thread is
// never the bottleneck — the main thread just pushes decoded samples
// across a port; the worklet copies them into its output buffer which
// feeds a `MediaStreamDestination`.
//
// Windows-only right now. On other platforms `startSystemAudioCapture`
// throws `SystemAudioUnavailable` and the caller is expected to fall
// back to the browser's getDisplayMedia path.
import { isTauriRuntime } from './globalShortcut';
export interface SystemAudioHandle {
/** Rust-side capture id. Pass to the Rust stop command via `stop()`. */
captureId: number;
/** MediaStream carrying a single audio track at 48kHz stereo. */
stream: MediaStream;
/** Teardown — stops the Rust thread, closes the AudioContext, ends the
* MediaStreamDestination track. Idempotent. */
stop: () => Promise<void>;
}
/** Thrown when the platform can't deliver native system-audio (non-Tauri
* runtime, non-Windows host, WebAudio unavailable, COM init failure). */
export class SystemAudioUnavailable extends Error {
constructor(reason: string) {
super('system audio unavailable: ' + reason);
this.name = 'SystemAudioUnavailable';
}
}
interface AudioFramePayload {
captureId: number;
sampleRate: number;
channels: number;
samplesBase64: string;
}
// AudioWorklet source embedded as a string. The worklet keeps a pair of
// ring buffers (one per channel) that the main thread appends to as
// samples arrive. `process()` drains the ring buffers into the output
// blocks; an underrun emits silence instead of propagating the stall
// upwards (a glitch is better than a freeze for LiveKit's Opus encoder).
//
// The worklet runs at AudioContext sample rate, which we pin to 48kHz via
// the AudioContext constructor. That matches what the Rust side already
// resamples to, so no further rate conversion is needed here.
const WORKLET_SOURCE = `
class LoopbackAudioProcessor extends AudioWorkletProcessor {
constructor() {
super();
// Ring buffer sized for latency, not for "never drop". 300ms hard cap,
// 80ms target — we aim for ~one WASAPI packet of headroom above the
// render quantum and drop excess whenever the producer gets ahead.
// Keeping the target small is the difference between "feels live" and
// "laggy" for screen-share audio.
this.bufferSize = 48000 * 0.3 | 0;
this.targetFrames = 48000 * 0.08 | 0;
this.bufL = new Float32Array(this.bufferSize);
this.bufR = new Float32Array(this.bufferSize);
this.writePos = 0;
this.readPos = 0;
this.available = 0;
this.port.onmessage = (e) => {
const { left, right } = e.data;
const len = left.length;
for (let i = 0; i < len; i++) {
this.bufL[this.writePos] = left[i];
this.bufR[this.writePos] = right[i];
this.writePos = (this.writePos + 1) % this.bufferSize;
if (this.available < this.bufferSize) {
this.available++;
} else {
// Buffer full — advance the read cursor to keep writing.
this.readPos = (this.readPos + 1) % this.bufferSize;
}
}
// Hard cap: if we're this far behind the producer, skip ahead to
// the target latency instead of playing out minutes of stale audio.
// Happens on: AudioContext resume after suspend, tab throttle
// recovery, any hiccup that left samples piling up.
if (this.available > this.targetFrames * 3) {
const drop = this.available - this.targetFrames;
this.readPos = (this.readPos + drop) % this.bufferSize;
this.available -= drop;
}
};
}
process(_inputs, outputs) {
const output = outputs[0];
if (!output || output.length === 0) return true;
const out0 = output[0];
const out1 = output[1] || output[0];
const n = out0.length;
for (let i = 0; i < n; i++) {
if (this.available > 0) {
out0[i] = this.bufL[this.readPos];
if (out1 !== out0) out1[i] = this.bufR[this.readPos];
this.readPos = (this.readPos + 1) % this.bufferSize;
this.available--;
} else {
out0[i] = 0;
if (out1 !== out0) out1[i] = 0;
}
}
return true;
}
}
registerProcessor('screen-audio-loopback', LoopbackAudioProcessor);
`;
let workletModuleUrl: string | null = null;
function getWorkletModuleUrl(): string {
if (workletModuleUrl) return workletModuleUrl;
const blob = new Blob([WORKLET_SOURCE], { type: 'application/javascript' });
workletModuleUrl = URL.createObjectURL(blob);
return workletModuleUrl;
}
export async function startSystemAudioCapture(): Promise<SystemAudioHandle> {
if (!isTauriRuntime()) {
throw new SystemAudioUnavailable('not a tauri runtime');
}
const AudioCtor: typeof AudioContext | undefined =
typeof window !== 'undefined'
? (window.AudioContext ??
(window as unknown as { webkitAudioContext?: typeof AudioContext })
.webkitAudioContext)
: undefined;
if (!AudioCtor) {
throw new SystemAudioUnavailable('WebAudio unavailable');
}
// Pin to 48kHz so the worklet's input rate matches the Rust-side
// output rate. If the OS forces a different rate the constructor
// throws on some browsers; we catch and surface as Unavailable so the
// caller can fall back.
let ctx: AudioContext;
try {
ctx = new AudioCtor({ sampleRate: 48000, latencyHint: 'interactive' });
} catch (err: unknown) {
throw new SystemAudioUnavailable(
err instanceof Error ? err.message : String(err),
);
}
try {
await ctx.audioWorklet.addModule(getWorkletModuleUrl());
} catch (err: unknown) {
await ctx.close().catch(() => undefined);
throw new SystemAudioUnavailable(
'audioWorklet load failed: ' +
(err instanceof Error ? err.message : String(err)),
);
}
const node = new AudioWorkletNode(ctx, 'screen-audio-loopback', {
numberOfInputs: 0,
numberOfOutputs: 1,
outputChannelCount: [2],
});
const dest = ctx.createMediaStreamDestination();
node.connect(dest);
// Kick the AudioContext out of `suspended` before any samples arrive —
// the share is triggered from a user click so autoplay policy allows
// this, and an un-resumed context would buffer everything the Rust
// side produces until the context eventually runs, giving seconds of
// initial latency.
if (ctx.state !== 'running') {
try {
await ctx.resume();
} catch (err: unknown) {
console.warn('system-audio ctx.resume failed', err);
}
}
const { Channel, invoke } = await import('@tauri-apps/api/core');
const channel = new Channel<AudioFramePayload>();
channel.onmessage = (frame: AudioFramePayload) => {
const bytes = base64ToBytes(frame.samplesBase64);
// Re-view the bytes as f32 little-endian. The byteLength is always
// a multiple of 8 (f32 stereo pairs) — if not, drop the trailing
// partial frame rather than risk a truncation artifact.
const sampleCount = Math.floor(bytes.byteLength / 4);
if (sampleCount < 2) return;
const floats = new Float32Array(
bytes.buffer,
bytes.byteOffset,
sampleCount,
);
// Interleaved L/R → deinterleaved for the worklet. Copying out of
// the base64 view also ensures the Float32Arrays we postMessage are
// owned (the underlying buffer is about to be garbage-collected).
const frames = floats.length >> 1;
const left = new Float32Array(frames);
const right = new Float32Array(frames);
for (let i = 0; i < frames; i++) {
left[i] = floats[i * 2] ?? 0;
right[i] = floats[i * 2 + 1] ?? 0;
}
// Transfer the buffers so postMessage is zero-copy.
node.port.postMessage(
{ left, right },
[left.buffer, right.buffer],
);
};
let captureId: number;
try {
captureId = await invoke<number>('start_system_audio_capture', { channel });
} catch (err: unknown) {
node.disconnect();
await ctx.close().catch(() => undefined);
throw new SystemAudioUnavailable(
err instanceof Error ? err.message : String(err),
);
}
const stream = dest.stream;
let stopped = false;
const stop = async (): Promise<void> => {
if (stopped) return;
stopped = true;
try {
await invoke('stop_system_audio_capture', { captureId });
} catch (err: unknown) {
console.warn('stop_system_audio_capture failed', err);
}
try {
node.disconnect();
} catch {
/* already disconnected */
}
for (const track of stream.getTracks()) {
try {
track.stop();
} catch {
/* already stopped */
}
}
await ctx.close().catch(() => undefined);
};
return { captureId, stream, stop };
}
function base64ToBytes(b64: string): Uint8Array {
const bin = atob(b64);
const bytes = new Uint8Array(bin.length);
for (let i = 0; i < bin.length; i++) {
bytes[i] = bin.charCodeAt(i);
}
return bytes;
}