feat(call): native WASAPI system-audio for screen-share (Windows)

Hooks the custom screen-share picker up to a native WASAPI loopback capture so "Mit System-Sound" no longer falls back to the OS picker on Windows. Rust side opens the default render endpoint, channels 48 kHz f32 stereo to an AudioWorklet, which feeds a MediaStreamDestination for LiveKit to publish as ScreenShareAudio. Ring buffer sized for latency (80 ms target, drop-to-target on overflow) and the AudioContext is resumed eagerly so initial burstiness can't pile up. Adds a temporary attachTrack:audio diagnostic log to confirm source tagging matches between old and new clients. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 23:03:15 +02:00
parent e2e8217b86
commit 665f450878
7 changed files with 827 additions and 21 deletions
@@ -0,0 +1,260 @@
+// Frontend side of the native system-audio pipeline. Pairs with the Rust
+// `screen_audio` module: it opens a Tauri Channel, receives interleaved
+// f32 stereo samples at 48kHz (base64-encoded), and surfaces them as a
+// real `MediaStream` that LiveKit can publish as a `ScreenShareAudio`
+// track. An AudioWorklet does the heavy lifting so the render thread is
+// never the bottleneck — the main thread just pushes decoded samples
+// across a port; the worklet copies them into its output buffer which
+// feeds a `MediaStreamDestination`.
+//
+// Windows-only right now. On other platforms `startSystemAudioCapture`
+// throws `SystemAudioUnavailable` and the caller is expected to fall
+// back to the browser's getDisplayMedia path.
+
+import { isTauriRuntime } from './globalShortcut';
+
+export interface SystemAudioHandle {
+  /** Rust-side capture id. Pass to the Rust stop command via `stop()`. */
+  captureId: number;
+  /** MediaStream carrying a single audio track at 48kHz stereo. */
+  stream: MediaStream;
+  /** Teardown — stops the Rust thread, closes the AudioContext, ends the
+   *  MediaStreamDestination track. Idempotent. */
+  stop: () => Promise<void>;
+}
+
+/** Thrown when the platform can't deliver native system-audio (non-Tauri
+ *  runtime, non-Windows host, WebAudio unavailable, COM init failure). */
+export class SystemAudioUnavailable extends Error {
+  constructor(reason: string) {
+    super('system audio unavailable: ' + reason);
+    this.name = 'SystemAudioUnavailable';
+  }
+}
+
+interface AudioFramePayload {
+  captureId: number;
+  sampleRate: number;
+  channels: number;
+  samplesBase64: string;
+}
+
+// AudioWorklet source embedded as a string. The worklet keeps a pair of
+// ring buffers (one per channel) that the main thread appends to as
+// samples arrive. `process()` drains the ring buffers into the output
+// blocks; an underrun emits silence instead of propagating the stall
+// upwards (a glitch is better than a freeze for LiveKit's Opus encoder).
+//
+// The worklet runs at AudioContext sample rate, which we pin to 48kHz via
+// the AudioContext constructor. That matches what the Rust side already
+// resamples to, so no further rate conversion is needed here.
+const WORKLET_SOURCE = `
+class LoopbackAudioProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    // Ring buffer sized for latency, not for "never drop". 300ms hard cap,
+    // 80ms target — we aim for ~one WASAPI packet of headroom above the
+    // render quantum and drop excess whenever the producer gets ahead.
+    // Keeping the target small is the difference between "feels live" and
+    // "laggy" for screen-share audio.
+    this.bufferSize = 48000 * 0.3 | 0;
+    this.targetFrames = 48000 * 0.08 | 0;
+    this.bufL = new Float32Array(this.bufferSize);
+    this.bufR = new Float32Array(this.bufferSize);
+    this.writePos = 0;
+    this.readPos = 0;
+    this.available = 0;
+    this.port.onmessage = (e) => {
+      const { left, right } = e.data;
+      const len = left.length;
+      for (let i = 0; i < len; i++) {
+        this.bufL[this.writePos] = left[i];
+        this.bufR[this.writePos] = right[i];
+        this.writePos = (this.writePos + 1) % this.bufferSize;
+        if (this.available < this.bufferSize) {
+          this.available++;
+        } else {
+          // Buffer full — advance the read cursor to keep writing.
+          this.readPos = (this.readPos + 1) % this.bufferSize;
+        }
+      }
+      // Hard cap: if we're this far behind the producer, skip ahead to
+      // the target latency instead of playing out minutes of stale audio.
+      // Happens on: AudioContext resume after suspend, tab throttle
+      // recovery, any hiccup that left samples piling up.
+      if (this.available > this.targetFrames * 3) {
+        const drop = this.available - this.targetFrames;
+        this.readPos = (this.readPos + drop) % this.bufferSize;
+        this.available -= drop;
+      }
+    };
+  }
+  process(_inputs, outputs) {
+    const output = outputs[0];
+    if (!output || output.length === 0) return true;
+    const out0 = output[0];
+    const out1 = output[1] || output[0];
+    const n = out0.length;
+    for (let i = 0; i < n; i++) {
+      if (this.available > 0) {
+        out0[i] = this.bufL[this.readPos];
+        if (out1 !== out0) out1[i] = this.bufR[this.readPos];
+        this.readPos = (this.readPos + 1) % this.bufferSize;
+        this.available--;
+      } else {
+        out0[i] = 0;
+        if (out1 !== out0) out1[i] = 0;
+      }
+    }
+    return true;
+  }
+}
+registerProcessor('screen-audio-loopback', LoopbackAudioProcessor);
+`;
+
+let workletModuleUrl: string | null = null;
+function getWorkletModuleUrl(): string {
+  if (workletModuleUrl) return workletModuleUrl;
+  const blob = new Blob([WORKLET_SOURCE], { type: 'application/javascript' });
+  workletModuleUrl = URL.createObjectURL(blob);
+  return workletModuleUrl;
+}
+
+export async function startSystemAudioCapture(): Promise<SystemAudioHandle> {
+  if (!isTauriRuntime()) {
+    throw new SystemAudioUnavailable('not a tauri runtime');
+  }
+  const AudioCtor: typeof AudioContext | undefined =
+    typeof window !== 'undefined'
+      ? (window.AudioContext ??
+          (window as unknown as { webkitAudioContext?: typeof AudioContext })
+            .webkitAudioContext)
+      : undefined;
+  if (!AudioCtor) {
+    throw new SystemAudioUnavailable('WebAudio unavailable');
+  }
+
+  // Pin to 48kHz so the worklet's input rate matches the Rust-side
+  // output rate. If the OS forces a different rate the constructor
+  // throws on some browsers; we catch and surface as Unavailable so the
+  // caller can fall back.
+  let ctx: AudioContext;
+  try {
+    ctx = new AudioCtor({ sampleRate: 48000, latencyHint: 'interactive' });
+  } catch (err: unknown) {
+    throw new SystemAudioUnavailable(
+      err instanceof Error ? err.message : String(err),
+    );
+  }
+
+  try {
+    await ctx.audioWorklet.addModule(getWorkletModuleUrl());
+  } catch (err: unknown) {
+    await ctx.close().catch(() => undefined);
+    throw new SystemAudioUnavailable(
+      'audioWorklet load failed: ' +
+        (err instanceof Error ? err.message : String(err)),
+    );
+  }
+
+  const node = new AudioWorkletNode(ctx, 'screen-audio-loopback', {
+    numberOfInputs: 0,
+    numberOfOutputs: 1,
+    outputChannelCount: [2],
+  });
+  const dest = ctx.createMediaStreamDestination();
+  node.connect(dest);
+
+  // Kick the AudioContext out of `suspended` before any samples arrive —
+  // the share is triggered from a user click so autoplay policy allows
+  // this, and an un-resumed context would buffer everything the Rust
+  // side produces until the context eventually runs, giving seconds of
+  // initial latency.
+  if (ctx.state !== 'running') {
+    try {
+      await ctx.resume();
+    } catch (err: unknown) {
+      console.warn('system-audio ctx.resume failed', err);
+    }
+  }
+
+  const { Channel, invoke } = await import('@tauri-apps/api/core');
+  const channel = new Channel<AudioFramePayload>();
+
+  channel.onmessage = (frame: AudioFramePayload) => {
+    const bytes = base64ToBytes(frame.samplesBase64);
+    // Re-view the bytes as f32 little-endian. The byteLength is always
+    // a multiple of 8 (f32 stereo pairs) — if not, drop the trailing
+    // partial frame rather than risk a truncation artifact.
+    const sampleCount = Math.floor(bytes.byteLength / 4);
+    if (sampleCount < 2) return;
+    const floats = new Float32Array(
+      bytes.buffer,
+      bytes.byteOffset,
+      sampleCount,
+    );
+    // Interleaved L/R → deinterleaved for the worklet. Copying out of
+    // the base64 view also ensures the Float32Arrays we postMessage are
+    // owned (the underlying buffer is about to be garbage-collected).
+    const frames = floats.length >> 1;
+    const left = new Float32Array(frames);
+    const right = new Float32Array(frames);
+    for (let i = 0; i < frames; i++) {
+      left[i] = floats[i * 2] ?? 0;
+      right[i] = floats[i * 2 + 1] ?? 0;
+    }
+    // Transfer the buffers so postMessage is zero-copy.
+    node.port.postMessage(
+      { left, right },
+      [left.buffer, right.buffer],
+    );
+  };
+
+  let captureId: number;
+  try {
+    captureId = await invoke<number>('start_system_audio_capture', { channel });
+  } catch (err: unknown) {
+    node.disconnect();
+    await ctx.close().catch(() => undefined);
+    throw new SystemAudioUnavailable(
+      err instanceof Error ? err.message : String(err),
+    );
+  }
+
+  const stream = dest.stream;
+
+  let stopped = false;
+  const stop = async (): Promise<void> => {
+    if (stopped) return;
+    stopped = true;
+    try {
+      await invoke('stop_system_audio_capture', { captureId });
+    } catch (err: unknown) {
+      console.warn('stop_system_audio_capture failed', err);
+    }
+    try {
+      node.disconnect();
+    } catch {
+      /* already disconnected */
+    }
+    for (const track of stream.getTracks()) {
+      try {
+        track.stop();
+      } catch {
+        /* already stopped */
+      }
+    }
+    await ctx.close().catch(() => undefined);
+  };
+
+  return { captureId, stream, stop };
+}
+
+function base64ToBytes(b64: string): Uint8Array {
+  const bin = atob(b64);
+  const bytes = new Uint8Array(bin.length);
+  for (let i = 0; i < bin.length; i++) {
+    bytes[i] = bin.charCodeAt(i);
+  }
+  return bytes;
+}