feat(desktop): port v0.11.4-v0.15.2 from Tauri to Electron + Discord-parity audio (v0.16.0)
Chronological port of every Tauri release commit (v0.11.4 -> v0.15.2)
into the Electron rebuild, plus Discord-style audio handling that goes
beyond the Tauri original.
Highlights:
- All 17 Tauri release commits ported (audio fixes, custom notification
sound, Discord-style chat UX, profile banner, changelog page,
Discord-parity call UX, screen-share echo + re-watch UX, audio-loop
fix).
- Native napi-rs audio-loopback addon with WASAPI process-loopback:
* EXCLUDE_TARGET_PROCESS_TREE for full-screen shares -> peers
never hear themselves echoed back through the capture.
* INCLUDE_TARGET_PROCESS_TREE for window shares -> only the
picked window's audio is captured, not the whole OS mixer
(Discord parity).
* HWND -> PID resolution via Win32 GetWindowThreadProcessId.
- Discord-style screen-source picker (thumbnail grid, screens vs
apps tabs, live-refreshing thumbnails).
- Hash routing fix for packaged builds (file:// can't resolve
BrowserRouter paths).
- Tauri sources removed (apps/desktop/src-tauri).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,30 +1,25 @@
|
||||
// Frontend side of the native system-audio pipeline. Pairs with the Rust
|
||||
// `screen_audio` module: it opens a Tauri Channel, receives interleaved
|
||||
// f32 stereo samples at 48kHz (base64-encoded), and surfaces them as a
|
||||
// real `MediaStream` that LiveKit can publish as a `ScreenShareAudio`
|
||||
// track. An AudioWorklet does the heavy lifting so the render thread is
|
||||
// never the bottleneck — the main thread just pushes decoded samples
|
||||
// across a port; the worklet copies them into its output buffer which
|
||||
// feeds a `MediaStreamDestination`.
|
||||
// System-audio loopback. Under Electron this is renderer-driven: we
|
||||
// ask main for the primary screen's capturer id, then call
|
||||
// getUserMedia with Chromium's `chromeMediaSource: 'desktop'`
|
||||
// constraint to obtain the OS-mixer MediaStream directly. The whole
|
||||
// Tauri WASAPI + AudioWorklet base64 pipeline is gone.
|
||||
//
|
||||
// Windows-only right now. On other platforms `startSystemAudioCapture`
|
||||
// throws `SystemAudioUnavailable` and the caller is expected to fall
|
||||
// back to the browser's getDisplayMedia path.
|
||||
// Windows-only in practice (loopback audio is a Windows feature of
|
||||
// Chromium's desktop source). On other platforms `startSystemAudioCapture`
|
||||
// throws `SystemAudioUnavailable`; callers are expected to fall back
|
||||
// to the standard getDisplayMedia flow.
|
||||
|
||||
import { isTauriRuntime } from './globalShortcut';
|
||||
|
||||
export interface SystemAudioHandle {
|
||||
/** Rust-side capture id. Pass to the Rust stop command via `stop()`. */
|
||||
/** Monotonic id, used by callers to correlate stop() with start. */
|
||||
captureId: number;
|
||||
/** MediaStream carrying a single audio track at 48kHz stereo. */
|
||||
/** MediaStream with a single audio track carrying the OS mixer. */
|
||||
stream: MediaStream;
|
||||
/** Teardown — stops the Rust thread, closes the AudioContext, ends the
|
||||
* MediaStreamDestination track. Idempotent. */
|
||||
/** Teardown — stops the MediaStreamTrack. Idempotent. */
|
||||
stop: () => Promise<void>;
|
||||
}
|
||||
|
||||
/** Thrown when the platform can't deliver native system-audio (non-Tauri
|
||||
* runtime, non-Windows host, WebAudio unavailable, COM init failure). */
|
||||
export class SystemAudioUnavailable extends Error {
|
||||
constructor(reason: string) {
|
||||
super('system audio unavailable: ' + reason);
|
||||
@@ -32,211 +27,58 @@ export class SystemAudioUnavailable extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
interface AudioFramePayload {
|
||||
captureId: number;
|
||||
sampleRate: number;
|
||||
channels: number;
|
||||
samplesBase64: string;
|
||||
}
|
||||
|
||||
// AudioWorklet source embedded as a string. The worklet keeps a pair of
|
||||
// ring buffers (one per channel) that the main thread appends to as
|
||||
// samples arrive. `process()` drains the ring buffers into the output
|
||||
// blocks; an underrun emits silence instead of propagating the stall
|
||||
// upwards (a glitch is better than a freeze for LiveKit's Opus encoder).
|
||||
//
|
||||
// The worklet runs at AudioContext sample rate, which we pin to 48kHz via
|
||||
// the AudioContext constructor. That matches what the Rust side already
|
||||
// resamples to, so no further rate conversion is needed here.
|
||||
const WORKLET_SOURCE = `
|
||||
class LoopbackAudioProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
// Ring buffer sized for latency, not for "never drop". 300ms hard cap,
|
||||
// 80ms target — we aim for ~one WASAPI packet of headroom above the
|
||||
// render quantum and drop excess whenever the producer gets ahead.
|
||||
// Keeping the target small is the difference between "feels live" and
|
||||
// "laggy" for screen-share audio.
|
||||
this.bufferSize = 48000 * 0.3 | 0;
|
||||
this.targetFrames = 48000 * 0.08 | 0;
|
||||
this.bufL = new Float32Array(this.bufferSize);
|
||||
this.bufR = new Float32Array(this.bufferSize);
|
||||
this.writePos = 0;
|
||||
this.readPos = 0;
|
||||
this.available = 0;
|
||||
this.port.onmessage = (e) => {
|
||||
const { left, right } = e.data;
|
||||
const len = left.length;
|
||||
for (let i = 0; i < len; i++) {
|
||||
this.bufL[this.writePos] = left[i];
|
||||
this.bufR[this.writePos] = right[i];
|
||||
this.writePos = (this.writePos + 1) % this.bufferSize;
|
||||
if (this.available < this.bufferSize) {
|
||||
this.available++;
|
||||
} else {
|
||||
// Buffer full — advance the read cursor to keep writing.
|
||||
this.readPos = (this.readPos + 1) % this.bufferSize;
|
||||
}
|
||||
}
|
||||
// Hard cap: if we're this far behind the producer, skip ahead to
|
||||
// the target latency instead of playing out minutes of stale audio.
|
||||
// Happens on: AudioContext resume after suspend, tab throttle
|
||||
// recovery, any hiccup that left samples piling up.
|
||||
if (this.available > this.targetFrames * 3) {
|
||||
const drop = this.available - this.targetFrames;
|
||||
this.readPos = (this.readPos + drop) % this.bufferSize;
|
||||
this.available -= drop;
|
||||
}
|
||||
};
|
||||
}
|
||||
process(_inputs, outputs) {
|
||||
const output = outputs[0];
|
||||
if (!output || output.length === 0) return true;
|
||||
const out0 = output[0];
|
||||
const out1 = output[1] || output[0];
|
||||
const n = out0.length;
|
||||
for (let i = 0; i < n; i++) {
|
||||
if (this.available > 0) {
|
||||
out0[i] = this.bufL[this.readPos];
|
||||
if (out1 !== out0) out1[i] = this.bufR[this.readPos];
|
||||
this.readPos = (this.readPos + 1) % this.bufferSize;
|
||||
this.available--;
|
||||
} else {
|
||||
out0[i] = 0;
|
||||
if (out1 !== out0) out1[i] = 0;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('screen-audio-loopback', LoopbackAudioProcessor);
|
||||
`;
|
||||
|
||||
let workletModuleUrl: string | null = null;
|
||||
function getWorkletModuleUrl(): string {
|
||||
if (workletModuleUrl) return workletModuleUrl;
|
||||
const blob = new Blob([WORKLET_SOURCE], { type: 'application/javascript' });
|
||||
workletModuleUrl = URL.createObjectURL(blob);
|
||||
return workletModuleUrl;
|
||||
interface ChromiumAudioConstraint {
|
||||
mandatory: {
|
||||
chromeMediaSource: 'desktop';
|
||||
chromeMediaSourceId: string;
|
||||
};
|
||||
}
|
||||
|
||||
export async function startSystemAudioCapture(): Promise<SystemAudioHandle> {
|
||||
if (!isTauriRuntime()) {
|
||||
throw new SystemAudioUnavailable('not a tauri runtime');
|
||||
throw new SystemAudioUnavailable('not an electron runtime');
|
||||
}
|
||||
const AudioCtor: typeof AudioContext | undefined =
|
||||
typeof window !== 'undefined'
|
||||
? (window.AudioContext ??
|
||||
(window as unknown as { webkitAudioContext?: typeof AudioContext })
|
||||
.webkitAudioContext)
|
||||
: undefined;
|
||||
if (!AudioCtor) {
|
||||
throw new SystemAudioUnavailable('WebAudio unavailable');
|
||||
if (typeof navigator === 'undefined' || !navigator.mediaDevices) {
|
||||
throw new SystemAudioUnavailable('mediaDevices unavailable');
|
||||
}
|
||||
|
||||
// Pin to 48kHz so the worklet's input rate matches the Rust-side
|
||||
// output rate. If the OS forces a different rate the constructor
|
||||
// throws on some browsers; we catch and surface as Unavailable so the
|
||||
// caller can fall back.
|
||||
let ctx: AudioContext;
|
||||
try {
|
||||
ctx = new AudioCtor({ sampleRate: 48000, latencyHint: 'interactive' });
|
||||
} catch (err: unknown) {
|
||||
throw new SystemAudioUnavailable(
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
const resolved = await window.electronAPI.resolveLoopbackSource();
|
||||
if (!resolved) {
|
||||
throw new SystemAudioUnavailable('no screen source available');
|
||||
}
|
||||
|
||||
try {
|
||||
await ctx.audioWorklet.addModule(getWorkletModuleUrl());
|
||||
} catch (err: unknown) {
|
||||
await ctx.close().catch(() => undefined);
|
||||
throw new SystemAudioUnavailable(
|
||||
'audioWorklet load failed: ' +
|
||||
(err instanceof Error ? err.message : String(err)),
|
||||
);
|
||||
}
|
||||
|
||||
const node = new AudioWorkletNode(ctx, 'screen-audio-loopback', {
|
||||
numberOfInputs: 0,
|
||||
numberOfOutputs: 1,
|
||||
outputChannelCount: [2],
|
||||
});
|
||||
const dest = ctx.createMediaStreamDestination();
|
||||
node.connect(dest);
|
||||
|
||||
// Kick the AudioContext out of `suspended` before any samples arrive —
|
||||
// the share is triggered from a user click so autoplay policy allows
|
||||
// this, and an un-resumed context would buffer everything the Rust
|
||||
// side produces until the context eventually runs, giving seconds of
|
||||
// initial latency.
|
||||
if (ctx.state !== 'running') {
|
||||
try {
|
||||
await ctx.resume();
|
||||
} catch (err: unknown) {
|
||||
console.warn('system-audio ctx.resume failed', err);
|
||||
}
|
||||
}
|
||||
|
||||
const { Channel, invoke } = await import('@tauri-apps/api/core');
|
||||
const channel = new Channel<AudioFramePayload>();
|
||||
|
||||
channel.onmessage = (frame: AudioFramePayload) => {
|
||||
const bytes = base64ToBytes(frame.samplesBase64);
|
||||
// Re-view the bytes as f32 little-endian. The byteLength is always
|
||||
// a multiple of 8 (f32 stereo pairs) — if not, drop the trailing
|
||||
// partial frame rather than risk a truncation artifact.
|
||||
const sampleCount = Math.floor(bytes.byteLength / 4);
|
||||
if (sampleCount < 2) return;
|
||||
const floats = new Float32Array(
|
||||
bytes.buffer,
|
||||
bytes.byteOffset,
|
||||
sampleCount,
|
||||
);
|
||||
// Interleaved L/R → deinterleaved for the worklet. Copying out of
|
||||
// the base64 view also ensures the Float32Arrays we postMessage are
|
||||
// owned (the underlying buffer is about to be garbage-collected).
|
||||
const frames = floats.length >> 1;
|
||||
const left = new Float32Array(frames);
|
||||
const right = new Float32Array(frames);
|
||||
for (let i = 0; i < frames; i++) {
|
||||
left[i] = floats[i * 2] ?? 0;
|
||||
right[i] = floats[i * 2 + 1] ?? 0;
|
||||
}
|
||||
// Transfer the buffers so postMessage is zero-copy.
|
||||
node.port.postMessage(
|
||||
{ left, right },
|
||||
[left.buffer, right.buffer],
|
||||
);
|
||||
const audioConstraint: ChromiumAudioConstraint = {
|
||||
mandatory: {
|
||||
chromeMediaSource: 'desktop',
|
||||
chromeMediaSourceId: resolved.sourceId,
|
||||
},
|
||||
};
|
||||
|
||||
let captureId: number;
|
||||
let stream: MediaStream;
|
||||
try {
|
||||
captureId = await invoke<number>('start_system_audio_capture', { channel });
|
||||
// Cast: the Chromium `mandatory` constraint is non-standard and
|
||||
// not covered by lib.dom.d.ts typings.
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: audioConstraint as unknown as MediaTrackConstraints,
|
||||
video: false,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
node.disconnect();
|
||||
await ctx.close().catch(() => undefined);
|
||||
throw new SystemAudioUnavailable(
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
}
|
||||
|
||||
const stream = dest.stream;
|
||||
const tracks = stream.getAudioTracks();
|
||||
if (tracks.length === 0) {
|
||||
for (const t of stream.getTracks()) t.stop();
|
||||
throw new SystemAudioUnavailable('no audio track in returned stream');
|
||||
}
|
||||
|
||||
const captureId = Date.now();
|
||||
let stopped = false;
|
||||
const stop = async (): Promise<void> => {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
try {
|
||||
await invoke('stop_system_audio_capture', { captureId });
|
||||
} catch (err: unknown) {
|
||||
console.warn('stop_system_audio_capture failed', err);
|
||||
}
|
||||
try {
|
||||
node.disconnect();
|
||||
} catch {
|
||||
/* already disconnected */
|
||||
}
|
||||
for (const track of stream.getTracks()) {
|
||||
try {
|
||||
track.stop();
|
||||
@@ -244,17 +86,7 @@ export async function startSystemAudioCapture(): Promise<SystemAudioHandle> {
|
||||
/* already stopped */
|
||||
}
|
||||
}
|
||||
await ctx.close().catch(() => undefined);
|
||||
};
|
||||
|
||||
return { captureId, stream, stop };
|
||||
}
|
||||
|
||||
function base64ToBytes(b64: string): Uint8Array {
|
||||
const bin = atob(b64);
|
||||
const bytes = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) {
|
||||
bytes[i] = bin.charCodeAt(i);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user