diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lock index 4554a46..2ae90fc 100644 --- a/apps/desktop/src-tauri/Cargo.lock +++ b/apps/desktop/src-tauri/Cargo.lock @@ -852,6 +852,7 @@ dependencies = [ "tauri-plugin-updater", "tauri-plugin-window-state", "tokio", + "wasapi", "windows 0.58.0", "xcap", ] @@ -8125,6 +8126,19 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasapi" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6b03b82e419f186fcdc06ac6068621bdadc88b89b2612067f1c021ad2c9449" +dependencies = [ + "log", + "num-integer", + "widestring", + "windows 0.57.0", + "windows-core 0.57.0", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -8431,6 +8445,12 @@ dependencies = [ "wasite", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 80445a1..86e6b51 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -57,6 +57,13 @@ windows = { version = "0.58", features = [ "Win32_UI_WindowsAndMessaging", ] } +# WASAPI loopback capture for system-audio screen-share. Lets the custom +# picker hand LiveKit a real audio track without falling back to the OS +# screen picker (which is the only way getDisplayMedia can grab system +# sound). Windows-only for v1; macOS needs ScreenCaptureKit-audio and +# Linux needs a PulseAudio / PipeWire path. +wasapi = "0.15" + # LiveKit client SDK — lives behind the `rust-livekit` feature flag so the # baseline build stays unaffected while the JS-SDK path is still the # default. Pulls libwebrtc-rs which adds ~20MB to the binary and ~5-10min diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 14e1730..fc93957 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -1,4 +1,5 @@ mod crypto; +mod screen_audio; mod screen_capture; mod screen_sources; @@ -99,6 +100,8 @@ pub fn run() { screen_sources::capture_screen_source_thumbnail_bytes, screen_capture::start_screen_capture, screen_capture::stop_screen_capture, + screen_audio::start_system_audio_capture, + screen_audio::stop_system_audio_capture, ]) .plugin(tauri_plugin_notification::init()); @@ -121,6 +124,8 @@ pub fn run() { screen_sources::capture_screen_source_thumbnail_bytes, screen_capture::start_screen_capture, screen_capture::stop_screen_capture, + screen_audio::start_system_audio_capture, + screen_audio::stop_system_audio_capture, livekit_bridge::livekit_connect, livekit_bridge::livekit_disconnect, livekit_bridge::livekit_send_data, diff --git a/apps/desktop/src-tauri/src/screen_audio.rs b/apps/desktop/src-tauri/src/screen_audio.rs new file mode 100644 index 0000000..ee62151 --- /dev/null +++ b/apps/desktop/src-tauri/src/screen_audio.rs @@ -0,0 +1,428 @@ +// Native system-audio capture for the custom screen-share picker. Without +// this path the picker has to fall back to getDisplayMedia whenever the +// user ticks "Mit System-Sound", because Chromium only wires audio into +// desktop captures that the OS picker produced. Here we grab the default +// render endpoint's loopback stream via WASAPI, convert it to 48kHz f32 +// stereo, and ship the samples to the JS side through a Tauri Channel. +// An AudioWorklet on the frontend feeds them into a MediaStreamDestination +// so LiveKit publishes a plain ScreenShareAudio track. +// +// Windows-only for v1. macOS + Linux stubs return a clear error so the +// frontend can fall back cleanly on those platforms until their native +// paths ship (ScreenCaptureKit-audio / PipeWire). + +#![allow(clippy::needless_return)] + +use base64::Engine; +use serde::Serialize; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::thread; +use tauri::ipc::Channel; + +// Output format we always deliver to the frontend. Picking a single fixed +// format means the AudioWorklet never has to renegotiate — it just assumes +// interleaved f32 stereo at 48kHz. WASAPI mix format is usually already +// this on Windows 10+, so the resample branch is rarely hit. +const OUTPUT_SAMPLE_RATE: u32 = 48_000; +const OUTPUT_CHANNELS: u16 = 2; + +static NEXT_ID: AtomicU32 = AtomicU32::new(1); +static SESSIONS: OnceLock>> = OnceLock::new(); + +fn sessions() -> &'static Mutex> { + SESSIONS.get_or_init(|| Mutex::new(HashMap::new())) +} + +struct Session { + stop: Arc, + handle: Option>, +} + +#[derive(Serialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct AudioFramePayload { + pub capture_id: u32, + pub sample_rate: u32, + pub channels: u16, + /// Interleaved little-endian f32 stereo samples, base64-encoded. + /// Frontend decodes via `atob` → `Uint8Array` → `Float32Array` view. + /// Base64 is used instead of a raw `Vec` because Tauri Channel + /// serialises via JSON — a JSON array of floats balloons to ~2–3× + /// the byte count, and at 48kHz stereo that's enough IPC traffic + /// to matter. + pub samples_base64: String, +} + +/// Start a loopback capture of the default render endpoint and begin +/// streaming audio frames on the provided channel. Returns a numeric +/// capture id that must be handed to `stop_system_audio_capture` when +/// the share ends. +#[tauri::command] +pub fn start_system_audio_capture( + channel: Channel, +) -> Result { + #[cfg(target_os = "windows")] + { + let capture_id = NEXT_ID.fetch_add(1, Ordering::Relaxed); + let stop = Arc::new(AtomicBool::new(false)); + let stop_clone = Arc::clone(&stop); + + let handle = thread::Builder::new() + .name(format!("screen-audio-{capture_id}")) + .spawn(move || { + if let Err(err) = + windows_loopback::capture_loop(capture_id, channel, stop_clone) + { + eprintln!("screen-audio {capture_id}: {err}"); + } + }) + .map_err(|e| format!("failed to spawn audio thread: {e}"))?; + + sessions().lock().unwrap().insert( + capture_id, + Session { + stop, + handle: Some(handle), + }, + ); + Ok(capture_id) + } + + #[cfg(not(target_os = "windows"))] + { + // Keep the `channel` binding alive so Tauri doesn't complain about + // an unused parameter on the non-Windows build. + let _ = channel; + Err("system audio capture only supported on Windows".into()) + } +} + +/// Tear down the capture for the given id. Safe to call on a missing id +/// (no-op) so the JS side doesn't have to track whether the stop has +/// already been issued by the screen-share teardown path. +#[tauri::command] +pub fn stop_system_audio_capture(capture_id: u32) -> Result<(), String> { + let session = sessions().lock().unwrap().remove(&capture_id); + let Some(mut session) = session else { + return Ok(()); + }; + session.stop.store(true, Ordering::Relaxed); + if let Some(handle) = session.handle.take() { + // Best-effort join — the capture loop polls `stop` every event + // cycle (≤100ms) so this usually returns promptly. If the WASAPI + // call is wedged we'd rather drop the handle than hang the stop. + let _ = handle.join(); + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Windows loopback implementation +// --------------------------------------------------------------------------- + +#[cfg(target_os = "windows")] +mod windows_loopback { + use super::*; + use wasapi::{initialize_mta, Direction, SampleType, ShareMode}; + + pub fn capture_loop( + capture_id: u32, + channel: Channel, + stop: Arc, + ) -> Result<(), String> { + // COM must be initialised on every thread that touches WASAPI. + // MTA is the right model for a background capture thread — STA + // would require message pumping we don't want to add. + initialize_mta() + .ok() + .map_err(|e| format!("initialize_mta: {e:?}"))?; + + let device = wasapi::get_default_device(&Direction::Render) + .map_err(|e| format!("get_default_device: {e:?}"))?; + let mut audio_client = device + .get_iaudioclient() + .map_err(|e| format!("get_iaudioclient: {e:?}"))?; + + // Use the mix format that Windows is already pushing to the + // endpoint. Loopback capture won't convert for us — asking for a + // fixed format here makes Initialize() fail on non-matching + // hardware. We resample + channel-mix ourselves downstream. + let mix_format = audio_client + .get_mixformat() + .map_err(|e| format!("get_mixformat: {e:?}"))?; + let input_rate = mix_format.get_samplespersec(); + let input_channels = mix_format.get_nchannels(); + let bits_per_sample = mix_format.get_bitspersample(); + let block_align = mix_format.get_blockalign(); + let sample_type = mix_format.get_subformat().unwrap_or(SampleType::Int); + + let (def_time, _min_time) = audio_client + .get_periods() + .map_err(|e| format!("get_periods: {e:?}"))?; + + // Direction::Capture + loopback: WASAPI streams what Windows is + // sending to the speakers instead of what an input device is + // producing. Shared mode so we coexist with other apps. + audio_client + .initialize_client( + &mix_format, + def_time, + &Direction::Capture, + &ShareMode::Shared, + true, + ) + .map_err(|e| format!("initialize_client: {e:?}"))?; + + let h_event = audio_client + .set_get_eventhandle() + .map_err(|e| format!("set_get_eventhandle: {e:?}"))?; + let capture_client = audio_client + .get_audiocaptureclient() + .map_err(|e| format!("get_audiocaptureclient: {e:?}"))?; + + audio_client + .start_stream() + .map_err(|e| format!("start_stream: {e:?}"))?; + + // Resampler state — last stereo frame from the previous buffer so + // linear interpolation at the buffer boundary doesn't click. + // Initialised to silence. + let mut last_stereo: [f32; 2] = [0.0, 0.0]; + + while !stop.load(Ordering::Relaxed) { + // 100ms timeout lets the loop check the stop flag even when + // the endpoint is silent (WASAPI doesn't signal the event at + // all for pure-silence streams on some driver versions). + if h_event.wait_for_event(100).is_err() { + continue; + } + + // Drain all packets available since the last wake — there + // can be several queued if we were preempted. + loop { + if stop.load(Ordering::Relaxed) { + break; + } + let frames_available = match capture_client.get_next_nbr_frames() { + Ok(Some(n)) if n > 0 => n, + Ok(_) => break, + Err(e) => { + eprintln!( + "screen-audio {capture_id}: get_next_nbr_frames: {e:?}" + ); + break; + } + }; + + let bytes_needed = + frames_available as usize * block_align as usize; + let mut raw = vec![0u8; bytes_needed]; + if let Err(e) = capture_client.read_from_device(&mut raw) { + eprintln!( + "screen-audio {capture_id}: read_from_device: {e:?}" + ); + break; + } + + // Decode PCM into interleaved f32 at the device's native + // rate + channel count. + let decoded = decode_pcm( + &raw, + input_channels, + bits_per_sample, + &sample_type, + ); + + // Channel-fold → 2ch, then resample → 48kHz. + let stereo = to_stereo(&decoded, input_channels); + let out_samples = resample_linear_stereo( + &stereo, + input_rate, + OUTPUT_SAMPLE_RATE, + &mut last_stereo, + ); + + if out_samples.is_empty() { + continue; + } + + // Pack f32s as little-endian bytes then base64. IPC-wise + // this is ~1.3× the raw byte count versus 5–10× for a + // JSON array of floats, which is the difference between + // "fine" and "wastes a CPU core" at 48kHz stereo. + let mut bytes = Vec::with_capacity(out_samples.len() * 4); + for s in &out_samples { + bytes.extend_from_slice(&s.to_le_bytes()); + } + let samples_b64 = + base64::engine::general_purpose::STANDARD.encode(&bytes); + + if channel + .send(AudioFramePayload { + capture_id, + sample_rate: OUTPUT_SAMPLE_RATE, + channels: OUTPUT_CHANNELS, + samples_base64: samples_b64, + }) + .is_err() + { + // Frontend went away — stop cleanly. + stop.store(true, Ordering::Relaxed); + break; + } + } + } + + let _ = audio_client.stop_stream(); + Ok(()) + } + + // Convert a raw WASAPI buffer into interleaved f32 at the device's + // native channel count. Handles the three formats that actually show + // up on Windows render endpoints: f32 (most modern hardware), i16 + // (older onboard codecs), and i32 (pro audio interfaces). Anything + // else falls through to zeros so a weird format doesn't crash the + // share — the user will notice silence and can retry. + fn decode_pcm( + raw: &[u8], + channels: u16, + bits_per_sample: u16, + sample_type: &SampleType, + ) -> Vec { + match (sample_type, bits_per_sample) { + (SampleType::Float, 32) => { + let mut out = Vec::with_capacity(raw.len() / 4); + for chunk in raw.chunks_exact(4) { + out.push(f32::from_le_bytes([ + chunk[0], chunk[1], chunk[2], chunk[3], + ])); + } + out + } + (SampleType::Int, 16) => { + let scale = 1.0_f32 / (i16::MAX as f32); + let mut out = Vec::with_capacity(raw.len() / 2); + for chunk in raw.chunks_exact(2) { + let s = i16::from_le_bytes([chunk[0], chunk[1]]); + out.push(s as f32 * scale); + } + out + } + (SampleType::Int, 32) => { + let scale = 1.0_f32 / (i32::MAX as f32); + let mut out = Vec::with_capacity(raw.len() / 4); + for chunk in raw.chunks_exact(4) { + let s = i32::from_le_bytes([ + chunk[0], chunk[1], chunk[2], chunk[3], + ]); + out.push(s as f32 * scale); + } + out + } + _ => { + // Unknown format — emit silence of the right frame count + // so downstream math stays correct. + let bytes_per_frame = + (bits_per_sample as usize / 8) * channels as usize; + let frames = if bytes_per_frame == 0 { + 0 + } else { + raw.len() / bytes_per_frame + }; + vec![0.0; frames * channels as usize] + } + } + } + + // Down- or up-mix to stereo. Surround layouts fold L+R only (center + // + surrounds get dropped) which is the simplest defensible choice + // for screen-share audio — most content is LR-centric and a proper + // ITU-R BS.775 downmix would pull in matrix coefficients we'd rather + // avoid in v1. + fn to_stereo(interleaved: &[f32], channels: u16) -> Vec { + if channels == 0 || interleaved.is_empty() { + return Vec::new(); + } + if channels == 2 { + return interleaved.to_vec(); + } + let ch = channels as usize; + let frames = interleaved.len() / ch; + let mut out = Vec::with_capacity(frames * 2); + if channels == 1 { + for i in 0..frames { + let s = interleaved[i]; + out.push(s); + out.push(s); + } + } else { + for i in 0..frames { + let base = i * ch; + out.push(interleaved[base]); + out.push(interleaved[base + 1]); + } + } + out + } + + // Linear-interpolation resampler for interleaved stereo f32. Not the + // prettiest option theoretically, but at 44.1→48 the audible + // artefacts stay below threshold for speech + game/music content. The + // `last_stereo` state preserves the final frame across invocations so + // the interpolation at the buffer boundary doesn't produce a click. + fn resample_linear_stereo( + input_stereo: &[f32], + input_rate: u32, + output_rate: u32, + last_stereo: &mut [f32; 2], + ) -> Vec { + if input_stereo.is_empty() { + return Vec::new(); + } + if input_rate == output_rate { + last_stereo[0] = input_stereo[input_stereo.len() - 2]; + last_stereo[1] = input_stereo[input_stereo.len() - 1]; + return input_stereo.to_vec(); + } + let ratio = output_rate as f64 / input_rate as f64; + let in_frames = input_stereo.len() / 2; + let out_frames = (in_frames as f64 * ratio).floor() as usize; + if out_frames == 0 { + last_stereo[0] = input_stereo[input_stereo.len() - 2]; + last_stereo[1] = input_stereo[input_stereo.len() - 1]; + return Vec::new(); + } + let mut out = Vec::with_capacity(out_frames * 2); + + let prev_l = last_stereo[0]; + let prev_r = last_stereo[1]; + + for i in 0..out_frames { + let src_pos = i as f64 / ratio; + let src_frame = src_pos.floor() as i64; + let frac = (src_pos - src_frame as f64) as f32; + + // `src_frame == -1` comes up for the very first output frame + // when ratio > 1 — interpolate against the previous buffer's + // final sample to bridge the two. + let (l0, r0) = if src_frame < 0 { + (prev_l, prev_r) + } else { + let idx = (src_frame as usize).min(in_frames - 1) * 2; + (input_stereo[idx], input_stereo[idx + 1]) + }; + let next = ((src_frame + 1) as usize).min(in_frames - 1); + let l1 = input_stereo[next * 2]; + let r1 = input_stereo[next * 2 + 1]; + + out.push(l0 + (l1 - l0) * frac); + out.push(r0 + (r1 - r0) * frac); + } + + last_stereo[0] = input_stereo[input_stereo.len() - 2]; + last_stereo[1] = input_stereo[input_stereo.len() - 1]; + out + } +} diff --git a/apps/desktop/src/components/ScreenSourcePicker.tsx b/apps/desktop/src/components/ScreenSourcePicker.tsx index 567e155..d9a5f1d 100644 --- a/apps/desktop/src/components/ScreenSourcePicker.tsx +++ b/apps/desktop/src/components/ScreenSourcePicker.tsx @@ -282,14 +282,6 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) { - {includeAudio && selectedId && ( -

- {t('app:call.share_audio_uses_os_picker', { - defaultValue: - 'Mit System-Sound fragt der Browser noch einmal nach der Quelle — Video-Direktpfad geht nur ohne Audio.', - })} -

- )} {error && (

{error} diff --git a/apps/desktop/src/context/CallContext.tsx b/apps/desktop/src/context/CallContext.tsx index 6b2664b..6b19a8c 100644 --- a/apps/desktop/src/context/CallContext.tsx +++ b/apps/desktop/src/context/CallContext.tsx @@ -80,6 +80,10 @@ import { NativeCaptureUnavailable, startNativeCapture, } from '../lib/screenCapture'; +import { + type SystemAudioHandle, + startSystemAudioCapture, +} from '../lib/screenAudio'; import { clearScreenShareVolumes, getScreenShareVolume, @@ -305,6 +309,11 @@ export function CallProvider({ children }: { children: ReactNode }) { // on the canvas track's 'ended' event. Not kept in React state because // it never feeds into a render. const nativeCaptureRef = useRef(null); + // Matching handle for the Windows-only WASAPI system-audio capture. + // Lives in lockstep with the video handle above when the user picks + // "Mit System-Sound"; teardown is wired so that stopping either track + // also stops the other, so stale audio can't outlive the video share. + const nativeAudioCaptureRef = useRef(null); const roomRef = useRef(null); // Web Audio graph that mixes live mic + soundboard sources into a single // published track. Created per call in joinRoom, destroyed in disconnectRoom. @@ -486,6 +495,11 @@ export function CallProvider({ children }: { children: ReactNode }) { nativeCaptureRef.current = null; await h.stop().catch(() => undefined); } + if (nativeAudioCaptureRef.current) { + const h = nativeAudioCaptureRef.current; + nativeAudioCaptureRef.current = null; + await h.stop().catch(() => undefined); + } roomRef.current = null; setRoom(null); setRemoteParticipants([]); @@ -1190,24 +1204,25 @@ export function CallProvider({ children }: { children: ReactNode }) { const sourceId = overrides?.sourceId ?? null; // Native capture path — tried first when the user came in via our - // custom picker. xcap on the Rust side grabs frames, streams them as - // JPEG over a Tauri channel, and we draw them onto a canvas whose - // captureStream() becomes the MediaStream LiveKit publishes. This - // completely skips the OS picker. Video-only (no system audio yet); - // if the user asked for audio we fall through to the legacy paths - // below so audio still works via getDisplayMedia. + // custom picker. xcap on the Rust side grabs video frames and, on + // Windows with "Mit System-Sound" on, the WASAPI loopback module + // grabs the render endpoint. Both stream over Tauri channels into + // tracks we publish directly to LiveKit — the OS picker never + // appears. If native audio fails on a platform that can't supply + // it (non-Windows v1), we continue with video-only and log; the + // user still gets their direct-video share. if (!sourceId) { console.info( 'screen-share: no sourceId supplied by picker, OS picker will open', ); - } else if (settings.includeSystemAudio) { - console.info( - 'screen-share: system audio requested — native path unavailable (needs WASAPI/ScreenCaptureKit), OS picker will open', - ); } - if (sourceId && !settings.includeSystemAudio) { + if (sourceId) { try { - console.info('screen-share: trying native capture path', { sourceId, fps }); + console.info('screen-share: trying native capture path', { + sourceId, + fps, + includeSystemAudio: settings.includeSystemAudio, + }); const { Track: LkTrack } = await import('livekit-client'); const maxWidth = ssParams.dims?.width ?? 1920; const maxHeight = ssParams.dims?.height ?? 1080; @@ -1228,9 +1243,54 @@ export function CallProvider({ children }: { children: ReactNode }) { source: LkTrack.Source.ScreenShare, videoCodec: 'vp9', }); + + // Optional native audio. Failure here is non-fatal — the video + // pipeline is already running and bailing out would be worse + // UX than shipping a silent share. The warning surfaces the + // platform gap so the user knows why their audio is missing. + let audioHandle: SystemAudioHandle | null = null; + if (settings.includeSystemAudio) { + try { + audioHandle = await startSystemAudioCapture(); + nativeAudioCaptureRef.current = audioHandle; + const audioMst = audioHandle.stream.getAudioTracks()[0]; + if (audioMst) { + const audioPub = await lp.publishTrack(audioMst, { + source: LkTrack.Source.ScreenShareAudio, + }); + audioMst.addEventListener('ended', () => { + void (async () => { + try { + if (audioPub.track) await lp.unpublishTrack(audioPub.track); + } catch { + /* already unpublished */ + } + const active = nativeAudioCaptureRef.current; + if (active && active.captureId === audioHandle!.captureId) { + nativeAudioCaptureRef.current = null; + await active.stop().catch(() => undefined); + } + })(); + }); + } + } catch (err: unknown) { + console.warn( + 'screen-share: native system-audio unavailable, sharing video only', + err instanceof Error ? err.message : err, + ); + if (nativeAudioCaptureRef.current) { + await nativeAudioCaptureRef.current.stop().catch(() => undefined); + nativeAudioCaptureRef.current = null; + } + audioHandle = null; + } + } + // Canvas stream 'ended' fires on handle.stop() (we track.stop() // each track) — chain unpublish + native teardown so one ended // event cleans everything up regardless of who triggered it. + // Also tear down any paired audio capture so sound can't + // outlive the video share. videoMst.addEventListener('ended', () => { void (async () => { try { @@ -1243,10 +1303,17 @@ export function CallProvider({ children }: { children: ReactNode }) { nativeCaptureRef.current = null; await active.stop().catch(() => undefined); } + const audioActive = nativeAudioCaptureRef.current; + if (audioActive) { + nativeAudioCaptureRef.current = null; + await audioActive.stop().catch(() => undefined); + } setIsScreenSharing(false); })(); }); - console.info('screen-share: native capture active'); + console.info('screen-share: native capture active', { + audio: audioHandle != null, + }); setIsScreenSharing(true); return; } catch (err: unknown) { @@ -1257,6 +1324,10 @@ export function CallProvider({ children }: { children: ReactNode }) { await nativeCaptureRef.current.stop().catch(() => undefined); nativeCaptureRef.current = null; } + if (nativeAudioCaptureRef.current) { + await nativeAudioCaptureRef.current.stop().catch(() => undefined); + nativeAudioCaptureRef.current = null; + } console.warn( 'screen-share: native path failed, falling back', err instanceof Error ? err.message : err, @@ -1403,6 +1474,15 @@ export function CallProvider({ children }: { children: ReactNode }) { console.warn('native capture stop failed', err); } } + if (nativeAudioCaptureRef.current) { + const h = nativeAudioCaptureRef.current; + nativeAudioCaptureRef.current = null; + try { + await h.stop(); + } catch (err: unknown) { + console.warn('native audio capture stop failed', err); + } + } const r = roomRef.current; if (!r) return; const lp = r.localParticipant; @@ -2284,6 +2364,20 @@ function attachTrack( if (isScreenShareAudio) { audio.setAttribute('data-track-source', 'screenshare'); } + // Diagnostic — surfaces source-tag mismatches between SDK versions. + // If a remote participant publishes system-audio but the tag never + // reaches us, `isScreenShareAudio` flips false and the watching + // gate is bypassed; seeing this in the console tells us whether + // the unwanted playback is a gating bug or a tagging mismatch. + console.info('attachTrack:audio', { + participant: participant.identity, + trackSource: track.source, + pubSource: publication.source, + isScreenShareAudio, + watching: participant.identity + ? watchingShareUserIdsMirror.has(participant.identity) + : null, + }); if (participant.identity) { audio.setAttribute('data-participant', participant.identity); } diff --git a/apps/desktop/src/lib/screenAudio.ts b/apps/desktop/src/lib/screenAudio.ts new file mode 100644 index 0000000..4d7ccc2 --- /dev/null +++ b/apps/desktop/src/lib/screenAudio.ts @@ -0,0 +1,260 @@ +// Frontend side of the native system-audio pipeline. Pairs with the Rust +// `screen_audio` module: it opens a Tauri Channel, receives interleaved +// f32 stereo samples at 48kHz (base64-encoded), and surfaces them as a +// real `MediaStream` that LiveKit can publish as a `ScreenShareAudio` +// track. An AudioWorklet does the heavy lifting so the render thread is +// never the bottleneck — the main thread just pushes decoded samples +// across a port; the worklet copies them into its output buffer which +// feeds a `MediaStreamDestination`. +// +// Windows-only right now. On other platforms `startSystemAudioCapture` +// throws `SystemAudioUnavailable` and the caller is expected to fall +// back to the browser's getDisplayMedia path. + +import { isTauriRuntime } from './globalShortcut'; + +export interface SystemAudioHandle { + /** Rust-side capture id. Pass to the Rust stop command via `stop()`. */ + captureId: number; + /** MediaStream carrying a single audio track at 48kHz stereo. */ + stream: MediaStream; + /** Teardown — stops the Rust thread, closes the AudioContext, ends the + * MediaStreamDestination track. Idempotent. */ + stop: () => Promise; +} + +/** Thrown when the platform can't deliver native system-audio (non-Tauri + * runtime, non-Windows host, WebAudio unavailable, COM init failure). */ +export class SystemAudioUnavailable extends Error { + constructor(reason: string) { + super('system audio unavailable: ' + reason); + this.name = 'SystemAudioUnavailable'; + } +} + +interface AudioFramePayload { + captureId: number; + sampleRate: number; + channels: number; + samplesBase64: string; +} + +// AudioWorklet source embedded as a string. The worklet keeps a pair of +// ring buffers (one per channel) that the main thread appends to as +// samples arrive. `process()` drains the ring buffers into the output +// blocks; an underrun emits silence instead of propagating the stall +// upwards (a glitch is better than a freeze for LiveKit's Opus encoder). +// +// The worklet runs at AudioContext sample rate, which we pin to 48kHz via +// the AudioContext constructor. That matches what the Rust side already +// resamples to, so no further rate conversion is needed here. +const WORKLET_SOURCE = ` +class LoopbackAudioProcessor extends AudioWorkletProcessor { + constructor() { + super(); + // Ring buffer sized for latency, not for "never drop". 300ms hard cap, + // 80ms target — we aim for ~one WASAPI packet of headroom above the + // render quantum and drop excess whenever the producer gets ahead. + // Keeping the target small is the difference between "feels live" and + // "laggy" for screen-share audio. + this.bufferSize = 48000 * 0.3 | 0; + this.targetFrames = 48000 * 0.08 | 0; + this.bufL = new Float32Array(this.bufferSize); + this.bufR = new Float32Array(this.bufferSize); + this.writePos = 0; + this.readPos = 0; + this.available = 0; + this.port.onmessage = (e) => { + const { left, right } = e.data; + const len = left.length; + for (let i = 0; i < len; i++) { + this.bufL[this.writePos] = left[i]; + this.bufR[this.writePos] = right[i]; + this.writePos = (this.writePos + 1) % this.bufferSize; + if (this.available < this.bufferSize) { + this.available++; + } else { + // Buffer full — advance the read cursor to keep writing. + this.readPos = (this.readPos + 1) % this.bufferSize; + } + } + // Hard cap: if we're this far behind the producer, skip ahead to + // the target latency instead of playing out minutes of stale audio. + // Happens on: AudioContext resume after suspend, tab throttle + // recovery, any hiccup that left samples piling up. + if (this.available > this.targetFrames * 3) { + const drop = this.available - this.targetFrames; + this.readPos = (this.readPos + drop) % this.bufferSize; + this.available -= drop; + } + }; + } + process(_inputs, outputs) { + const output = outputs[0]; + if (!output || output.length === 0) return true; + const out0 = output[0]; + const out1 = output[1] || output[0]; + const n = out0.length; + for (let i = 0; i < n; i++) { + if (this.available > 0) { + out0[i] = this.bufL[this.readPos]; + if (out1 !== out0) out1[i] = this.bufR[this.readPos]; + this.readPos = (this.readPos + 1) % this.bufferSize; + this.available--; + } else { + out0[i] = 0; + if (out1 !== out0) out1[i] = 0; + } + } + return true; + } +} +registerProcessor('screen-audio-loopback', LoopbackAudioProcessor); +`; + +let workletModuleUrl: string | null = null; +function getWorkletModuleUrl(): string { + if (workletModuleUrl) return workletModuleUrl; + const blob = new Blob([WORKLET_SOURCE], { type: 'application/javascript' }); + workletModuleUrl = URL.createObjectURL(blob); + return workletModuleUrl; +} + +export async function startSystemAudioCapture(): Promise { + if (!isTauriRuntime()) { + throw new SystemAudioUnavailable('not a tauri runtime'); + } + const AudioCtor: typeof AudioContext | undefined = + typeof window !== 'undefined' + ? (window.AudioContext ?? + (window as unknown as { webkitAudioContext?: typeof AudioContext }) + .webkitAudioContext) + : undefined; + if (!AudioCtor) { + throw new SystemAudioUnavailable('WebAudio unavailable'); + } + + // Pin to 48kHz so the worklet's input rate matches the Rust-side + // output rate. If the OS forces a different rate the constructor + // throws on some browsers; we catch and surface as Unavailable so the + // caller can fall back. + let ctx: AudioContext; + try { + ctx = new AudioCtor({ sampleRate: 48000, latencyHint: 'interactive' }); + } catch (err: unknown) { + throw new SystemAudioUnavailable( + err instanceof Error ? err.message : String(err), + ); + } + + try { + await ctx.audioWorklet.addModule(getWorkletModuleUrl()); + } catch (err: unknown) { + await ctx.close().catch(() => undefined); + throw new SystemAudioUnavailable( + 'audioWorklet load failed: ' + + (err instanceof Error ? err.message : String(err)), + ); + } + + const node = new AudioWorkletNode(ctx, 'screen-audio-loopback', { + numberOfInputs: 0, + numberOfOutputs: 1, + outputChannelCount: [2], + }); + const dest = ctx.createMediaStreamDestination(); + node.connect(dest); + + // Kick the AudioContext out of `suspended` before any samples arrive — + // the share is triggered from a user click so autoplay policy allows + // this, and an un-resumed context would buffer everything the Rust + // side produces until the context eventually runs, giving seconds of + // initial latency. + if (ctx.state !== 'running') { + try { + await ctx.resume(); + } catch (err: unknown) { + console.warn('system-audio ctx.resume failed', err); + } + } + + const { Channel, invoke } = await import('@tauri-apps/api/core'); + const channel = new Channel(); + + channel.onmessage = (frame: AudioFramePayload) => { + const bytes = base64ToBytes(frame.samplesBase64); + // Re-view the bytes as f32 little-endian. The byteLength is always + // a multiple of 8 (f32 stereo pairs) — if not, drop the trailing + // partial frame rather than risk a truncation artifact. + const sampleCount = Math.floor(bytes.byteLength / 4); + if (sampleCount < 2) return; + const floats = new Float32Array( + bytes.buffer, + bytes.byteOffset, + sampleCount, + ); + // Interleaved L/R → deinterleaved for the worklet. Copying out of + // the base64 view also ensures the Float32Arrays we postMessage are + // owned (the underlying buffer is about to be garbage-collected). + const frames = floats.length >> 1; + const left = new Float32Array(frames); + const right = new Float32Array(frames); + for (let i = 0; i < frames; i++) { + left[i] = floats[i * 2] ?? 0; + right[i] = floats[i * 2 + 1] ?? 0; + } + // Transfer the buffers so postMessage is zero-copy. + node.port.postMessage( + { left, right }, + [left.buffer, right.buffer], + ); + }; + + let captureId: number; + try { + captureId = await invoke('start_system_audio_capture', { channel }); + } catch (err: unknown) { + node.disconnect(); + await ctx.close().catch(() => undefined); + throw new SystemAudioUnavailable( + err instanceof Error ? err.message : String(err), + ); + } + + const stream = dest.stream; + + let stopped = false; + const stop = async (): Promise => { + if (stopped) return; + stopped = true; + try { + await invoke('stop_system_audio_capture', { captureId }); + } catch (err: unknown) { + console.warn('stop_system_audio_capture failed', err); + } + try { + node.disconnect(); + } catch { + /* already disconnected */ + } + for (const track of stream.getTracks()) { + try { + track.stop(); + } catch { + /* already stopped */ + } + } + await ctx.close().catch(() => undefined); + }; + + return { captureId, stream, stop }; +} + +function base64ToBytes(b64: string): Uint8Array { + const bin = atob(b64); + const bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) { + bytes[i] = bin.charCodeAt(i); + } + return bytes; +}