feat(call): native WASAPI system-audio for screen-share (Windows)
Hooks the custom screen-share picker up to a native WASAPI loopback capture so "Mit System-Sound" no longer falls back to the OS picker on Windows. Rust side opens the default render endpoint, channels 48 kHz f32 stereo to an AudioWorklet, which feeds a MediaStreamDestination for LiveKit to publish as ScreenShareAudio. Ring buffer sized for latency (80 ms target, drop-to-target on overflow) and the AudioContext is resumed eagerly so initial burstiness can't pile up. Adds a temporary attachTrack:audio diagnostic log to confirm source tagging matches between old and new clients. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Generated
+20
@@ -852,6 +852,7 @@ dependencies = [
|
||||
"tauri-plugin-updater",
|
||||
"tauri-plugin-window-state",
|
||||
"tokio",
|
||||
"wasapi",
|
||||
"windows 0.58.0",
|
||||
"xcap",
|
||||
]
|
||||
@@ -8125,6 +8126,19 @@ dependencies = [
|
||||
"try-lock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasapi"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f6b03b82e419f186fcdc06ac6068621bdadc88b89b2612067f1c021ad2c9449"
|
||||
dependencies = [
|
||||
"log",
|
||||
"num-integer",
|
||||
"widestring",
|
||||
"windows 0.57.0",
|
||||
"windows-core 0.57.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.9.0+wasi-snapshot-preview1"
|
||||
@@ -8431,6 +8445,12 @@ dependencies = [
|
||||
"wasite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "widestring"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
|
||||
@@ -57,6 +57,13 @@ windows = { version = "0.58", features = [
|
||||
"Win32_UI_WindowsAndMessaging",
|
||||
] }
|
||||
|
||||
# WASAPI loopback capture for system-audio screen-share. Lets the custom
|
||||
# picker hand LiveKit a real audio track without falling back to the OS
|
||||
# screen picker (which is the only way getDisplayMedia can grab system
|
||||
# sound). Windows-only for v1; macOS needs ScreenCaptureKit-audio and
|
||||
# Linux needs a PulseAudio / PipeWire path.
|
||||
wasapi = "0.15"
|
||||
|
||||
# LiveKit client SDK — lives behind the `rust-livekit` feature flag so the
|
||||
# baseline build stays unaffected while the JS-SDK path is still the
|
||||
# default. Pulls libwebrtc-rs which adds ~20MB to the binary and ~5-10min
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
mod crypto;
|
||||
mod screen_audio;
|
||||
mod screen_capture;
|
||||
mod screen_sources;
|
||||
|
||||
@@ -99,6 +100,8 @@ pub fn run() {
|
||||
screen_sources::capture_screen_source_thumbnail_bytes,
|
||||
screen_capture::start_screen_capture,
|
||||
screen_capture::stop_screen_capture,
|
||||
screen_audio::start_system_audio_capture,
|
||||
screen_audio::stop_system_audio_capture,
|
||||
])
|
||||
.plugin(tauri_plugin_notification::init());
|
||||
|
||||
@@ -121,6 +124,8 @@ pub fn run() {
|
||||
screen_sources::capture_screen_source_thumbnail_bytes,
|
||||
screen_capture::start_screen_capture,
|
||||
screen_capture::stop_screen_capture,
|
||||
screen_audio::start_system_audio_capture,
|
||||
screen_audio::stop_system_audio_capture,
|
||||
livekit_bridge::livekit_connect,
|
||||
livekit_bridge::livekit_disconnect,
|
||||
livekit_bridge::livekit_send_data,
|
||||
|
||||
@@ -0,0 +1,428 @@
|
||||
// Native system-audio capture for the custom screen-share picker. Without
|
||||
// this path the picker has to fall back to getDisplayMedia whenever the
|
||||
// user ticks "Mit System-Sound", because Chromium only wires audio into
|
||||
// desktop captures that the OS picker produced. Here we grab the default
|
||||
// render endpoint's loopback stream via WASAPI, convert it to 48kHz f32
|
||||
// stereo, and ship the samples to the JS side through a Tauri Channel.
|
||||
// An AudioWorklet on the frontend feeds them into a MediaStreamDestination
|
||||
// so LiveKit publishes a plain ScreenShareAudio track.
|
||||
//
|
||||
// Windows-only for v1. macOS + Linux stubs return a clear error so the
|
||||
// frontend can fall back cleanly on those platforms until their native
|
||||
// paths ship (ScreenCaptureKit-audio / PipeWire).
|
||||
|
||||
#![allow(clippy::needless_return)]
|
||||
|
||||
use base64::Engine;
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::thread;
|
||||
use tauri::ipc::Channel;
|
||||
|
||||
// Output format we always deliver to the frontend. Picking a single fixed
|
||||
// format means the AudioWorklet never has to renegotiate — it just assumes
|
||||
// interleaved f32 stereo at 48kHz. WASAPI mix format is usually already
|
||||
// this on Windows 10+, so the resample branch is rarely hit.
|
||||
const OUTPUT_SAMPLE_RATE: u32 = 48_000;
|
||||
const OUTPUT_CHANNELS: u16 = 2;
|
||||
|
||||
static NEXT_ID: AtomicU32 = AtomicU32::new(1);
|
||||
static SESSIONS: OnceLock<Mutex<HashMap<u32, Session>>> = OnceLock::new();
|
||||
|
||||
fn sessions() -> &'static Mutex<HashMap<u32, Session>> {
|
||||
SESSIONS.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
}
|
||||
|
||||
struct Session {
|
||||
stop: Arc<AtomicBool>,
|
||||
handle: Option<thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Clone)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AudioFramePayload {
|
||||
pub capture_id: u32,
|
||||
pub sample_rate: u32,
|
||||
pub channels: u16,
|
||||
/// Interleaved little-endian f32 stereo samples, base64-encoded.
|
||||
/// Frontend decodes via `atob` → `Uint8Array` → `Float32Array` view.
|
||||
/// Base64 is used instead of a raw `Vec<f32>` because Tauri Channel
|
||||
/// serialises via JSON — a JSON array of floats balloons to ~2–3×
|
||||
/// the byte count, and at 48kHz stereo that's enough IPC traffic
|
||||
/// to matter.
|
||||
pub samples_base64: String,
|
||||
}
|
||||
|
||||
/// Start a loopback capture of the default render endpoint and begin
|
||||
/// streaming audio frames on the provided channel. Returns a numeric
|
||||
/// capture id that must be handed to `stop_system_audio_capture` when
|
||||
/// the share ends.
|
||||
#[tauri::command]
|
||||
pub fn start_system_audio_capture(
|
||||
channel: Channel<AudioFramePayload>,
|
||||
) -> Result<u32, String> {
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
let capture_id = NEXT_ID.fetch_add(1, Ordering::Relaxed);
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let stop_clone = Arc::clone(&stop);
|
||||
|
||||
let handle = thread::Builder::new()
|
||||
.name(format!("screen-audio-{capture_id}"))
|
||||
.spawn(move || {
|
||||
if let Err(err) =
|
||||
windows_loopback::capture_loop(capture_id, channel, stop_clone)
|
||||
{
|
||||
eprintln!("screen-audio {capture_id}: {err}");
|
||||
}
|
||||
})
|
||||
.map_err(|e| format!("failed to spawn audio thread: {e}"))?;
|
||||
|
||||
sessions().lock().unwrap().insert(
|
||||
capture_id,
|
||||
Session {
|
||||
stop,
|
||||
handle: Some(handle),
|
||||
},
|
||||
);
|
||||
Ok(capture_id)
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
{
|
||||
// Keep the `channel` binding alive so Tauri doesn't complain about
|
||||
// an unused parameter on the non-Windows build.
|
||||
let _ = channel;
|
||||
Err("system audio capture only supported on Windows".into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Tear down the capture for the given id. Safe to call on a missing id
|
||||
/// (no-op) so the JS side doesn't have to track whether the stop has
|
||||
/// already been issued by the screen-share teardown path.
|
||||
#[tauri::command]
|
||||
pub fn stop_system_audio_capture(capture_id: u32) -> Result<(), String> {
|
||||
let session = sessions().lock().unwrap().remove(&capture_id);
|
||||
let Some(mut session) = session else {
|
||||
return Ok(());
|
||||
};
|
||||
session.stop.store(true, Ordering::Relaxed);
|
||||
if let Some(handle) = session.handle.take() {
|
||||
// Best-effort join — the capture loop polls `stop` every event
|
||||
// cycle (≤100ms) so this usually returns promptly. If the WASAPI
|
||||
// call is wedged we'd rather drop the handle than hang the stop.
|
||||
let _ = handle.join();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Windows loopback implementation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows_loopback {
|
||||
use super::*;
|
||||
use wasapi::{initialize_mta, Direction, SampleType, ShareMode};
|
||||
|
||||
pub fn capture_loop(
|
||||
capture_id: u32,
|
||||
channel: Channel<AudioFramePayload>,
|
||||
stop: Arc<AtomicBool>,
|
||||
) -> Result<(), String> {
|
||||
// COM must be initialised on every thread that touches WASAPI.
|
||||
// MTA is the right model for a background capture thread — STA
|
||||
// would require message pumping we don't want to add.
|
||||
initialize_mta()
|
||||
.ok()
|
||||
.map_err(|e| format!("initialize_mta: {e:?}"))?;
|
||||
|
||||
let device = wasapi::get_default_device(&Direction::Render)
|
||||
.map_err(|e| format!("get_default_device: {e:?}"))?;
|
||||
let mut audio_client = device
|
||||
.get_iaudioclient()
|
||||
.map_err(|e| format!("get_iaudioclient: {e:?}"))?;
|
||||
|
||||
// Use the mix format that Windows is already pushing to the
|
||||
// endpoint. Loopback capture won't convert for us — asking for a
|
||||
// fixed format here makes Initialize() fail on non-matching
|
||||
// hardware. We resample + channel-mix ourselves downstream.
|
||||
let mix_format = audio_client
|
||||
.get_mixformat()
|
||||
.map_err(|e| format!("get_mixformat: {e:?}"))?;
|
||||
let input_rate = mix_format.get_samplespersec();
|
||||
let input_channels = mix_format.get_nchannels();
|
||||
let bits_per_sample = mix_format.get_bitspersample();
|
||||
let block_align = mix_format.get_blockalign();
|
||||
let sample_type = mix_format.get_subformat().unwrap_or(SampleType::Int);
|
||||
|
||||
let (def_time, _min_time) = audio_client
|
||||
.get_periods()
|
||||
.map_err(|e| format!("get_periods: {e:?}"))?;
|
||||
|
||||
// Direction::Capture + loopback: WASAPI streams what Windows is
|
||||
// sending to the speakers instead of what an input device is
|
||||
// producing. Shared mode so we coexist with other apps.
|
||||
audio_client
|
||||
.initialize_client(
|
||||
&mix_format,
|
||||
def_time,
|
||||
&Direction::Capture,
|
||||
&ShareMode::Shared,
|
||||
true,
|
||||
)
|
||||
.map_err(|e| format!("initialize_client: {e:?}"))?;
|
||||
|
||||
let h_event = audio_client
|
||||
.set_get_eventhandle()
|
||||
.map_err(|e| format!("set_get_eventhandle: {e:?}"))?;
|
||||
let capture_client = audio_client
|
||||
.get_audiocaptureclient()
|
||||
.map_err(|e| format!("get_audiocaptureclient: {e:?}"))?;
|
||||
|
||||
audio_client
|
||||
.start_stream()
|
||||
.map_err(|e| format!("start_stream: {e:?}"))?;
|
||||
|
||||
// Resampler state — last stereo frame from the previous buffer so
|
||||
// linear interpolation at the buffer boundary doesn't click.
|
||||
// Initialised to silence.
|
||||
let mut last_stereo: [f32; 2] = [0.0, 0.0];
|
||||
|
||||
while !stop.load(Ordering::Relaxed) {
|
||||
// 100ms timeout lets the loop check the stop flag even when
|
||||
// the endpoint is silent (WASAPI doesn't signal the event at
|
||||
// all for pure-silence streams on some driver versions).
|
||||
if h_event.wait_for_event(100).is_err() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Drain all packets available since the last wake — there
|
||||
// can be several queued if we were preempted.
|
||||
loop {
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let frames_available = match capture_client.get_next_nbr_frames() {
|
||||
Ok(Some(n)) if n > 0 => n,
|
||||
Ok(_) => break,
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"screen-audio {capture_id}: get_next_nbr_frames: {e:?}"
|
||||
);
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let bytes_needed =
|
||||
frames_available as usize * block_align as usize;
|
||||
let mut raw = vec![0u8; bytes_needed];
|
||||
if let Err(e) = capture_client.read_from_device(&mut raw) {
|
||||
eprintln!(
|
||||
"screen-audio {capture_id}: read_from_device: {e:?}"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
// Decode PCM into interleaved f32 at the device's native
|
||||
// rate + channel count.
|
||||
let decoded = decode_pcm(
|
||||
&raw,
|
||||
input_channels,
|
||||
bits_per_sample,
|
||||
&sample_type,
|
||||
);
|
||||
|
||||
// Channel-fold → 2ch, then resample → 48kHz.
|
||||
let stereo = to_stereo(&decoded, input_channels);
|
||||
let out_samples = resample_linear_stereo(
|
||||
&stereo,
|
||||
input_rate,
|
||||
OUTPUT_SAMPLE_RATE,
|
||||
&mut last_stereo,
|
||||
);
|
||||
|
||||
if out_samples.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Pack f32s as little-endian bytes then base64. IPC-wise
|
||||
// this is ~1.3× the raw byte count versus 5–10× for a
|
||||
// JSON array of floats, which is the difference between
|
||||
// "fine" and "wastes a CPU core" at 48kHz stereo.
|
||||
let mut bytes = Vec::with_capacity(out_samples.len() * 4);
|
||||
for s in &out_samples {
|
||||
bytes.extend_from_slice(&s.to_le_bytes());
|
||||
}
|
||||
let samples_b64 =
|
||||
base64::engine::general_purpose::STANDARD.encode(&bytes);
|
||||
|
||||
if channel
|
||||
.send(AudioFramePayload {
|
||||
capture_id,
|
||||
sample_rate: OUTPUT_SAMPLE_RATE,
|
||||
channels: OUTPUT_CHANNELS,
|
||||
samples_base64: samples_b64,
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
// Frontend went away — stop cleanly.
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let _ = audio_client.stop_stream();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Convert a raw WASAPI buffer into interleaved f32 at the device's
|
||||
// native channel count. Handles the three formats that actually show
|
||||
// up on Windows render endpoints: f32 (most modern hardware), i16
|
||||
// (older onboard codecs), and i32 (pro audio interfaces). Anything
|
||||
// else falls through to zeros so a weird format doesn't crash the
|
||||
// share — the user will notice silence and can retry.
|
||||
fn decode_pcm(
|
||||
raw: &[u8],
|
||||
channels: u16,
|
||||
bits_per_sample: u16,
|
||||
sample_type: &SampleType,
|
||||
) -> Vec<f32> {
|
||||
match (sample_type, bits_per_sample) {
|
||||
(SampleType::Float, 32) => {
|
||||
let mut out = Vec::with_capacity(raw.len() / 4);
|
||||
for chunk in raw.chunks_exact(4) {
|
||||
out.push(f32::from_le_bytes([
|
||||
chunk[0], chunk[1], chunk[2], chunk[3],
|
||||
]));
|
||||
}
|
||||
out
|
||||
}
|
||||
(SampleType::Int, 16) => {
|
||||
let scale = 1.0_f32 / (i16::MAX as f32);
|
||||
let mut out = Vec::with_capacity(raw.len() / 2);
|
||||
for chunk in raw.chunks_exact(2) {
|
||||
let s = i16::from_le_bytes([chunk[0], chunk[1]]);
|
||||
out.push(s as f32 * scale);
|
||||
}
|
||||
out
|
||||
}
|
||||
(SampleType::Int, 32) => {
|
||||
let scale = 1.0_f32 / (i32::MAX as f32);
|
||||
let mut out = Vec::with_capacity(raw.len() / 4);
|
||||
for chunk in raw.chunks_exact(4) {
|
||||
let s = i32::from_le_bytes([
|
||||
chunk[0], chunk[1], chunk[2], chunk[3],
|
||||
]);
|
||||
out.push(s as f32 * scale);
|
||||
}
|
||||
out
|
||||
}
|
||||
_ => {
|
||||
// Unknown format — emit silence of the right frame count
|
||||
// so downstream math stays correct.
|
||||
let bytes_per_frame =
|
||||
(bits_per_sample as usize / 8) * channels as usize;
|
||||
let frames = if bytes_per_frame == 0 {
|
||||
0
|
||||
} else {
|
||||
raw.len() / bytes_per_frame
|
||||
};
|
||||
vec![0.0; frames * channels as usize]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Down- or up-mix to stereo. Surround layouts fold L+R only (center
|
||||
// + surrounds get dropped) which is the simplest defensible choice
|
||||
// for screen-share audio — most content is LR-centric and a proper
|
||||
// ITU-R BS.775 downmix would pull in matrix coefficients we'd rather
|
||||
// avoid in v1.
|
||||
fn to_stereo(interleaved: &[f32], channels: u16) -> Vec<f32> {
|
||||
if channels == 0 || interleaved.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
if channels == 2 {
|
||||
return interleaved.to_vec();
|
||||
}
|
||||
let ch = channels as usize;
|
||||
let frames = interleaved.len() / ch;
|
||||
let mut out = Vec::with_capacity(frames * 2);
|
||||
if channels == 1 {
|
||||
for i in 0..frames {
|
||||
let s = interleaved[i];
|
||||
out.push(s);
|
||||
out.push(s);
|
||||
}
|
||||
} else {
|
||||
for i in 0..frames {
|
||||
let base = i * ch;
|
||||
out.push(interleaved[base]);
|
||||
out.push(interleaved[base + 1]);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// Linear-interpolation resampler for interleaved stereo f32. Not the
|
||||
// prettiest option theoretically, but at 44.1→48 the audible
|
||||
// artefacts stay below threshold for speech + game/music content. The
|
||||
// `last_stereo` state preserves the final frame across invocations so
|
||||
// the interpolation at the buffer boundary doesn't produce a click.
|
||||
fn resample_linear_stereo(
|
||||
input_stereo: &[f32],
|
||||
input_rate: u32,
|
||||
output_rate: u32,
|
||||
last_stereo: &mut [f32; 2],
|
||||
) -> Vec<f32> {
|
||||
if input_stereo.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
if input_rate == output_rate {
|
||||
last_stereo[0] = input_stereo[input_stereo.len() - 2];
|
||||
last_stereo[1] = input_stereo[input_stereo.len() - 1];
|
||||
return input_stereo.to_vec();
|
||||
}
|
||||
let ratio = output_rate as f64 / input_rate as f64;
|
||||
let in_frames = input_stereo.len() / 2;
|
||||
let out_frames = (in_frames as f64 * ratio).floor() as usize;
|
||||
if out_frames == 0 {
|
||||
last_stereo[0] = input_stereo[input_stereo.len() - 2];
|
||||
last_stereo[1] = input_stereo[input_stereo.len() - 1];
|
||||
return Vec::new();
|
||||
}
|
||||
let mut out = Vec::with_capacity(out_frames * 2);
|
||||
|
||||
let prev_l = last_stereo[0];
|
||||
let prev_r = last_stereo[1];
|
||||
|
||||
for i in 0..out_frames {
|
||||
let src_pos = i as f64 / ratio;
|
||||
let src_frame = src_pos.floor() as i64;
|
||||
let frac = (src_pos - src_frame as f64) as f32;
|
||||
|
||||
// `src_frame == -1` comes up for the very first output frame
|
||||
// when ratio > 1 — interpolate against the previous buffer's
|
||||
// final sample to bridge the two.
|
||||
let (l0, r0) = if src_frame < 0 {
|
||||
(prev_l, prev_r)
|
||||
} else {
|
||||
let idx = (src_frame as usize).min(in_frames - 1) * 2;
|
||||
(input_stereo[idx], input_stereo[idx + 1])
|
||||
};
|
||||
let next = ((src_frame + 1) as usize).min(in_frames - 1);
|
||||
let l1 = input_stereo[next * 2];
|
||||
let r1 = input_stereo[next * 2 + 1];
|
||||
|
||||
out.push(l0 + (l1 - l0) * frac);
|
||||
out.push(r0 + (r1 - r0) * frac);
|
||||
}
|
||||
|
||||
last_stereo[0] = input_stereo[input_stereo.len() - 2];
|
||||
last_stereo[1] = input_stereo[input_stereo.len() - 1];
|
||||
out
|
||||
}
|
||||
}
|
||||
@@ -282,14 +282,6 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
|
||||
</span>
|
||||
</label>
|
||||
</div>
|
||||
{includeAudio && selectedId && (
|
||||
<p className="rounded-md border border-amber-500/30 bg-amber-500/10 px-3 py-2 text-[11px] text-amber-700 dark:text-amber-300">
|
||||
{t('app:call.share_audio_uses_os_picker', {
|
||||
defaultValue:
|
||||
'Mit System-Sound fragt der Browser noch einmal nach der Quelle — Video-Direktpfad geht nur ohne Audio.',
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
{error && (
|
||||
<p role="alert" className="text-xs text-rose-600 dark:text-rose-300">
|
||||
{error}
|
||||
|
||||
@@ -80,6 +80,10 @@ import {
|
||||
NativeCaptureUnavailable,
|
||||
startNativeCapture,
|
||||
} from '../lib/screenCapture';
|
||||
import {
|
||||
type SystemAudioHandle,
|
||||
startSystemAudioCapture,
|
||||
} from '../lib/screenAudio';
|
||||
import {
|
||||
clearScreenShareVolumes,
|
||||
getScreenShareVolume,
|
||||
@@ -305,6 +309,11 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
// on the canvas track's 'ended' event. Not kept in React state because
|
||||
// it never feeds into a render.
|
||||
const nativeCaptureRef = useRef<NativeCaptureHandle | null>(null);
|
||||
// Matching handle for the Windows-only WASAPI system-audio capture.
|
||||
// Lives in lockstep with the video handle above when the user picks
|
||||
// "Mit System-Sound"; teardown is wired so that stopping either track
|
||||
// also stops the other, so stale audio can't outlive the video share.
|
||||
const nativeAudioCaptureRef = useRef<SystemAudioHandle | null>(null);
|
||||
const roomRef = useRef<Room | null>(null);
|
||||
// Web Audio graph that mixes live mic + soundboard sources into a single
|
||||
// published track. Created per call in joinRoom, destroyed in disconnectRoom.
|
||||
@@ -486,6 +495,11 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
nativeCaptureRef.current = null;
|
||||
await h.stop().catch(() => undefined);
|
||||
}
|
||||
if (nativeAudioCaptureRef.current) {
|
||||
const h = nativeAudioCaptureRef.current;
|
||||
nativeAudioCaptureRef.current = null;
|
||||
await h.stop().catch(() => undefined);
|
||||
}
|
||||
roomRef.current = null;
|
||||
setRoom(null);
|
||||
setRemoteParticipants([]);
|
||||
@@ -1190,24 +1204,25 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
const sourceId = overrides?.sourceId ?? null;
|
||||
|
||||
// Native capture path — tried first when the user came in via our
|
||||
// custom picker. xcap on the Rust side grabs frames, streams them as
|
||||
// JPEG over a Tauri channel, and we draw them onto a canvas whose
|
||||
// captureStream() becomes the MediaStream LiveKit publishes. This
|
||||
// completely skips the OS picker. Video-only (no system audio yet);
|
||||
// if the user asked for audio we fall through to the legacy paths
|
||||
// below so audio still works via getDisplayMedia.
|
||||
// custom picker. xcap on the Rust side grabs video frames and, on
|
||||
// Windows with "Mit System-Sound" on, the WASAPI loopback module
|
||||
// grabs the render endpoint. Both stream over Tauri channels into
|
||||
// tracks we publish directly to LiveKit — the OS picker never
|
||||
// appears. If native audio fails on a platform that can't supply
|
||||
// it (non-Windows v1), we continue with video-only and log; the
|
||||
// user still gets their direct-video share.
|
||||
if (!sourceId) {
|
||||
console.info(
|
||||
'screen-share: no sourceId supplied by picker, OS picker will open',
|
||||
);
|
||||
} else if (settings.includeSystemAudio) {
|
||||
console.info(
|
||||
'screen-share: system audio requested — native path unavailable (needs WASAPI/ScreenCaptureKit), OS picker will open',
|
||||
);
|
||||
}
|
||||
if (sourceId && !settings.includeSystemAudio) {
|
||||
if (sourceId) {
|
||||
try {
|
||||
console.info('screen-share: trying native capture path', { sourceId, fps });
|
||||
console.info('screen-share: trying native capture path', {
|
||||
sourceId,
|
||||
fps,
|
||||
includeSystemAudio: settings.includeSystemAudio,
|
||||
});
|
||||
const { Track: LkTrack } = await import('livekit-client');
|
||||
const maxWidth = ssParams.dims?.width ?? 1920;
|
||||
const maxHeight = ssParams.dims?.height ?? 1080;
|
||||
@@ -1228,9 +1243,54 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
source: LkTrack.Source.ScreenShare,
|
||||
videoCodec: 'vp9',
|
||||
});
|
||||
|
||||
// Optional native audio. Failure here is non-fatal — the video
|
||||
// pipeline is already running and bailing out would be worse
|
||||
// UX than shipping a silent share. The warning surfaces the
|
||||
// platform gap so the user knows why their audio is missing.
|
||||
let audioHandle: SystemAudioHandle | null = null;
|
||||
if (settings.includeSystemAudio) {
|
||||
try {
|
||||
audioHandle = await startSystemAudioCapture();
|
||||
nativeAudioCaptureRef.current = audioHandle;
|
||||
const audioMst = audioHandle.stream.getAudioTracks()[0];
|
||||
if (audioMst) {
|
||||
const audioPub = await lp.publishTrack(audioMst, {
|
||||
source: LkTrack.Source.ScreenShareAudio,
|
||||
});
|
||||
audioMst.addEventListener('ended', () => {
|
||||
void (async () => {
|
||||
try {
|
||||
if (audioPub.track) await lp.unpublishTrack(audioPub.track);
|
||||
} catch {
|
||||
/* already unpublished */
|
||||
}
|
||||
const active = nativeAudioCaptureRef.current;
|
||||
if (active && active.captureId === audioHandle!.captureId) {
|
||||
nativeAudioCaptureRef.current = null;
|
||||
await active.stop().catch(() => undefined);
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
console.warn(
|
||||
'screen-share: native system-audio unavailable, sharing video only',
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
if (nativeAudioCaptureRef.current) {
|
||||
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
|
||||
nativeAudioCaptureRef.current = null;
|
||||
}
|
||||
audioHandle = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Canvas stream 'ended' fires on handle.stop() (we track.stop()
|
||||
// each track) — chain unpublish + native teardown so one ended
|
||||
// event cleans everything up regardless of who triggered it.
|
||||
// Also tear down any paired audio capture so sound can't
|
||||
// outlive the video share.
|
||||
videoMst.addEventListener('ended', () => {
|
||||
void (async () => {
|
||||
try {
|
||||
@@ -1243,10 +1303,17 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
nativeCaptureRef.current = null;
|
||||
await active.stop().catch(() => undefined);
|
||||
}
|
||||
const audioActive = nativeAudioCaptureRef.current;
|
||||
if (audioActive) {
|
||||
nativeAudioCaptureRef.current = null;
|
||||
await audioActive.stop().catch(() => undefined);
|
||||
}
|
||||
setIsScreenSharing(false);
|
||||
})();
|
||||
});
|
||||
console.info('screen-share: native capture active');
|
||||
console.info('screen-share: native capture active', {
|
||||
audio: audioHandle != null,
|
||||
});
|
||||
setIsScreenSharing(true);
|
||||
return;
|
||||
} catch (err: unknown) {
|
||||
@@ -1257,6 +1324,10 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
await nativeCaptureRef.current.stop().catch(() => undefined);
|
||||
nativeCaptureRef.current = null;
|
||||
}
|
||||
if (nativeAudioCaptureRef.current) {
|
||||
await nativeAudioCaptureRef.current.stop().catch(() => undefined);
|
||||
nativeAudioCaptureRef.current = null;
|
||||
}
|
||||
console.warn(
|
||||
'screen-share: native path failed, falling back',
|
||||
err instanceof Error ? err.message : err,
|
||||
@@ -1403,6 +1474,15 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
console.warn('native capture stop failed', err);
|
||||
}
|
||||
}
|
||||
if (nativeAudioCaptureRef.current) {
|
||||
const h = nativeAudioCaptureRef.current;
|
||||
nativeAudioCaptureRef.current = null;
|
||||
try {
|
||||
await h.stop();
|
||||
} catch (err: unknown) {
|
||||
console.warn('native audio capture stop failed', err);
|
||||
}
|
||||
}
|
||||
const r = roomRef.current;
|
||||
if (!r) return;
|
||||
const lp = r.localParticipant;
|
||||
@@ -2284,6 +2364,20 @@ function attachTrack(
|
||||
if (isScreenShareAudio) {
|
||||
audio.setAttribute('data-track-source', 'screenshare');
|
||||
}
|
||||
// Diagnostic — surfaces source-tag mismatches between SDK versions.
|
||||
// If a remote participant publishes system-audio but the tag never
|
||||
// reaches us, `isScreenShareAudio` flips false and the watching
|
||||
// gate is bypassed; seeing this in the console tells us whether
|
||||
// the unwanted playback is a gating bug or a tagging mismatch.
|
||||
console.info('attachTrack:audio', {
|
||||
participant: participant.identity,
|
||||
trackSource: track.source,
|
||||
pubSource: publication.source,
|
||||
isScreenShareAudio,
|
||||
watching: participant.identity
|
||||
? watchingShareUserIdsMirror.has(participant.identity)
|
||||
: null,
|
||||
});
|
||||
if (participant.identity) {
|
||||
audio.setAttribute('data-participant', participant.identity);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
// Frontend side of the native system-audio pipeline. Pairs with the Rust
|
||||
// `screen_audio` module: it opens a Tauri Channel, receives interleaved
|
||||
// f32 stereo samples at 48kHz (base64-encoded), and surfaces them as a
|
||||
// real `MediaStream` that LiveKit can publish as a `ScreenShareAudio`
|
||||
// track. An AudioWorklet does the heavy lifting so the render thread is
|
||||
// never the bottleneck — the main thread just pushes decoded samples
|
||||
// across a port; the worklet copies them into its output buffer which
|
||||
// feeds a `MediaStreamDestination`.
|
||||
//
|
||||
// Windows-only right now. On other platforms `startSystemAudioCapture`
|
||||
// throws `SystemAudioUnavailable` and the caller is expected to fall
|
||||
// back to the browser's getDisplayMedia path.
|
||||
|
||||
import { isTauriRuntime } from './globalShortcut';
|
||||
|
||||
export interface SystemAudioHandle {
|
||||
/** Rust-side capture id. Pass to the Rust stop command via `stop()`. */
|
||||
captureId: number;
|
||||
/** MediaStream carrying a single audio track at 48kHz stereo. */
|
||||
stream: MediaStream;
|
||||
/** Teardown — stops the Rust thread, closes the AudioContext, ends the
|
||||
* MediaStreamDestination track. Idempotent. */
|
||||
stop: () => Promise<void>;
|
||||
}
|
||||
|
||||
/** Thrown when the platform can't deliver native system-audio (non-Tauri
|
||||
* runtime, non-Windows host, WebAudio unavailable, COM init failure). */
|
||||
export class SystemAudioUnavailable extends Error {
|
||||
constructor(reason: string) {
|
||||
super('system audio unavailable: ' + reason);
|
||||
this.name = 'SystemAudioUnavailable';
|
||||
}
|
||||
}
|
||||
|
||||
interface AudioFramePayload {
|
||||
captureId: number;
|
||||
sampleRate: number;
|
||||
channels: number;
|
||||
samplesBase64: string;
|
||||
}
|
||||
|
||||
// AudioWorklet source embedded as a string. The worklet keeps a pair of
|
||||
// ring buffers (one per channel) that the main thread appends to as
|
||||
// samples arrive. `process()` drains the ring buffers into the output
|
||||
// blocks; an underrun emits silence instead of propagating the stall
|
||||
// upwards (a glitch is better than a freeze for LiveKit's Opus encoder).
|
||||
//
|
||||
// The worklet runs at AudioContext sample rate, which we pin to 48kHz via
|
||||
// the AudioContext constructor. That matches what the Rust side already
|
||||
// resamples to, so no further rate conversion is needed here.
|
||||
const WORKLET_SOURCE = `
|
||||
class LoopbackAudioProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
// Ring buffer sized for latency, not for "never drop". 300ms hard cap,
|
||||
// 80ms target — we aim for ~one WASAPI packet of headroom above the
|
||||
// render quantum and drop excess whenever the producer gets ahead.
|
||||
// Keeping the target small is the difference between "feels live" and
|
||||
// "laggy" for screen-share audio.
|
||||
this.bufferSize = 48000 * 0.3 | 0;
|
||||
this.targetFrames = 48000 * 0.08 | 0;
|
||||
this.bufL = new Float32Array(this.bufferSize);
|
||||
this.bufR = new Float32Array(this.bufferSize);
|
||||
this.writePos = 0;
|
||||
this.readPos = 0;
|
||||
this.available = 0;
|
||||
this.port.onmessage = (e) => {
|
||||
const { left, right } = e.data;
|
||||
const len = left.length;
|
||||
for (let i = 0; i < len; i++) {
|
||||
this.bufL[this.writePos] = left[i];
|
||||
this.bufR[this.writePos] = right[i];
|
||||
this.writePos = (this.writePos + 1) % this.bufferSize;
|
||||
if (this.available < this.bufferSize) {
|
||||
this.available++;
|
||||
} else {
|
||||
// Buffer full — advance the read cursor to keep writing.
|
||||
this.readPos = (this.readPos + 1) % this.bufferSize;
|
||||
}
|
||||
}
|
||||
// Hard cap: if we're this far behind the producer, skip ahead to
|
||||
// the target latency instead of playing out minutes of stale audio.
|
||||
// Happens on: AudioContext resume after suspend, tab throttle
|
||||
// recovery, any hiccup that left samples piling up.
|
||||
if (this.available > this.targetFrames * 3) {
|
||||
const drop = this.available - this.targetFrames;
|
||||
this.readPos = (this.readPos + drop) % this.bufferSize;
|
||||
this.available -= drop;
|
||||
}
|
||||
};
|
||||
}
|
||||
process(_inputs, outputs) {
|
||||
const output = outputs[0];
|
||||
if (!output || output.length === 0) return true;
|
||||
const out0 = output[0];
|
||||
const out1 = output[1] || output[0];
|
||||
const n = out0.length;
|
||||
for (let i = 0; i < n; i++) {
|
||||
if (this.available > 0) {
|
||||
out0[i] = this.bufL[this.readPos];
|
||||
if (out1 !== out0) out1[i] = this.bufR[this.readPos];
|
||||
this.readPos = (this.readPos + 1) % this.bufferSize;
|
||||
this.available--;
|
||||
} else {
|
||||
out0[i] = 0;
|
||||
if (out1 !== out0) out1[i] = 0;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('screen-audio-loopback', LoopbackAudioProcessor);
|
||||
`;
|
||||
|
||||
let workletModuleUrl: string | null = null;
|
||||
function getWorkletModuleUrl(): string {
|
||||
if (workletModuleUrl) return workletModuleUrl;
|
||||
const blob = new Blob([WORKLET_SOURCE], { type: 'application/javascript' });
|
||||
workletModuleUrl = URL.createObjectURL(blob);
|
||||
return workletModuleUrl;
|
||||
}
|
||||
|
||||
export async function startSystemAudioCapture(): Promise<SystemAudioHandle> {
|
||||
if (!isTauriRuntime()) {
|
||||
throw new SystemAudioUnavailable('not a tauri runtime');
|
||||
}
|
||||
const AudioCtor: typeof AudioContext | undefined =
|
||||
typeof window !== 'undefined'
|
||||
? (window.AudioContext ??
|
||||
(window as unknown as { webkitAudioContext?: typeof AudioContext })
|
||||
.webkitAudioContext)
|
||||
: undefined;
|
||||
if (!AudioCtor) {
|
||||
throw new SystemAudioUnavailable('WebAudio unavailable');
|
||||
}
|
||||
|
||||
// Pin to 48kHz so the worklet's input rate matches the Rust-side
|
||||
// output rate. If the OS forces a different rate the constructor
|
||||
// throws on some browsers; we catch and surface as Unavailable so the
|
||||
// caller can fall back.
|
||||
let ctx: AudioContext;
|
||||
try {
|
||||
ctx = new AudioCtor({ sampleRate: 48000, latencyHint: 'interactive' });
|
||||
} catch (err: unknown) {
|
||||
throw new SystemAudioUnavailable(
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await ctx.audioWorklet.addModule(getWorkletModuleUrl());
|
||||
} catch (err: unknown) {
|
||||
await ctx.close().catch(() => undefined);
|
||||
throw new SystemAudioUnavailable(
|
||||
'audioWorklet load failed: ' +
|
||||
(err instanceof Error ? err.message : String(err)),
|
||||
);
|
||||
}
|
||||
|
||||
const node = new AudioWorkletNode(ctx, 'screen-audio-loopback', {
|
||||
numberOfInputs: 0,
|
||||
numberOfOutputs: 1,
|
||||
outputChannelCount: [2],
|
||||
});
|
||||
const dest = ctx.createMediaStreamDestination();
|
||||
node.connect(dest);
|
||||
|
||||
// Kick the AudioContext out of `suspended` before any samples arrive —
|
||||
// the share is triggered from a user click so autoplay policy allows
|
||||
// this, and an un-resumed context would buffer everything the Rust
|
||||
// side produces until the context eventually runs, giving seconds of
|
||||
// initial latency.
|
||||
if (ctx.state !== 'running') {
|
||||
try {
|
||||
await ctx.resume();
|
||||
} catch (err: unknown) {
|
||||
console.warn('system-audio ctx.resume failed', err);
|
||||
}
|
||||
}
|
||||
|
||||
const { Channel, invoke } = await import('@tauri-apps/api/core');
|
||||
const channel = new Channel<AudioFramePayload>();
|
||||
|
||||
channel.onmessage = (frame: AudioFramePayload) => {
|
||||
const bytes = base64ToBytes(frame.samplesBase64);
|
||||
// Re-view the bytes as f32 little-endian. The byteLength is always
|
||||
// a multiple of 8 (f32 stereo pairs) — if not, drop the trailing
|
||||
// partial frame rather than risk a truncation artifact.
|
||||
const sampleCount = Math.floor(bytes.byteLength / 4);
|
||||
if (sampleCount < 2) return;
|
||||
const floats = new Float32Array(
|
||||
bytes.buffer,
|
||||
bytes.byteOffset,
|
||||
sampleCount,
|
||||
);
|
||||
// Interleaved L/R → deinterleaved for the worklet. Copying out of
|
||||
// the base64 view also ensures the Float32Arrays we postMessage are
|
||||
// owned (the underlying buffer is about to be garbage-collected).
|
||||
const frames = floats.length >> 1;
|
||||
const left = new Float32Array(frames);
|
||||
const right = new Float32Array(frames);
|
||||
for (let i = 0; i < frames; i++) {
|
||||
left[i] = floats[i * 2] ?? 0;
|
||||
right[i] = floats[i * 2 + 1] ?? 0;
|
||||
}
|
||||
// Transfer the buffers so postMessage is zero-copy.
|
||||
node.port.postMessage(
|
||||
{ left, right },
|
||||
[left.buffer, right.buffer],
|
||||
);
|
||||
};
|
||||
|
||||
let captureId: number;
|
||||
try {
|
||||
captureId = await invoke<number>('start_system_audio_capture', { channel });
|
||||
} catch (err: unknown) {
|
||||
node.disconnect();
|
||||
await ctx.close().catch(() => undefined);
|
||||
throw new SystemAudioUnavailable(
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
}
|
||||
|
||||
const stream = dest.stream;
|
||||
|
||||
let stopped = false;
|
||||
const stop = async (): Promise<void> => {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
try {
|
||||
await invoke('stop_system_audio_capture', { captureId });
|
||||
} catch (err: unknown) {
|
||||
console.warn('stop_system_audio_capture failed', err);
|
||||
}
|
||||
try {
|
||||
node.disconnect();
|
||||
} catch {
|
||||
/* already disconnected */
|
||||
}
|
||||
for (const track of stream.getTracks()) {
|
||||
try {
|
||||
track.stop();
|
||||
} catch {
|
||||
/* already stopped */
|
||||
}
|
||||
}
|
||||
await ctx.close().catch(() => undefined);
|
||||
};
|
||||
|
||||
return { captureId, stream, stop };
|
||||
}
|
||||
|
||||
function base64ToBytes(b64: string): Uint8Array {
|
||||
const bin = atob(b64);
|
||||
const bytes = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) {
|
||||
bytes[i] = bin.charCodeAt(i);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
Reference in New Issue
Block a user