From c3e0c47d3289659e8ab06d3f292244a0d3752627 Mon Sep 17 00:00:00 2001 From: byGalax Date: Wed, 22 Apr 2026 21:42:52 +0200 Subject: [PATCH] feat(call): native screen-capture pipeline + faster picker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Picker speed (Phase 1+2): - screen_sources.rs split into list_screen_sources (metadata only, returns in ~10ms) + capture_screen_source_thumbnail (single source, by id). ScreenSourcePicker now shows names + placeholders instantly and streams thumbnails in as each capture lands. Total wall-clock is bounded by the slowest source instead of the serial sum. - enumerate_screen_sources kept as a dead_code fallback so any rollout regression can switch the frontend back without code loss. Native capture (Phase 3): - New src-tauri/src/screen_capture.rs. start_screen_capture spawns a Rust thread per share that grabs frames via xcap, downscales to the user's quality preset, JPEG-encodes at Q72, and streams each frame through a Tauri Channel. stop_screen_capture signals the stop flag and joins the worker. - Worker re-resolves the xcap handle inside the thread because xcap::Window holds a !Send HWND — passing the source id string across the thread boundary sidesteps that. - New lib/screenCapture.ts: decodes each frame into an ImageBitmap, draws to an offscreen canvas, exposes canvas.captureStream() as the MediaStream LiveKit publishes. Latest-wins frame queue drops stale frames when the JS side falls behind the Rust producer. 3s first- frame timeout so a silently-failing source (locked screen, DRM window) surfaces as a clean NativeCaptureUnavailable and we fall back to getDisplayMedia. - CallContext.startScreenShare takes the native path first when the picker provided a sourceId and system audio wasn't requested. The old chromeMediaSourceId attempt and final setScreenShareEnabled fallback stay in place for the audio case + non-Tauri runtimes. - stopScreenShare kills the native handle first, then unpublishes any manually-published ScreenShare/ScreenShareAudio tracks, then falls back to setScreenShareEnabled(false). disconnectRoom also stops the handle so we don't leak Rust threads across calls. Scope note: native path is video-only. System-audio capture needs WASAPI-loopback (Windows) or ScreenCaptureKit-audio (macOS); until those are wired, requesting audio in the picker falls through to the getDisplayMedia path and shows the OS picker for that one case. Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/desktop/src-tauri/src/lib.rs | 9 + apps/desktop/src-tauri/src/screen_capture.rs | 266 ++++++++++++++++++ apps/desktop/src-tauri/src/screen_sources.rs | 146 ++++++++-- .../src/components/ScreenSourcePicker.tsx | 31 +- apps/desktop/src/context/CallContext.tsx | 118 +++++++- apps/desktop/src/lib/screenCapture.ts | 184 ++++++++++++ apps/desktop/src/lib/screenSources.ts | 38 +++ 7 files changed, 752 insertions(+), 40 deletions(-) create mode 100644 apps/desktop/src-tauri/src/screen_capture.rs create mode 100644 apps/desktop/src/lib/screenCapture.ts diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 79de4a4..558cf0a 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -1,4 +1,5 @@ mod crypto; +mod screen_capture; mod screen_sources; #[cfg(feature = "rust-livekit")] @@ -93,6 +94,10 @@ pub fn run() { crypto::crypto_box_seal_open, crypto::crypto_pwhash, screen_sources::enumerate_screen_sources, + screen_sources::list_screen_sources, + screen_sources::capture_screen_source_thumbnail, + screen_capture::start_screen_capture, + screen_capture::stop_screen_capture, ]) .plugin(tauri_plugin_notification::init()); @@ -110,6 +115,10 @@ pub fn run() { crypto::crypto_box_seal_open, crypto::crypto_pwhash, screen_sources::enumerate_screen_sources, + screen_sources::list_screen_sources, + screen_sources::capture_screen_source_thumbnail, + screen_capture::start_screen_capture, + screen_capture::stop_screen_capture, livekit_bridge::livekit_connect, livekit_bridge::livekit_disconnect, livekit_bridge::livekit_send_data, diff --git a/apps/desktop/src-tauri/src/screen_capture.rs b/apps/desktop/src-tauri/src/screen_capture.rs new file mode 100644 index 0000000..692eaae --- /dev/null +++ b/apps/desktop/src-tauri/src/screen_capture.rs @@ -0,0 +1,266 @@ +// Native screen / window capture pipeline. Spawns a Rust thread per active +// capture that grabs frames via xcap, downscales them to the user's target +// resolution, encodes JPEG, and streams each frame to the JS side through +// a Tauri `Channel`. The JS end decodes into an +// ImageBitmap, draws to a , and exposes the canvas as a +// MediaStream via captureStream() — that stream is what LiveKit publishes. +// Net result: the user picks a source in our custom picker and the share +// starts directly, without the OS/browser picker appearing. +// +// JPEG on the Rust side + decode on the JS side introduces a second +// encoding hop (LiveKit re-encodes VP9 later) but keeps IPC bandwidth +// manageable — raw RGBA at 1920×1080×30fps would be ~240MB/s and cannot +// go over Tauri's JSON-serialised IPC. JPEG frames at Q72 land around +// 50–150KB each, so 30fps = 2–5MB/s of base64 traffic, which is fine. + +use base64::Engine; +use serde::Serialize; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::thread; +use std::time::{Duration, Instant}; +use tauri::ipc::Channel; + +static NEXT_ID: AtomicU32 = AtomicU32::new(1); +static SESSIONS: OnceLock>> = OnceLock::new(); + +fn sessions() -> &'static Mutex> { + SESSIONS.get_or_init(|| Mutex::new(HashMap::new())) +} + +struct Session { + stop: Arc, + handle: Option>, +} + +#[derive(Serialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct FramePayload { + pub capture_id: u32, + pub width: u32, + pub height: u32, + /// JPEG image bytes, base64-encoded (no data-URL prefix). Frontend + /// reconstructs via `Uint8Array.from(atob(...))` and decodes with + /// `createImageBitmap(blob)`. + pub jpeg_base64: String, +} + +enum Source { + Window(xcap::Window), + Monitor(xcap::Monitor), +} + +/// Start a continuous capture for the given source id and begin streaming +/// JPEG frames via the provided channel. Returns a numeric capture id that +/// must be passed to `stop_screen_capture` to tear the pipeline down. +#[tauri::command] +pub fn start_screen_capture( + source_id: String, + max_width: u32, + max_height: u32, + fps: u32, + channel: Channel, +) -> Result { + let clamped_fps = fps.clamp(5, 30); + let clamped_w = max_width.max(320).min(3840); + let clamped_h = max_height.max(180).min(2160); + + // Probe once on the command thread so we can surface a clear error + // before spawning; the actual capture handle is re-resolved inside the + // worker thread (xcap::Window holds a Windows HWND which is !Send). + if find_source(&source_id).is_none() { + return Err(format!("screen source {source_id} not found")); + } + + let capture_id = NEXT_ID.fetch_add(1, Ordering::Relaxed); + let stop = Arc::new(AtomicBool::new(false)); + let stop_clone = Arc::clone(&stop); + let source_id_owned = source_id; + + let handle = thread::Builder::new() + .name(format!("screen-capture-{capture_id}")) + .spawn(move || { + capture_loop( + source_id_owned, + clamped_w, + clamped_h, + clamped_fps, + capture_id, + channel, + stop_clone, + ); + }) + .map_err(|e| format!("failed to spawn capture thread: {e}"))?; + + sessions().lock().unwrap().insert( + capture_id, + Session { + stop, + handle: Some(handle), + }, + ); + Ok(capture_id) +} + +/// Signal the capture thread to stop and join it. Safe to call more than +/// once — missing ids are silently no-ops so the JS side doesn't need to +/// track whether a stop was already issued by the OS "stop sharing" path. +#[tauri::command] +pub fn stop_screen_capture(capture_id: u32) -> Result<(), String> { + let session = sessions().lock().unwrap().remove(&capture_id); + let Some(mut session) = session else { + return Ok(()); + }; + session.stop.store(true, Ordering::Relaxed); + if let Some(handle) = session.handle.take() { + // Best-effort join. If the thread is stuck in a long OS capture + // call we don't want to hang the command — detach after a brief + // wait by dropping the handle. + let _ = handle.join(); + } + Ok(()) +} + +fn capture_loop( + source_id: String, + max_w: u32, + max_h: u32, + fps: u32, + capture_id: u32, + channel: Channel, + stop: Arc, +) { + let frame_interval = Duration::from_nanos(1_000_000_000 / fps as u64); + let jpeg_quality: u8 = 72; + + // Re-resolve the source inside the worker thread — xcap::Window holds + // an HWND which is !Send so we can't move it across threads. Caching + // the handle for the lifetime of the loop keeps per-frame cost to the + // actual pixel capture + encode. + let source = match find_source(&source_id) { + Some(s) => s, + None => { + eprintln!("screen-capture {capture_id}: source vanished before capture started"); + return; + } + }; + + while !stop.load(Ordering::Relaxed) { + let frame_start = Instant::now(); + + let img_result = match &source { + Source::Window(w) => w.capture_image(), + Source::Monitor(m) => m.capture_image(), + }; + let img = match img_result { + Ok(i) => i, + Err(err) => { + eprintln!("screen-capture {capture_id}: capture failed: {err}"); + // Brief backoff before retrying — transient Windows GDI + // errors (e.g. during screen lock) tend to recover within + // a frame or two. + thread::sleep(frame_interval); + continue; + } + }; + + let (raw_w, raw_h) = img.dimensions(); + let (tgt_w, tgt_h) = scale_to_fit(raw_w, raw_h, max_w, max_h); + let scaled = if (tgt_w, tgt_h) != (raw_w, raw_h) { + image::imageops::resize( + &img, + tgt_w, + tgt_h, + image::imageops::FilterType::Triangle, + ) + } else { + img + }; + + // JPEG doesn't support alpha; strip it before encoding. + let rgb = rgba_to_rgb(&scaled); + + let mut jpeg_buf: Vec = Vec::with_capacity((tgt_w * tgt_h) as usize); + { + use image::codecs::jpeg::JpegEncoder; + let mut encoder = JpegEncoder::new_with_quality(&mut jpeg_buf, jpeg_quality); + if let Err(err) = + encoder.encode(&rgb, tgt_w, tgt_h, image::ExtendedColorType::Rgb8) + { + eprintln!("screen-capture {capture_id}: encode failed: {err}"); + thread::sleep(frame_interval); + continue; + } + } + + let jpeg_base64 = base64::engine::general_purpose::STANDARD.encode(&jpeg_buf); + + let send_result = channel.send(FramePayload { + capture_id, + width: tgt_w, + height: tgt_h, + jpeg_base64, + }); + if send_result.is_err() { + // Frontend went away (window closed, renderer crashed). + break; + } + + // Pace to target framerate. If the capture + encode already took + // longer than one frame interval, yield a millisecond to avoid + // pegging a single core when the target is unreachable. + let elapsed = frame_start.elapsed(); + if elapsed < frame_interval { + thread::sleep(frame_interval - elapsed); + } else { + thread::sleep(Duration::from_millis(1)); + } + } +} + +fn rgba_to_rgb(buf: &image::RgbaImage) -> Vec { + let (w, h) = buf.dimensions(); + let mut out = Vec::with_capacity((w * h * 3) as usize); + for p in buf.pixels() { + out.extend_from_slice(&[p.0[0], p.0[1], p.0[2]]); + } + out +} + +fn scale_to_fit(w: u32, h: u32, max_w: u32, max_h: u32) -> (u32, u32) { + if w == 0 || h == 0 { + return (w, h); + } + let scale = (max_w as f32 / w as f32) + .min(max_h as f32 / h as f32) + .min(1.0); + if scale >= 1.0 { + return (w, h); + } + let new_w = ((w as f32) * scale).round().max(1.0) as u32; + let new_h = ((h as f32) * scale).round().max(1.0) as u32; + (new_w, new_h) +} + +fn find_source(source_id: &str) -> Option { + if let Some(rest) = source_id.strip_prefix("window:") { + let raw = rest.strip_suffix(":0")?; + let raw_id: u32 = raw.parse().ok()?; + let windows = xcap::Window::all().ok()?; + return windows + .into_iter() + .find(|w| w.id() == raw_id) + .map(Source::Window); + } + if let Some(rest) = source_id.strip_prefix("screen:") { + let raw = rest.strip_suffix(":0")?; + let raw_id: u32 = raw.parse().ok()?; + let monitors = xcap::Monitor::all().ok()?; + return monitors + .into_iter() + .find(|m| m.id() == raw_id) + .map(Source::Monitor); + } + None +} diff --git a/apps/desktop/src-tauri/src/screen_sources.rs b/apps/desktop/src-tauri/src/screen_sources.rs index 72aed8d..d50f865 100644 --- a/apps/desktop/src-tauri/src/screen_sources.rs +++ b/apps/desktop/src-tauri/src/screen_sources.rs @@ -1,13 +1,18 @@ -// Source enumeration for the Discord-style screen-share picker. Returns a -// flat list of screens + windows with small PNG thumbnails so the JS -// picker can render a grid without the browser's native picker. +// Source enumeration for the Discord-style screen-share picker. Split +// into two commands on the slow-vs-fast axis: // +// - list_screen_sources — metadata only (no thumbnails). Fast; the +// picker shows names + placeholders instantly. +// - capture_screen_source_thumbnail — one thumbnail at a time, keyed by +// the id returned from the list. +// +// The frontend fans out the thumbnail calls via Promise.all so Tauri's +// command thread pool captures them in parallel — wall-clock time ends up +// bounded by the *slowest* source rather than the sum of all captures. // The `id` field is emitted in Chromium's internal desktopCapturer format // ("screen::0" / "window::0") so the JS side can try to pass it -// straight into getUserMedia's `chromeMediaSourceId` constraint. That -// bypass may or may not be accepted by WebView2 depending on the version -// — if it isn't, the JS falls back to the ordinary getDisplayMedia flow -// and at least the user has seen an informed preview first. +// straight into getUserMedia's `chromeMediaSourceId` constraint, or use +// it as the source key for the native capture pipeline in screen_capture. // // xcap abstracts the platform-specific capture APIs (Windows GDI + DXGI, // macOS CoreGraphics/ScreenCaptureKit, X11) so the code here stays flat. @@ -40,6 +45,39 @@ pub struct ScreenSource { pub height: u32, } +// Fast metadata-only enumeration. No image capture happens here — that's +// why it returns in tens of milliseconds instead of the multi-second +// wait the single-shot enumerate_screen_sources command had. +#[tauri::command] +pub fn list_screen_sources() -> Result, String> { + let mut out: Vec = Vec::new(); + append_monitor_metadata(&mut out); + append_window_metadata(&mut out); + Ok(out) +} + +// One-shot thumbnail capture by source id. Called N times in parallel from +// the frontend after the list lands so the total wall-clock is bounded by +// the slowest capture rather than the sum. +#[tauri::command] +pub fn capture_screen_source_thumbnail(source_id: String) -> Result, String> { + if let Some(raw) = source_id.strip_prefix("screen:").and_then(strip_zero_suffix) { + return Ok(capture_monitor_by_id(raw)); + } + if let Some(raw) = source_id.strip_prefix("window:").and_then(strip_zero_suffix) { + return Ok(capture_window_by_id(raw)); + } + Err(format!("unknown source id format: {source_id}")) +} + +fn strip_zero_suffix(s: &str) -> Option<&str> { + s.strip_suffix(":0") +} + +// Legacy one-shot all-in-one enumeration. Kept around so the frontend can +// fall back during rollout if the split pair throws; marked dead_code so +// the linker doesn't grumble when only the split variant is wired up. +#[allow(dead_code)] #[tauri::command] pub fn enumerate_screen_sources() -> Result, String> { let mut out: Vec = Vec::new(); @@ -52,6 +90,27 @@ pub fn enumerate_screen_sources() -> Result, String> { // Monitors // --------------------------------------------------------------------------- +fn append_monitor_metadata(out: &mut Vec) { + let monitors = match xcap::Monitor::all() { + Ok(m) => m, + Err(err) => { + eprintln!("xcap Monitor::all failed: {err}"); + return; + } + }; + for (idx, m) in monitors.iter().enumerate() { + out.push(ScreenSource { + id: format!("screen:{}:0", m.id()), + name: monitor_label(m, idx), + kind: "screen", + thumbnail_png: None, + width: m.width(), + height: m.height(), + }); + } +} + +#[allow(dead_code)] fn append_monitors(out: &mut Vec) { let monitors = match xcap::Monitor::all() { Ok(m) => m, @@ -61,25 +120,24 @@ fn append_monitors(out: &mut Vec) { } }; for (idx, m) in monitors.iter().enumerate() { - let name = monitor_label(m, idx); - let width = m.width(); - let height = m.height(); - // xcap exposes an OS-level monitor id; pass it through as the - // middle component so the JS side can correlate repeat enumerations. - let raw_id = m.id(); - let id = format!("screen:{raw_id}:0"); - let thumb = capture_monitor_thumbnail(m); out.push(ScreenSource { - id, - name, + id: format!("screen:{}:0", m.id()), + name: monitor_label(m, idx), kind: "screen", - thumbnail_png: thumb, - width, - height, + thumbnail_png: capture_monitor_thumbnail(m), + width: m.width(), + height: m.height(), }); } } +fn capture_monitor_by_id(raw: &str) -> Option { + let raw_id: u32 = raw.parse().ok()?; + let monitors = xcap::Monitor::all().ok()?; + let target = monitors.into_iter().find(|m| m.id() == raw_id)?; + capture_monitor_thumbnail(&target) +} + fn monitor_label(m: &xcap::Monitor, idx: usize) -> String { let name = m.name(); if name.is_empty() { @@ -98,6 +156,34 @@ fn capture_monitor_thumbnail(m: &xcap::Monitor) -> Option { // Windows // --------------------------------------------------------------------------- +fn append_window_metadata(out: &mut Vec) { + let windows = match xcap::Window::all() { + Ok(w) => w, + Err(err) => { + eprintln!("xcap Window::all failed: {err}"); + return; + } + }; + for w in windows.iter() { + if !is_shareable_window(w) { + continue; + } + let title = w.title(); + if title.trim().is_empty() { + continue; + } + out.push(ScreenSource { + id: format!("window:{}:0", w.id()), + name: title.to_string(), + kind: "window", + thumbnail_png: None, + width: w.width(), + height: w.height(), + }); + } +} + +#[allow(dead_code)] fn append_windows(out: &mut Vec) { let windows = match xcap::Window::all() { Ok(w) => w, @@ -114,22 +200,24 @@ fn append_windows(out: &mut Vec) { if title.trim().is_empty() { continue; } - let width = w.width(); - let height = w.height(); - let raw_id = w.id(); - let id = format!("window:{raw_id}:0"); - let thumb = capture_window_thumbnail(w); out.push(ScreenSource { - id, + id: format!("window:{}:0", w.id()), name: title.to_string(), kind: "window", - thumbnail_png: thumb, - width, - height, + thumbnail_png: capture_window_thumbnail(w), + width: w.width(), + height: w.height(), }); } } +fn capture_window_by_id(raw: &str) -> Option { + let raw_id: u32 = raw.parse().ok()?; + let windows = xcap::Window::all().ok()?; + let target = windows.into_iter().find(|w| w.id() == raw_id)?; + capture_window_thumbnail(&target) +} + fn is_shareable_window(w: &xcap::Window) -> bool { if w.is_minimized() { return false; diff --git a/apps/desktop/src/components/ScreenSourcePicker.tsx b/apps/desktop/src/components/ScreenSourcePicker.tsx index 1da4c79..a380304 100644 --- a/apps/desktop/src/components/ScreenSourcePicker.tsx +++ b/apps/desktop/src/components/ScreenSourcePicker.tsx @@ -10,7 +10,8 @@ import { updateScreenShareSettings, } from '../lib/screenShareSettings'; import { - enumerateScreenSources, + captureScreenSourceThumbnail, + listScreenSources, type ScreenSource, thumbnailDataUrl, } from '../lib/screenSources'; @@ -47,9 +48,11 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) { const [busy, setBusy] = useState(false); const [error, setError] = useState(null); - // Re-enumerate every time the picker opens so closed windows + new ones - // stay accurate. A previous stale list would surface sources the user - // can't actually share anymore. + // Two-phase load: (1) fast list returns names + placeholders so the grid + // paints instantly, (2) fire one thumbnail capture per source in + // parallel. Thumbnails fill in as each capture completes — Tauri's + // command thread pool runs them concurrently so wall-clock is bounded + // by the slowest source, not the sum. useEffect(() => { if (!open) { setSources(null); @@ -59,9 +62,27 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) { } let cancelled = false; void (async () => { - const list = await enumerateScreenSources(); + const list = await listScreenSources(); if (cancelled) return; setSources(list); + // Fan out thumbnail captures. No Promise.all — we want each result + // to render as it lands, not wait for the full batch. The id-based + // setState patch means the slowest source can still be in flight + // while the user already picked one of the fast ones. + for (const src of list) { + void (async () => { + const png = await captureScreenSourceThumbnail(src.id); + if (cancelled || png === null) return; + setSources((prev) => { + if (!prev) return prev; + const idx = prev.findIndex((s) => s.id === src.id); + if (idx === -1) return prev; + const next = prev.slice(); + next[idx] = { ...prev[idx]!, thumbnailPng: png }; + return next; + }); + })(); + } })(); return () => { cancelled = true; diff --git a/apps/desktop/src/context/CallContext.tsx b/apps/desktop/src/context/CallContext.tsx index 87479cb..e505609 100644 --- a/apps/desktop/src/context/CallContext.tsx +++ b/apps/desktop/src/context/CallContext.tsx @@ -75,6 +75,11 @@ import { setAllPipelinesSinkId, setPipelineGain, } from '../lib/remoteAudioPipelines'; +import { + type NativeCaptureHandle, + NativeCaptureUnavailable, + startNativeCapture, +} from '../lib/screenCapture'; import { clearScreenShareVolumes, getScreenShareVolume, @@ -295,6 +300,11 @@ export function CallProvider({ children }: { children: ReactNode }) { // to `connected` after a few seconds so the UI doesn't hang in "Verbinde…" // indefinitely. The solo-timeout will then cleanly close if nobody arrives. const joinFallbackTimerRef = useRef(null); + // Active native screen-capture handle (Rust side). Set when + // startScreenShare takes the xcap path; cleared on stopScreenShare or + // on the canvas track's 'ended' event. Not kept in React state because + // it never feeds into a render. + const nativeCaptureRef = useRef(null); const roomRef = useRef(null); // Web Audio graph that mixes live mic + soundboard sources into a single // published track. Created per call in joinRoom, destroyed in disconnectRoom. @@ -469,6 +479,13 @@ export function CallProvider({ children }: { children: ReactNode }) { /* ignore */ } } + // Stop any lingering native screen-capture thread so we don't leak + // Rust threads when the call ends mid-share. + if (nativeCaptureRef.current) { + const h = nativeCaptureRef.current; + nativeCaptureRef.current = null; + await h.stop().catch(() => undefined); + } roomRef.current = null; setRoom(null); setRemoteParticipants([]); @@ -1172,6 +1189,69 @@ export function CallProvider({ children }: { children: ReactNode }) { const fps = framerateOverride ?? ssParams.framerate; const sourceId = overrides?.sourceId ?? null; + // Native capture path — tried first when the user came in via our + // custom picker. xcap on the Rust side grabs frames, streams them as + // JPEG over a Tauri channel, and we draw them onto a canvas whose + // captureStream() becomes the MediaStream LiveKit publishes. This + // completely skips the OS picker. Video-only (no system audio yet); + // if the user asked for audio we fall through to the legacy paths + // below so audio still works via getDisplayMedia. + if (sourceId && !settings.includeSystemAudio) { + try { + const { Track: LkTrack } = await import('livekit-client'); + const maxWidth = ssParams.dims?.width ?? 1920; + const maxHeight = ssParams.dims?.height ?? 1080; + const handle = await startNativeCapture({ + sourceId, + maxWidth, + maxHeight, + fps, + }); + nativeCaptureRef.current = handle; + const videoMst = handle.stream.getVideoTracks()[0]; + if (!videoMst) { + await handle.stop(); + nativeCaptureRef.current = null; + throw new NativeCaptureUnavailable('canvas stream produced no video track'); + } + const videoPub = await lp.publishTrack(videoMst, { + source: LkTrack.Source.ScreenShare, + videoCodec: 'vp9', + }); + // Canvas stream 'ended' fires on handle.stop() (we track.stop() + // each track) — chain unpublish + native teardown so one ended + // event cleans everything up regardless of who triggered it. + videoMst.addEventListener('ended', () => { + void (async () => { + try { + if (videoPub.track) await lp.unpublishTrack(videoPub.track); + } catch { + /* already unpublished */ + } + const active = nativeCaptureRef.current; + if (active && active.captureId === handle.captureId) { + nativeCaptureRef.current = null; + await active.stop().catch(() => undefined); + } + setIsScreenSharing(false); + })(); + }); + setIsScreenSharing(true); + return; + } catch (err: unknown) { + // Native path unavailable (non-Tauri runtime, source vanished, + // first-frame timeout). Clean up any partial handle and fall + // through to the getUserMedia / getDisplayMedia paths. + if (nativeCaptureRef.current) { + await nativeCaptureRef.current.stop().catch(() => undefined); + nativeCaptureRef.current = null; + } + if (!(err instanceof NativeCaptureUnavailable)) { + console.warn('native screen-capture failed, falling back', err); + } + } + } + // Direct-publish path when our custom picker supplied a Chromium- // format source id. Bypasses the OS picker so the user shares exactly // the window/monitor they clicked in the grid. getUserMedia with the @@ -1299,16 +1379,42 @@ export function CallProvider({ children }: { children: ReactNode }) { ); const stopScreenShare = useCallback(async () => { + // Native path first — stopping the handle kills the canvas track, + // which fires 'ended' on the MediaStreamTrack, which the start-handler + // already listens to for unpublishing and flipping isScreenSharing. + if (nativeCaptureRef.current) { + const h = nativeCaptureRef.current; + nativeCaptureRef.current = null; + try { + await h.stop(); + } catch (err: unknown) { + console.warn('native capture stop failed', err); + } + } const r = roomRef.current; if (!r) return; const lp = r.localParticipant; - if (!lp.isScreenShareEnabled) return; - try { - await lp.setScreenShareEnabled(false); - setIsScreenSharing(false); - } catch (err: unknown) { - console.error('stopScreenShare failed', err); + // Unpublish any manually-published ScreenShare/ScreenShareAudio tracks + // (from the chromeMediaSourceId fallback path). setScreenShareEnabled + // only manages LK's own internally-captured tracks. + const toUnpublish: import('livekit-client').LocalTrackPublication[] = []; + for (const pub of lp.videoTrackPublications.values()) { + if (pub.source === Track.Source.ScreenShare && pub.track) toUnpublish.push(pub); } + for (const pub of lp.audioTrackPublications.values()) { + if (pub.source === Track.Source.ScreenShareAudio && pub.track) toUnpublish.push(pub); + } + for (const pub of toUnpublish) { + if (pub.track) await lp.unpublishTrack(pub.track).catch(() => undefined); + } + if (lp.isScreenShareEnabled) { + try { + await lp.setScreenShareEnabled(false); + } catch (err: unknown) { + console.error('stopScreenShare failed', err); + } + } + setIsScreenSharing(false); }, []); // Legacy toggle kept for convenience elsewhere — opens/closes with the diff --git a/apps/desktop/src/lib/screenCapture.ts b/apps/desktop/src/lib/screenCapture.ts new file mode 100644 index 0000000..3165cfb --- /dev/null +++ b/apps/desktop/src/lib/screenCapture.ts @@ -0,0 +1,184 @@ +// Frontend side of the native screen-capture pipeline. Starts a Rust-side +// capture thread via `start_screen_capture` and streams JPEG frames back +// through a Tauri Channel. Each frame is decoded into an ImageBitmap, +// drawn onto an offscreen canvas, and the canvas' captureStream() is +// returned as a MediaStream that LiveKit can publishTrack() directly — +// no OS/browser screen picker is involved. +// +// Video-only: system audio would require WASAPI / ScreenCaptureKit hooks +// that xcap doesn't provide. Callers that request shared audio must +// either fall back to the browser picker or accept video-without-audio. + +import { isTauriRuntime } from './globalShortcut'; + +export interface NativeCaptureHandle { + /** Rust-side capture id. Pass to `stopNativeCapture` to tear down. */ + captureId: number; + /** MediaStream fed by a canvas that's drawing each incoming frame. */ + stream: MediaStream; + /** Cleanup — stops the Rust thread, closes channels, revokes the canvas + * stream. Idempotent. */ + stop: () => Promise; +} + +interface FramePayload { + captureId: number; + width: number; + height: number; + jpegBase64: string; +} + +/** Returned when the runtime can't support native capture (no Tauri, no + * WebAudio, source vanished between enumeration and start, etc.). The + * caller is expected to fall back to the browser's getDisplayMedia path. */ +export class NativeCaptureUnavailable extends Error { + constructor(reason: string) { + super('native capture unavailable: ' + reason); + this.name = 'NativeCaptureUnavailable'; + } +} + +export async function startNativeCapture(opts: { + sourceId: string; + maxWidth: number; + maxHeight: number; + fps: number; +}): Promise { + if (!isTauriRuntime()) { + throw new NativeCaptureUnavailable('not a tauri runtime'); + } + + const canvas = document.createElement('canvas'); + canvas.width = opts.maxWidth; + canvas.height = opts.maxHeight; + const ctx = canvas.getContext('2d'); + if (!ctx) { + throw new NativeCaptureUnavailable('canvas 2d context unavailable'); + } + + // Track whether we got the first frame so we can fail fast if Rust + // reports "found" but then produces no output (e.g. screen was locked). + let firstFrameResolved = false; + let firstFrameResolve!: () => void; + let firstFrameReject!: (err: Error) => void; + const firstFramePromise = new Promise((resolve, reject) => { + firstFrameResolve = resolve; + firstFrameReject = reject; + }); + + const { Channel, invoke } = await import('@tauri-apps/api/core'); + const channel = new Channel(); + + // Latest-wins frame queue: if the JS side falls behind the Rust producer, + // we drop stale frames rather than queue them. Keeps memory flat and + // latency sensible for live screenshare. + let pendingFrame: FramePayload | null = null; + let decoding = false; + + const drainQueue = async () => { + if (decoding) return; + decoding = true; + try { + while (pendingFrame) { + const frame = pendingFrame; + pendingFrame = null; + const bytes = base64ToBytes(frame.jpegBase64); + // Uint8Array's buffer type is `ArrayBufferLike` (could be a + // SharedArrayBuffer in theory); Blob wants plain ArrayBuffer. + // Pass the underlying buffer explicitly so the type narrows. + const blob = new Blob([bytes.buffer as ArrayBuffer], { type: 'image/jpeg' }); + let bitmap: ImageBitmap; + try { + bitmap = await createImageBitmap(blob); + } catch (err: unknown) { + console.warn('createImageBitmap failed', err); + continue; + } + if (canvas.width !== bitmap.width || canvas.height !== bitmap.height) { + canvas.width = bitmap.width; + canvas.height = bitmap.height; + } + ctx.drawImage(bitmap, 0, 0); + bitmap.close(); + if (!firstFrameResolved) { + firstFrameResolved = true; + firstFrameResolve(); + } + } + } finally { + decoding = false; + } + }; + + channel.onmessage = (frame: FramePayload) => { + pendingFrame = frame; + void drainQueue(); + }; + + let captureId: number; + try { + captureId = await invoke('start_screen_capture', { + sourceId: opts.sourceId, + maxWidth: opts.maxWidth, + maxHeight: opts.maxHeight, + fps: opts.fps, + channel, + }); + } catch (err: unknown) { + throw new NativeCaptureUnavailable( + err instanceof Error ? err.message : String(err), + ); + } + + // Bound the wait: the capture thread may fail silently on some sources + // (locked screens, protected windows). Fall back to getDisplayMedia in + // that case rather than hang the user. + const firstFrameTimeout = window.setTimeout(() => { + firstFrameReject(new Error('first frame timed out (3s)')); + }, 3000); + try { + await firstFramePromise; + } catch (err: unknown) { + window.clearTimeout(firstFrameTimeout); + try { + await invoke('stop_screen_capture', { captureId }); + } catch { + /* ignore */ + } + throw new NativeCaptureUnavailable( + err instanceof Error ? err.message : String(err), + ); + } + window.clearTimeout(firstFrameTimeout); + + const stream = canvas.captureStream(opts.fps); + + let stopped = false; + const stop = async (): Promise => { + if (stopped) return; + stopped = true; + try { + await invoke('stop_screen_capture', { captureId }); + } catch (err: unknown) { + console.warn('stop_screen_capture failed', err); + } + for (const track of stream.getTracks()) { + try { + track.stop(); + } catch { + /* already stopped */ + } + } + }; + + return { captureId, stream, stop }; +} + +function base64ToBytes(b64: string): Uint8Array { + const bin = atob(b64); + const bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) { + bytes[i] = bin.charCodeAt(i); + } + return bytes; +} diff --git a/apps/desktop/src/lib/screenSources.ts b/apps/desktop/src/lib/screenSources.ts index 82d42d4..fead693 100644 --- a/apps/desktop/src/lib/screenSources.ts +++ b/apps/desktop/src/lib/screenSources.ts @@ -18,6 +18,44 @@ export interface ScreenSource { height: number; } +// Fast, metadata-only list. The picker uses this first so names show up +// immediately; thumbnails stream in via captureScreenSourceThumbnail below. +export async function listScreenSources(): Promise { + if (!isTauriRuntime()) return []; + try { + const { invoke } = await import('@tauri-apps/api/core'); + const raw = await invoke('list_screen_sources'); + return raw ?? []; + } catch (err: unknown) { + console.warn('list_screen_sources failed', err); + return []; + } +} + +// Single-source thumbnail capture. Called N times in parallel from the +// picker so Tauri's command thread pool runs captures concurrently — total +// wall-clock time becomes bounded by the slowest source, not the sum. +// Returns the base64 PNG or null when the source disappeared / capture +// permission was denied. +export async function captureScreenSourceThumbnail( + sourceId: string, +): Promise { + if (!isTauriRuntime()) return null; + try { + const { invoke } = await import('@tauri-apps/api/core'); + const result = await invoke('capture_screen_source_thumbnail', { + sourceId, + }); + return result ?? null; + } catch (err: unknown) { + console.warn('capture_screen_source_thumbnail failed', { sourceId, err }); + return null; + } +} + +// Legacy single-shot variant. Captures everything serially on the Rust side +// before returning. Prefer listScreenSources + captureScreenSourceThumbnail +// for user-facing flows — they feel 5–10× more responsive in practice. export async function enumerateScreenSources(): Promise { if (!isTauriRuntime()) return []; try {