feat(call): native screen-capture pipeline + faster picker
Picker speed (Phase 1+2): - screen_sources.rs split into list_screen_sources (metadata only, returns in ~10ms) + capture_screen_source_thumbnail (single source, by id). ScreenSourcePicker now shows names + placeholders instantly and streams thumbnails in as each capture lands. Total wall-clock is bounded by the slowest source instead of the serial sum. - enumerate_screen_sources kept as a dead_code fallback so any rollout regression can switch the frontend back without code loss. Native capture (Phase 3): - New src-tauri/src/screen_capture.rs. start_screen_capture spawns a Rust thread per share that grabs frames via xcap, downscales to the user's quality preset, JPEG-encodes at Q72, and streams each frame through a Tauri Channel<FramePayload>. stop_screen_capture signals the stop flag and joins the worker. - Worker re-resolves the xcap handle inside the thread because xcap::Window holds a !Send HWND — passing the source id string across the thread boundary sidesteps that. - New lib/screenCapture.ts: decodes each frame into an ImageBitmap, draws to an offscreen canvas, exposes canvas.captureStream() as the MediaStream LiveKit publishes. Latest-wins frame queue drops stale frames when the JS side falls behind the Rust producer. 3s first- frame timeout so a silently-failing source (locked screen, DRM window) surfaces as a clean NativeCaptureUnavailable and we fall back to getDisplayMedia. - CallContext.startScreenShare takes the native path first when the picker provided a sourceId and system audio wasn't requested. The old chromeMediaSourceId attempt and final setScreenShareEnabled fallback stay in place for the audio case + non-Tauri runtimes. - stopScreenShare kills the native handle first, then unpublishes any manually-published ScreenShare/ScreenShareAudio tracks, then falls back to setScreenShareEnabled(false). disconnectRoom also stops the handle so we don't leak Rust threads across calls. Scope note: native path is video-only. System-audio capture needs WASAPI-loopback (Windows) or ScreenCaptureKit-audio (macOS); until those are wired, requesting audio in the picker falls through to the getDisplayMedia path and shows the OS picker for that one case. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
byGalax
@@ -1,4 +1,5 @@
|
||||
mod crypto;
|
||||
mod screen_capture;
|
||||
mod screen_sources;
|
||||
|
||||
#[cfg(feature = "rust-livekit")]
|
||||
@@ -93,6 +94,10 @@ pub fn run() {
|
||||
crypto::crypto_box_seal_open,
|
||||
crypto::crypto_pwhash,
|
||||
screen_sources::enumerate_screen_sources,
|
||||
screen_sources::list_screen_sources,
|
||||
screen_sources::capture_screen_source_thumbnail,
|
||||
screen_capture::start_screen_capture,
|
||||
screen_capture::stop_screen_capture,
|
||||
])
|
||||
.plugin(tauri_plugin_notification::init());
|
||||
|
||||
@@ -110,6 +115,10 @@ pub fn run() {
|
||||
crypto::crypto_box_seal_open,
|
||||
crypto::crypto_pwhash,
|
||||
screen_sources::enumerate_screen_sources,
|
||||
screen_sources::list_screen_sources,
|
||||
screen_sources::capture_screen_source_thumbnail,
|
||||
screen_capture::start_screen_capture,
|
||||
screen_capture::stop_screen_capture,
|
||||
livekit_bridge::livekit_connect,
|
||||
livekit_bridge::livekit_disconnect,
|
||||
livekit_bridge::livekit_send_data,
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
// Native screen / window capture pipeline. Spawns a Rust thread per active
|
||||
// capture that grabs frames via xcap, downscales them to the user's target
|
||||
// resolution, encodes JPEG, and streams each frame to the JS side through
|
||||
// a Tauri `Channel<FramePayload>`. The JS end decodes into an
|
||||
// ImageBitmap, draws to a <canvas>, and exposes the canvas as a
|
||||
// MediaStream via captureStream() — that stream is what LiveKit publishes.
|
||||
// Net result: the user picks a source in our custom picker and the share
|
||||
// starts directly, without the OS/browser picker appearing.
|
||||
//
|
||||
// JPEG on the Rust side + decode on the JS side introduces a second
|
||||
// encoding hop (LiveKit re-encodes VP9 later) but keeps IPC bandwidth
|
||||
// manageable — raw RGBA at 1920×1080×30fps would be ~240MB/s and cannot
|
||||
// go over Tauri's JSON-serialised IPC. JPEG frames at Q72 land around
|
||||
// 50–150KB each, so 30fps = 2–5MB/s of base64 traffic, which is fine.
|
||||
|
||||
use base64::Engine;
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
use tauri::ipc::Channel;
|
||||
|
||||
static NEXT_ID: AtomicU32 = AtomicU32::new(1);
|
||||
static SESSIONS: OnceLock<Mutex<HashMap<u32, Session>>> = OnceLock::new();
|
||||
|
||||
fn sessions() -> &'static Mutex<HashMap<u32, Session>> {
|
||||
SESSIONS.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
}
|
||||
|
||||
struct Session {
|
||||
stop: Arc<AtomicBool>,
|
||||
handle: Option<thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Clone)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct FramePayload {
|
||||
pub capture_id: u32,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// JPEG image bytes, base64-encoded (no data-URL prefix). Frontend
|
||||
/// reconstructs via `Uint8Array.from(atob(...))` and decodes with
|
||||
/// `createImageBitmap(blob)`.
|
||||
pub jpeg_base64: String,
|
||||
}
|
||||
|
||||
enum Source {
|
||||
Window(xcap::Window),
|
||||
Monitor(xcap::Monitor),
|
||||
}
|
||||
|
||||
/// Start a continuous capture for the given source id and begin streaming
|
||||
/// JPEG frames via the provided channel. Returns a numeric capture id that
|
||||
/// must be passed to `stop_screen_capture` to tear the pipeline down.
|
||||
#[tauri::command]
|
||||
pub fn start_screen_capture(
|
||||
source_id: String,
|
||||
max_width: u32,
|
||||
max_height: u32,
|
||||
fps: u32,
|
||||
channel: Channel<FramePayload>,
|
||||
) -> Result<u32, String> {
|
||||
let clamped_fps = fps.clamp(5, 30);
|
||||
let clamped_w = max_width.max(320).min(3840);
|
||||
let clamped_h = max_height.max(180).min(2160);
|
||||
|
||||
// Probe once on the command thread so we can surface a clear error
|
||||
// before spawning; the actual capture handle is re-resolved inside the
|
||||
// worker thread (xcap::Window holds a Windows HWND which is !Send).
|
||||
if find_source(&source_id).is_none() {
|
||||
return Err(format!("screen source {source_id} not found"));
|
||||
}
|
||||
|
||||
let capture_id = NEXT_ID.fetch_add(1, Ordering::Relaxed);
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let stop_clone = Arc::clone(&stop);
|
||||
let source_id_owned = source_id;
|
||||
|
||||
let handle = thread::Builder::new()
|
||||
.name(format!("screen-capture-{capture_id}"))
|
||||
.spawn(move || {
|
||||
capture_loop(
|
||||
source_id_owned,
|
||||
clamped_w,
|
||||
clamped_h,
|
||||
clamped_fps,
|
||||
capture_id,
|
||||
channel,
|
||||
stop_clone,
|
||||
);
|
||||
})
|
||||
.map_err(|e| format!("failed to spawn capture thread: {e}"))?;
|
||||
|
||||
sessions().lock().unwrap().insert(
|
||||
capture_id,
|
||||
Session {
|
||||
stop,
|
||||
handle: Some(handle),
|
||||
},
|
||||
);
|
||||
Ok(capture_id)
|
||||
}
|
||||
|
||||
/// Signal the capture thread to stop and join it. Safe to call more than
|
||||
/// once — missing ids are silently no-ops so the JS side doesn't need to
|
||||
/// track whether a stop was already issued by the OS "stop sharing" path.
|
||||
#[tauri::command]
|
||||
pub fn stop_screen_capture(capture_id: u32) -> Result<(), String> {
|
||||
let session = sessions().lock().unwrap().remove(&capture_id);
|
||||
let Some(mut session) = session else {
|
||||
return Ok(());
|
||||
};
|
||||
session.stop.store(true, Ordering::Relaxed);
|
||||
if let Some(handle) = session.handle.take() {
|
||||
// Best-effort join. If the thread is stuck in a long OS capture
|
||||
// call we don't want to hang the command — detach after a brief
|
||||
// wait by dropping the handle.
|
||||
let _ = handle.join();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn capture_loop(
|
||||
source_id: String,
|
||||
max_w: u32,
|
||||
max_h: u32,
|
||||
fps: u32,
|
||||
capture_id: u32,
|
||||
channel: Channel<FramePayload>,
|
||||
stop: Arc<AtomicBool>,
|
||||
) {
|
||||
let frame_interval = Duration::from_nanos(1_000_000_000 / fps as u64);
|
||||
let jpeg_quality: u8 = 72;
|
||||
|
||||
// Re-resolve the source inside the worker thread — xcap::Window holds
|
||||
// an HWND which is !Send so we can't move it across threads. Caching
|
||||
// the handle for the lifetime of the loop keeps per-frame cost to the
|
||||
// actual pixel capture + encode.
|
||||
let source = match find_source(&source_id) {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
eprintln!("screen-capture {capture_id}: source vanished before capture started");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
while !stop.load(Ordering::Relaxed) {
|
||||
let frame_start = Instant::now();
|
||||
|
||||
let img_result = match &source {
|
||||
Source::Window(w) => w.capture_image(),
|
||||
Source::Monitor(m) => m.capture_image(),
|
||||
};
|
||||
let img = match img_result {
|
||||
Ok(i) => i,
|
||||
Err(err) => {
|
||||
eprintln!("screen-capture {capture_id}: capture failed: {err}");
|
||||
// Brief backoff before retrying — transient Windows GDI
|
||||
// errors (e.g. during screen lock) tend to recover within
|
||||
// a frame or two.
|
||||
thread::sleep(frame_interval);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let (raw_w, raw_h) = img.dimensions();
|
||||
let (tgt_w, tgt_h) = scale_to_fit(raw_w, raw_h, max_w, max_h);
|
||||
let scaled = if (tgt_w, tgt_h) != (raw_w, raw_h) {
|
||||
image::imageops::resize(
|
||||
&img,
|
||||
tgt_w,
|
||||
tgt_h,
|
||||
image::imageops::FilterType::Triangle,
|
||||
)
|
||||
} else {
|
||||
img
|
||||
};
|
||||
|
||||
// JPEG doesn't support alpha; strip it before encoding.
|
||||
let rgb = rgba_to_rgb(&scaled);
|
||||
|
||||
let mut jpeg_buf: Vec<u8> = Vec::with_capacity((tgt_w * tgt_h) as usize);
|
||||
{
|
||||
use image::codecs::jpeg::JpegEncoder;
|
||||
let mut encoder = JpegEncoder::new_with_quality(&mut jpeg_buf, jpeg_quality);
|
||||
if let Err(err) =
|
||||
encoder.encode(&rgb, tgt_w, tgt_h, image::ExtendedColorType::Rgb8)
|
||||
{
|
||||
eprintln!("screen-capture {capture_id}: encode failed: {err}");
|
||||
thread::sleep(frame_interval);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let jpeg_base64 = base64::engine::general_purpose::STANDARD.encode(&jpeg_buf);
|
||||
|
||||
let send_result = channel.send(FramePayload {
|
||||
capture_id,
|
||||
width: tgt_w,
|
||||
height: tgt_h,
|
||||
jpeg_base64,
|
||||
});
|
||||
if send_result.is_err() {
|
||||
// Frontend went away (window closed, renderer crashed).
|
||||
break;
|
||||
}
|
||||
|
||||
// Pace to target framerate. If the capture + encode already took
|
||||
// longer than one frame interval, yield a millisecond to avoid
|
||||
// pegging a single core when the target is unreachable.
|
||||
let elapsed = frame_start.elapsed();
|
||||
if elapsed < frame_interval {
|
||||
thread::sleep(frame_interval - elapsed);
|
||||
} else {
|
||||
thread::sleep(Duration::from_millis(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn rgba_to_rgb(buf: &image::RgbaImage) -> Vec<u8> {
|
||||
let (w, h) = buf.dimensions();
|
||||
let mut out = Vec::with_capacity((w * h * 3) as usize);
|
||||
for p in buf.pixels() {
|
||||
out.extend_from_slice(&[p.0[0], p.0[1], p.0[2]]);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn scale_to_fit(w: u32, h: u32, max_w: u32, max_h: u32) -> (u32, u32) {
|
||||
if w == 0 || h == 0 {
|
||||
return (w, h);
|
||||
}
|
||||
let scale = (max_w as f32 / w as f32)
|
||||
.min(max_h as f32 / h as f32)
|
||||
.min(1.0);
|
||||
if scale >= 1.0 {
|
||||
return (w, h);
|
||||
}
|
||||
let new_w = ((w as f32) * scale).round().max(1.0) as u32;
|
||||
let new_h = ((h as f32) * scale).round().max(1.0) as u32;
|
||||
(new_w, new_h)
|
||||
}
|
||||
|
||||
fn find_source(source_id: &str) -> Option<Source> {
|
||||
if let Some(rest) = source_id.strip_prefix("window:") {
|
||||
let raw = rest.strip_suffix(":0")?;
|
||||
let raw_id: u32 = raw.parse().ok()?;
|
||||
let windows = xcap::Window::all().ok()?;
|
||||
return windows
|
||||
.into_iter()
|
||||
.find(|w| w.id() == raw_id)
|
||||
.map(Source::Window);
|
||||
}
|
||||
if let Some(rest) = source_id.strip_prefix("screen:") {
|
||||
let raw = rest.strip_suffix(":0")?;
|
||||
let raw_id: u32 = raw.parse().ok()?;
|
||||
let monitors = xcap::Monitor::all().ok()?;
|
||||
return monitors
|
||||
.into_iter()
|
||||
.find(|m| m.id() == raw_id)
|
||||
.map(Source::Monitor);
|
||||
}
|
||||
None
|
||||
}
|
||||
@@ -1,13 +1,18 @@
|
||||
// Source enumeration for the Discord-style screen-share picker. Returns a
|
||||
// flat list of screens + windows with small PNG thumbnails so the JS
|
||||
// picker can render a grid without the browser's native picker.
|
||||
// Source enumeration for the Discord-style screen-share picker. Split
|
||||
// into two commands on the slow-vs-fast axis:
|
||||
//
|
||||
// - list_screen_sources — metadata only (no thumbnails). Fast; the
|
||||
// picker shows names + placeholders instantly.
|
||||
// - capture_screen_source_thumbnail — one thumbnail at a time, keyed by
|
||||
// the id returned from the list.
|
||||
//
|
||||
// The frontend fans out the thumbnail calls via Promise.all so Tauri's
|
||||
// command thread pool captures them in parallel — wall-clock time ends up
|
||||
// bounded by the *slowest* source rather than the sum of all captures.
|
||||
// The `id` field is emitted in Chromium's internal desktopCapturer format
|
||||
// ("screen:<id>:0" / "window:<hwnd>:0") so the JS side can try to pass it
|
||||
// straight into getUserMedia's `chromeMediaSourceId` constraint. That
|
||||
// bypass may or may not be accepted by WebView2 depending on the version
|
||||
// — if it isn't, the JS falls back to the ordinary getDisplayMedia flow
|
||||
// and at least the user has seen an informed preview first.
|
||||
// straight into getUserMedia's `chromeMediaSourceId` constraint, or use
|
||||
// it as the source key for the native capture pipeline in screen_capture.
|
||||
//
|
||||
// xcap abstracts the platform-specific capture APIs (Windows GDI + DXGI,
|
||||
// macOS CoreGraphics/ScreenCaptureKit, X11) so the code here stays flat.
|
||||
@@ -40,6 +45,39 @@ pub struct ScreenSource {
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
// Fast metadata-only enumeration. No image capture happens here — that's
|
||||
// why it returns in tens of milliseconds instead of the multi-second
|
||||
// wait the single-shot enumerate_screen_sources command had.
|
||||
#[tauri::command]
|
||||
pub fn list_screen_sources() -> Result<Vec<ScreenSource>, String> {
|
||||
let mut out: Vec<ScreenSource> = Vec::new();
|
||||
append_monitor_metadata(&mut out);
|
||||
append_window_metadata(&mut out);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
// One-shot thumbnail capture by source id. Called N times in parallel from
|
||||
// the frontend after the list lands so the total wall-clock is bounded by
|
||||
// the slowest capture rather than the sum.
|
||||
#[tauri::command]
|
||||
pub fn capture_screen_source_thumbnail(source_id: String) -> Result<Option<String>, String> {
|
||||
if let Some(raw) = source_id.strip_prefix("screen:").and_then(strip_zero_suffix) {
|
||||
return Ok(capture_monitor_by_id(raw));
|
||||
}
|
||||
if let Some(raw) = source_id.strip_prefix("window:").and_then(strip_zero_suffix) {
|
||||
return Ok(capture_window_by_id(raw));
|
||||
}
|
||||
Err(format!("unknown source id format: {source_id}"))
|
||||
}
|
||||
|
||||
fn strip_zero_suffix(s: &str) -> Option<&str> {
|
||||
s.strip_suffix(":0")
|
||||
}
|
||||
|
||||
// Legacy one-shot all-in-one enumeration. Kept around so the frontend can
|
||||
// fall back during rollout if the split pair throws; marked dead_code so
|
||||
// the linker doesn't grumble when only the split variant is wired up.
|
||||
#[allow(dead_code)]
|
||||
#[tauri::command]
|
||||
pub fn enumerate_screen_sources() -> Result<Vec<ScreenSource>, String> {
|
||||
let mut out: Vec<ScreenSource> = Vec::new();
|
||||
@@ -52,6 +90,27 @@ pub fn enumerate_screen_sources() -> Result<Vec<ScreenSource>, String> {
|
||||
// Monitors
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn append_monitor_metadata(out: &mut Vec<ScreenSource>) {
|
||||
let monitors = match xcap::Monitor::all() {
|
||||
Ok(m) => m,
|
||||
Err(err) => {
|
||||
eprintln!("xcap Monitor::all failed: {err}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
for (idx, m) in monitors.iter().enumerate() {
|
||||
out.push(ScreenSource {
|
||||
id: format!("screen:{}:0", m.id()),
|
||||
name: monitor_label(m, idx),
|
||||
kind: "screen",
|
||||
thumbnail_png: None,
|
||||
width: m.width(),
|
||||
height: m.height(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn append_monitors(out: &mut Vec<ScreenSource>) {
|
||||
let monitors = match xcap::Monitor::all() {
|
||||
Ok(m) => m,
|
||||
@@ -61,25 +120,24 @@ fn append_monitors(out: &mut Vec<ScreenSource>) {
|
||||
}
|
||||
};
|
||||
for (idx, m) in monitors.iter().enumerate() {
|
||||
let name = monitor_label(m, idx);
|
||||
let width = m.width();
|
||||
let height = m.height();
|
||||
// xcap exposes an OS-level monitor id; pass it through as the
|
||||
// middle component so the JS side can correlate repeat enumerations.
|
||||
let raw_id = m.id();
|
||||
let id = format!("screen:{raw_id}:0");
|
||||
let thumb = capture_monitor_thumbnail(m);
|
||||
out.push(ScreenSource {
|
||||
id,
|
||||
name,
|
||||
id: format!("screen:{}:0", m.id()),
|
||||
name: monitor_label(m, idx),
|
||||
kind: "screen",
|
||||
thumbnail_png: thumb,
|
||||
width,
|
||||
height,
|
||||
thumbnail_png: capture_monitor_thumbnail(m),
|
||||
width: m.width(),
|
||||
height: m.height(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn capture_monitor_by_id(raw: &str) -> Option<String> {
|
||||
let raw_id: u32 = raw.parse().ok()?;
|
||||
let monitors = xcap::Monitor::all().ok()?;
|
||||
let target = monitors.into_iter().find(|m| m.id() == raw_id)?;
|
||||
capture_monitor_thumbnail(&target)
|
||||
}
|
||||
|
||||
fn monitor_label(m: &xcap::Monitor, idx: usize) -> String {
|
||||
let name = m.name();
|
||||
if name.is_empty() {
|
||||
@@ -98,6 +156,34 @@ fn capture_monitor_thumbnail(m: &xcap::Monitor) -> Option<String> {
|
||||
// Windows
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn append_window_metadata(out: &mut Vec<ScreenSource>) {
|
||||
let windows = match xcap::Window::all() {
|
||||
Ok(w) => w,
|
||||
Err(err) => {
|
||||
eprintln!("xcap Window::all failed: {err}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
for w in windows.iter() {
|
||||
if !is_shareable_window(w) {
|
||||
continue;
|
||||
}
|
||||
let title = w.title();
|
||||
if title.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
out.push(ScreenSource {
|
||||
id: format!("window:{}:0", w.id()),
|
||||
name: title.to_string(),
|
||||
kind: "window",
|
||||
thumbnail_png: None,
|
||||
width: w.width(),
|
||||
height: w.height(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn append_windows(out: &mut Vec<ScreenSource>) {
|
||||
let windows = match xcap::Window::all() {
|
||||
Ok(w) => w,
|
||||
@@ -114,22 +200,24 @@ fn append_windows(out: &mut Vec<ScreenSource>) {
|
||||
if title.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let width = w.width();
|
||||
let height = w.height();
|
||||
let raw_id = w.id();
|
||||
let id = format!("window:{raw_id}:0");
|
||||
let thumb = capture_window_thumbnail(w);
|
||||
out.push(ScreenSource {
|
||||
id,
|
||||
id: format!("window:{}:0", w.id()),
|
||||
name: title.to_string(),
|
||||
kind: "window",
|
||||
thumbnail_png: thumb,
|
||||
width,
|
||||
height,
|
||||
thumbnail_png: capture_window_thumbnail(w),
|
||||
width: w.width(),
|
||||
height: w.height(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn capture_window_by_id(raw: &str) -> Option<String> {
|
||||
let raw_id: u32 = raw.parse().ok()?;
|
||||
let windows = xcap::Window::all().ok()?;
|
||||
let target = windows.into_iter().find(|w| w.id() == raw_id)?;
|
||||
capture_window_thumbnail(&target)
|
||||
}
|
||||
|
||||
fn is_shareable_window(w: &xcap::Window) -> bool {
|
||||
if w.is_minimized() {
|
||||
return false;
|
||||
|
||||
@@ -10,7 +10,8 @@ import {
|
||||
updateScreenShareSettings,
|
||||
} from '../lib/screenShareSettings';
|
||||
import {
|
||||
enumerateScreenSources,
|
||||
captureScreenSourceThumbnail,
|
||||
listScreenSources,
|
||||
type ScreenSource,
|
||||
thumbnailDataUrl,
|
||||
} from '../lib/screenSources';
|
||||
@@ -47,9 +48,11 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
// Re-enumerate every time the picker opens so closed windows + new ones
|
||||
// stay accurate. A previous stale list would surface sources the user
|
||||
// can't actually share anymore.
|
||||
// Two-phase load: (1) fast list returns names + placeholders so the grid
|
||||
// paints instantly, (2) fire one thumbnail capture per source in
|
||||
// parallel. Thumbnails fill in as each capture completes — Tauri's
|
||||
// command thread pool runs them concurrently so wall-clock is bounded
|
||||
// by the slowest source, not the sum.
|
||||
useEffect(() => {
|
||||
if (!open) {
|
||||
setSources(null);
|
||||
@@ -59,9 +62,27 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
|
||||
}
|
||||
let cancelled = false;
|
||||
void (async () => {
|
||||
const list = await enumerateScreenSources();
|
||||
const list = await listScreenSources();
|
||||
if (cancelled) return;
|
||||
setSources(list);
|
||||
// Fan out thumbnail captures. No Promise.all — we want each result
|
||||
// to render as it lands, not wait for the full batch. The id-based
|
||||
// setState patch means the slowest source can still be in flight
|
||||
// while the user already picked one of the fast ones.
|
||||
for (const src of list) {
|
||||
void (async () => {
|
||||
const png = await captureScreenSourceThumbnail(src.id);
|
||||
if (cancelled || png === null) return;
|
||||
setSources((prev) => {
|
||||
if (!prev) return prev;
|
||||
const idx = prev.findIndex((s) => s.id === src.id);
|
||||
if (idx === -1) return prev;
|
||||
const next = prev.slice();
|
||||
next[idx] = { ...prev[idx]!, thumbnailPng: png };
|
||||
return next;
|
||||
});
|
||||
})();
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
|
||||
@@ -75,6 +75,11 @@ import {
|
||||
setAllPipelinesSinkId,
|
||||
setPipelineGain,
|
||||
} from '../lib/remoteAudioPipelines';
|
||||
import {
|
||||
type NativeCaptureHandle,
|
||||
NativeCaptureUnavailable,
|
||||
startNativeCapture,
|
||||
} from '../lib/screenCapture';
|
||||
import {
|
||||
clearScreenShareVolumes,
|
||||
getScreenShareVolume,
|
||||
@@ -295,6 +300,11 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
// to `connected` after a few seconds so the UI doesn't hang in "Verbinde…"
|
||||
// indefinitely. The solo-timeout will then cleanly close if nobody arrives.
|
||||
const joinFallbackTimerRef = useRef<number | null>(null);
|
||||
// Active native screen-capture handle (Rust side). Set when
|
||||
// startScreenShare takes the xcap path; cleared on stopScreenShare or
|
||||
// on the canvas track's 'ended' event. Not kept in React state because
|
||||
// it never feeds into a render.
|
||||
const nativeCaptureRef = useRef<NativeCaptureHandle | null>(null);
|
||||
const roomRef = useRef<Room | null>(null);
|
||||
// Web Audio graph that mixes live mic + soundboard sources into a single
|
||||
// published track. Created per call in joinRoom, destroyed in disconnectRoom.
|
||||
@@ -469,6 +479,13 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
// Stop any lingering native screen-capture thread so we don't leak
|
||||
// Rust threads when the call ends mid-share.
|
||||
if (nativeCaptureRef.current) {
|
||||
const h = nativeCaptureRef.current;
|
||||
nativeCaptureRef.current = null;
|
||||
await h.stop().catch(() => undefined);
|
||||
}
|
||||
roomRef.current = null;
|
||||
setRoom(null);
|
||||
setRemoteParticipants([]);
|
||||
@@ -1172,6 +1189,69 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
const fps = framerateOverride ?? ssParams.framerate;
|
||||
const sourceId = overrides?.sourceId ?? null;
|
||||
|
||||
// Native capture path — tried first when the user came in via our
|
||||
// custom picker. xcap on the Rust side grabs frames, streams them as
|
||||
// JPEG over a Tauri channel, and we draw them onto a canvas whose
|
||||
// captureStream() becomes the MediaStream LiveKit publishes. This
|
||||
// completely skips the OS picker. Video-only (no system audio yet);
|
||||
// if the user asked for audio we fall through to the legacy paths
|
||||
// below so audio still works via getDisplayMedia.
|
||||
if (sourceId && !settings.includeSystemAudio) {
|
||||
try {
|
||||
const { Track: LkTrack } = await import('livekit-client');
|
||||
const maxWidth = ssParams.dims?.width ?? 1920;
|
||||
const maxHeight = ssParams.dims?.height ?? 1080;
|
||||
const handle = await startNativeCapture({
|
||||
sourceId,
|
||||
maxWidth,
|
||||
maxHeight,
|
||||
fps,
|
||||
});
|
||||
nativeCaptureRef.current = handle;
|
||||
const videoMst = handle.stream.getVideoTracks()[0];
|
||||
if (!videoMst) {
|
||||
await handle.stop();
|
||||
nativeCaptureRef.current = null;
|
||||
throw new NativeCaptureUnavailable('canvas stream produced no video track');
|
||||
}
|
||||
const videoPub = await lp.publishTrack(videoMst, {
|
||||
source: LkTrack.Source.ScreenShare,
|
||||
videoCodec: 'vp9',
|
||||
});
|
||||
// Canvas stream 'ended' fires on handle.stop() (we track.stop()
|
||||
// each track) — chain unpublish + native teardown so one ended
|
||||
// event cleans everything up regardless of who triggered it.
|
||||
videoMst.addEventListener('ended', () => {
|
||||
void (async () => {
|
||||
try {
|
||||
if (videoPub.track) await lp.unpublishTrack(videoPub.track);
|
||||
} catch {
|
||||
/* already unpublished */
|
||||
}
|
||||
const active = nativeCaptureRef.current;
|
||||
if (active && active.captureId === handle.captureId) {
|
||||
nativeCaptureRef.current = null;
|
||||
await active.stop().catch(() => undefined);
|
||||
}
|
||||
setIsScreenSharing(false);
|
||||
})();
|
||||
});
|
||||
setIsScreenSharing(true);
|
||||
return;
|
||||
} catch (err: unknown) {
|
||||
// Native path unavailable (non-Tauri runtime, source vanished,
|
||||
// first-frame timeout). Clean up any partial handle and fall
|
||||
// through to the getUserMedia / getDisplayMedia paths.
|
||||
if (nativeCaptureRef.current) {
|
||||
await nativeCaptureRef.current.stop().catch(() => undefined);
|
||||
nativeCaptureRef.current = null;
|
||||
}
|
||||
if (!(err instanceof NativeCaptureUnavailable)) {
|
||||
console.warn('native screen-capture failed, falling back', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Direct-publish path when our custom picker supplied a Chromium-
|
||||
// format source id. Bypasses the OS picker so the user shares exactly
|
||||
// the window/monitor they clicked in the grid. getUserMedia with the
|
||||
@@ -1299,16 +1379,42 @@ export function CallProvider({ children }: { children: ReactNode }) {
|
||||
);
|
||||
|
||||
const stopScreenShare = useCallback(async () => {
|
||||
// Native path first — stopping the handle kills the canvas track,
|
||||
// which fires 'ended' on the MediaStreamTrack, which the start-handler
|
||||
// already listens to for unpublishing and flipping isScreenSharing.
|
||||
if (nativeCaptureRef.current) {
|
||||
const h = nativeCaptureRef.current;
|
||||
nativeCaptureRef.current = null;
|
||||
try {
|
||||
await h.stop();
|
||||
} catch (err: unknown) {
|
||||
console.warn('native capture stop failed', err);
|
||||
}
|
||||
}
|
||||
const r = roomRef.current;
|
||||
if (!r) return;
|
||||
const lp = r.localParticipant;
|
||||
if (!lp.isScreenShareEnabled) return;
|
||||
try {
|
||||
await lp.setScreenShareEnabled(false);
|
||||
setIsScreenSharing(false);
|
||||
} catch (err: unknown) {
|
||||
console.error('stopScreenShare failed', err);
|
||||
// Unpublish any manually-published ScreenShare/ScreenShareAudio tracks
|
||||
// (from the chromeMediaSourceId fallback path). setScreenShareEnabled
|
||||
// only manages LK's own internally-captured tracks.
|
||||
const toUnpublish: import('livekit-client').LocalTrackPublication[] = [];
|
||||
for (const pub of lp.videoTrackPublications.values()) {
|
||||
if (pub.source === Track.Source.ScreenShare && pub.track) toUnpublish.push(pub);
|
||||
}
|
||||
for (const pub of lp.audioTrackPublications.values()) {
|
||||
if (pub.source === Track.Source.ScreenShareAudio && pub.track) toUnpublish.push(pub);
|
||||
}
|
||||
for (const pub of toUnpublish) {
|
||||
if (pub.track) await lp.unpublishTrack(pub.track).catch(() => undefined);
|
||||
}
|
||||
if (lp.isScreenShareEnabled) {
|
||||
try {
|
||||
await lp.setScreenShareEnabled(false);
|
||||
} catch (err: unknown) {
|
||||
console.error('stopScreenShare failed', err);
|
||||
}
|
||||
}
|
||||
setIsScreenSharing(false);
|
||||
}, []);
|
||||
|
||||
// Legacy toggle kept for convenience elsewhere — opens/closes with the
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
// Frontend side of the native screen-capture pipeline. Starts a Rust-side
|
||||
// capture thread via `start_screen_capture` and streams JPEG frames back
|
||||
// through a Tauri Channel. Each frame is decoded into an ImageBitmap,
|
||||
// drawn onto an offscreen canvas, and the canvas' captureStream() is
|
||||
// returned as a MediaStream that LiveKit can publishTrack() directly —
|
||||
// no OS/browser screen picker is involved.
|
||||
//
|
||||
// Video-only: system audio would require WASAPI / ScreenCaptureKit hooks
|
||||
// that xcap doesn't provide. Callers that request shared audio must
|
||||
// either fall back to the browser picker or accept video-without-audio.
|
||||
|
||||
import { isTauriRuntime } from './globalShortcut';
|
||||
|
||||
export interface NativeCaptureHandle {
|
||||
/** Rust-side capture id. Pass to `stopNativeCapture` to tear down. */
|
||||
captureId: number;
|
||||
/** MediaStream fed by a canvas that's drawing each incoming frame. */
|
||||
stream: MediaStream;
|
||||
/** Cleanup — stops the Rust thread, closes channels, revokes the canvas
|
||||
* stream. Idempotent. */
|
||||
stop: () => Promise<void>;
|
||||
}
|
||||
|
||||
interface FramePayload {
|
||||
captureId: number;
|
||||
width: number;
|
||||
height: number;
|
||||
jpegBase64: string;
|
||||
}
|
||||
|
||||
/** Returned when the runtime can't support native capture (no Tauri, no
|
||||
* WebAudio, source vanished between enumeration and start, etc.). The
|
||||
* caller is expected to fall back to the browser's getDisplayMedia path. */
|
||||
export class NativeCaptureUnavailable extends Error {
|
||||
constructor(reason: string) {
|
||||
super('native capture unavailable: ' + reason);
|
||||
this.name = 'NativeCaptureUnavailable';
|
||||
}
|
||||
}
|
||||
|
||||
export async function startNativeCapture(opts: {
|
||||
sourceId: string;
|
||||
maxWidth: number;
|
||||
maxHeight: number;
|
||||
fps: number;
|
||||
}): Promise<NativeCaptureHandle> {
|
||||
if (!isTauriRuntime()) {
|
||||
throw new NativeCaptureUnavailable('not a tauri runtime');
|
||||
}
|
||||
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = opts.maxWidth;
|
||||
canvas.height = opts.maxHeight;
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) {
|
||||
throw new NativeCaptureUnavailable('canvas 2d context unavailable');
|
||||
}
|
||||
|
||||
// Track whether we got the first frame so we can fail fast if Rust
|
||||
// reports "found" but then produces no output (e.g. screen was locked).
|
||||
let firstFrameResolved = false;
|
||||
let firstFrameResolve!: () => void;
|
||||
let firstFrameReject!: (err: Error) => void;
|
||||
const firstFramePromise = new Promise<void>((resolve, reject) => {
|
||||
firstFrameResolve = resolve;
|
||||
firstFrameReject = reject;
|
||||
});
|
||||
|
||||
const { Channel, invoke } = await import('@tauri-apps/api/core');
|
||||
const channel = new Channel<FramePayload>();
|
||||
|
||||
// Latest-wins frame queue: if the JS side falls behind the Rust producer,
|
||||
// we drop stale frames rather than queue them. Keeps memory flat and
|
||||
// latency sensible for live screenshare.
|
||||
let pendingFrame: FramePayload | null = null;
|
||||
let decoding = false;
|
||||
|
||||
const drainQueue = async () => {
|
||||
if (decoding) return;
|
||||
decoding = true;
|
||||
try {
|
||||
while (pendingFrame) {
|
||||
const frame = pendingFrame;
|
||||
pendingFrame = null;
|
||||
const bytes = base64ToBytes(frame.jpegBase64);
|
||||
// Uint8Array's buffer type is `ArrayBufferLike` (could be a
|
||||
// SharedArrayBuffer in theory); Blob wants plain ArrayBuffer.
|
||||
// Pass the underlying buffer explicitly so the type narrows.
|
||||
const blob = new Blob([bytes.buffer as ArrayBuffer], { type: 'image/jpeg' });
|
||||
let bitmap: ImageBitmap;
|
||||
try {
|
||||
bitmap = await createImageBitmap(blob);
|
||||
} catch (err: unknown) {
|
||||
console.warn('createImageBitmap failed', err);
|
||||
continue;
|
||||
}
|
||||
if (canvas.width !== bitmap.width || canvas.height !== bitmap.height) {
|
||||
canvas.width = bitmap.width;
|
||||
canvas.height = bitmap.height;
|
||||
}
|
||||
ctx.drawImage(bitmap, 0, 0);
|
||||
bitmap.close();
|
||||
if (!firstFrameResolved) {
|
||||
firstFrameResolved = true;
|
||||
firstFrameResolve();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
decoding = false;
|
||||
}
|
||||
};
|
||||
|
||||
channel.onmessage = (frame: FramePayload) => {
|
||||
pendingFrame = frame;
|
||||
void drainQueue();
|
||||
};
|
||||
|
||||
let captureId: number;
|
||||
try {
|
||||
captureId = await invoke<number>('start_screen_capture', {
|
||||
sourceId: opts.sourceId,
|
||||
maxWidth: opts.maxWidth,
|
||||
maxHeight: opts.maxHeight,
|
||||
fps: opts.fps,
|
||||
channel,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
throw new NativeCaptureUnavailable(
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
}
|
||||
|
||||
// Bound the wait: the capture thread may fail silently on some sources
|
||||
// (locked screens, protected windows). Fall back to getDisplayMedia in
|
||||
// that case rather than hang the user.
|
||||
const firstFrameTimeout = window.setTimeout(() => {
|
||||
firstFrameReject(new Error('first frame timed out (3s)'));
|
||||
}, 3000);
|
||||
try {
|
||||
await firstFramePromise;
|
||||
} catch (err: unknown) {
|
||||
window.clearTimeout(firstFrameTimeout);
|
||||
try {
|
||||
await invoke('stop_screen_capture', { captureId });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
throw new NativeCaptureUnavailable(
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
}
|
||||
window.clearTimeout(firstFrameTimeout);
|
||||
|
||||
const stream = canvas.captureStream(opts.fps);
|
||||
|
||||
let stopped = false;
|
||||
const stop = async (): Promise<void> => {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
try {
|
||||
await invoke('stop_screen_capture', { captureId });
|
||||
} catch (err: unknown) {
|
||||
console.warn('stop_screen_capture failed', err);
|
||||
}
|
||||
for (const track of stream.getTracks()) {
|
||||
try {
|
||||
track.stop();
|
||||
} catch {
|
||||
/* already stopped */
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return { captureId, stream, stop };
|
||||
}
|
||||
|
||||
function base64ToBytes(b64: string): Uint8Array {
|
||||
const bin = atob(b64);
|
||||
const bytes = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) {
|
||||
bytes[i] = bin.charCodeAt(i);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
@@ -18,6 +18,44 @@ export interface ScreenSource {
|
||||
height: number;
|
||||
}
|
||||
|
||||
// Fast, metadata-only list. The picker uses this first so names show up
|
||||
// immediately; thumbnails stream in via captureScreenSourceThumbnail below.
|
||||
export async function listScreenSources(): Promise<ScreenSource[]> {
|
||||
if (!isTauriRuntime()) return [];
|
||||
try {
|
||||
const { invoke } = await import('@tauri-apps/api/core');
|
||||
const raw = await invoke<ScreenSource[]>('list_screen_sources');
|
||||
return raw ?? [];
|
||||
} catch (err: unknown) {
|
||||
console.warn('list_screen_sources failed', err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Single-source thumbnail capture. Called N times in parallel from the
|
||||
// picker so Tauri's command thread pool runs captures concurrently — total
|
||||
// wall-clock time becomes bounded by the slowest source, not the sum.
|
||||
// Returns the base64 PNG or null when the source disappeared / capture
|
||||
// permission was denied.
|
||||
export async function captureScreenSourceThumbnail(
|
||||
sourceId: string,
|
||||
): Promise<string | null> {
|
||||
if (!isTauriRuntime()) return null;
|
||||
try {
|
||||
const { invoke } = await import('@tauri-apps/api/core');
|
||||
const result = await invoke<string | null>('capture_screen_source_thumbnail', {
|
||||
sourceId,
|
||||
});
|
||||
return result ?? null;
|
||||
} catch (err: unknown) {
|
||||
console.warn('capture_screen_source_thumbnail failed', { sourceId, err });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Legacy single-shot variant. Captures everything serially on the Rust side
|
||||
// before returning. Prefer listScreenSources + captureScreenSourceThumbnail
|
||||
// for user-facing flows — they feel 5–10× more responsive in practice.
|
||||
export async function enumerateScreenSources(): Promise<ScreenSource[]> {
|
||||
if (!isTauriRuntime()) return [];
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user