feat(call): native screen-capture pipeline + faster picker
Picker speed (Phase 1+2): - screen_sources.rs split into list_screen_sources (metadata only, returns in ~10ms) + capture_screen_source_thumbnail (single source, by id). ScreenSourcePicker now shows names + placeholders instantly and streams thumbnails in as each capture lands. Total wall-clock is bounded by the slowest source instead of the serial sum. - enumerate_screen_sources kept as a dead_code fallback so any rollout regression can switch the frontend back without code loss. Native capture (Phase 3): - New src-tauri/src/screen_capture.rs. start_screen_capture spawns a Rust thread per share that grabs frames via xcap, downscales to the user's quality preset, JPEG-encodes at Q72, and streams each frame through a Tauri Channel<FramePayload>. stop_screen_capture signals the stop flag and joins the worker. - Worker re-resolves the xcap handle inside the thread because xcap::Window holds a !Send HWND — passing the source id string across the thread boundary sidesteps that. - New lib/screenCapture.ts: decodes each frame into an ImageBitmap, draws to an offscreen canvas, exposes canvas.captureStream() as the MediaStream LiveKit publishes. Latest-wins frame queue drops stale frames when the JS side falls behind the Rust producer. 3s first- frame timeout so a silently-failing source (locked screen, DRM window) surfaces as a clean NativeCaptureUnavailable and we fall back to getDisplayMedia. - CallContext.startScreenShare takes the native path first when the picker provided a sourceId and system audio wasn't requested. The old chromeMediaSourceId attempt and final setScreenShareEnabled fallback stay in place for the audio case + non-Tauri runtimes. - stopScreenShare kills the native handle first, then unpublishes any manually-published ScreenShare/ScreenShareAudio tracks, then falls back to setScreenShareEnabled(false). disconnectRoom also stops the handle so we don't leak Rust threads across calls. Scope note: native path is video-only. System-audio capture needs WASAPI-loopback (Windows) or ScreenCaptureKit-audio (macOS); until those are wired, requesting audio in the picker falls through to the getDisplayMedia path and shows the OS picker for that one case. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
byGalax
@@ -1,13 +1,18 @@
|
||||
// Source enumeration for the Discord-style screen-share picker. Returns a
|
||||
// flat list of screens + windows with small PNG thumbnails so the JS
|
||||
// picker can render a grid without the browser's native picker.
|
||||
// Source enumeration for the Discord-style screen-share picker. Split
|
||||
// into two commands on the slow-vs-fast axis:
|
||||
//
|
||||
// - list_screen_sources — metadata only (no thumbnails). Fast; the
|
||||
// picker shows names + placeholders instantly.
|
||||
// - capture_screen_source_thumbnail — one thumbnail at a time, keyed by
|
||||
// the id returned from the list.
|
||||
//
|
||||
// The frontend fans out the thumbnail calls via Promise.all so Tauri's
|
||||
// command thread pool captures them in parallel — wall-clock time ends up
|
||||
// bounded by the *slowest* source rather than the sum of all captures.
|
||||
// The `id` field is emitted in Chromium's internal desktopCapturer format
|
||||
// ("screen:<id>:0" / "window:<hwnd>:0") so the JS side can try to pass it
|
||||
// straight into getUserMedia's `chromeMediaSourceId` constraint. That
|
||||
// bypass may or may not be accepted by WebView2 depending on the version
|
||||
// — if it isn't, the JS falls back to the ordinary getDisplayMedia flow
|
||||
// and at least the user has seen an informed preview first.
|
||||
// straight into getUserMedia's `chromeMediaSourceId` constraint, or use
|
||||
// it as the source key for the native capture pipeline in screen_capture.
|
||||
//
|
||||
// xcap abstracts the platform-specific capture APIs (Windows GDI + DXGI,
|
||||
// macOS CoreGraphics/ScreenCaptureKit, X11) so the code here stays flat.
|
||||
@@ -40,6 +45,39 @@ pub struct ScreenSource {
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
// Fast metadata-only enumeration. No image capture happens here — that's
|
||||
// why it returns in tens of milliseconds instead of the multi-second
|
||||
// wait the single-shot enumerate_screen_sources command had.
|
||||
#[tauri::command]
|
||||
pub fn list_screen_sources() -> Result<Vec<ScreenSource>, String> {
|
||||
let mut out: Vec<ScreenSource> = Vec::new();
|
||||
append_monitor_metadata(&mut out);
|
||||
append_window_metadata(&mut out);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
// One-shot thumbnail capture by source id. Called N times in parallel from
|
||||
// the frontend after the list lands so the total wall-clock is bounded by
|
||||
// the slowest capture rather than the sum.
|
||||
#[tauri::command]
|
||||
pub fn capture_screen_source_thumbnail(source_id: String) -> Result<Option<String>, String> {
|
||||
if let Some(raw) = source_id.strip_prefix("screen:").and_then(strip_zero_suffix) {
|
||||
return Ok(capture_monitor_by_id(raw));
|
||||
}
|
||||
if let Some(raw) = source_id.strip_prefix("window:").and_then(strip_zero_suffix) {
|
||||
return Ok(capture_window_by_id(raw));
|
||||
}
|
||||
Err(format!("unknown source id format: {source_id}"))
|
||||
}
|
||||
|
||||
fn strip_zero_suffix(s: &str) -> Option<&str> {
|
||||
s.strip_suffix(":0")
|
||||
}
|
||||
|
||||
// Legacy one-shot all-in-one enumeration. Kept around so the frontend can
|
||||
// fall back during rollout if the split pair throws; marked dead_code so
|
||||
// the linker doesn't grumble when only the split variant is wired up.
|
||||
#[allow(dead_code)]
|
||||
#[tauri::command]
|
||||
pub fn enumerate_screen_sources() -> Result<Vec<ScreenSource>, String> {
|
||||
let mut out: Vec<ScreenSource> = Vec::new();
|
||||
@@ -52,6 +90,27 @@ pub fn enumerate_screen_sources() -> Result<Vec<ScreenSource>, String> {
|
||||
// Monitors
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn append_monitor_metadata(out: &mut Vec<ScreenSource>) {
|
||||
let monitors = match xcap::Monitor::all() {
|
||||
Ok(m) => m,
|
||||
Err(err) => {
|
||||
eprintln!("xcap Monitor::all failed: {err}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
for (idx, m) in monitors.iter().enumerate() {
|
||||
out.push(ScreenSource {
|
||||
id: format!("screen:{}:0", m.id()),
|
||||
name: monitor_label(m, idx),
|
||||
kind: "screen",
|
||||
thumbnail_png: None,
|
||||
width: m.width(),
|
||||
height: m.height(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn append_monitors(out: &mut Vec<ScreenSource>) {
|
||||
let monitors = match xcap::Monitor::all() {
|
||||
Ok(m) => m,
|
||||
@@ -61,25 +120,24 @@ fn append_monitors(out: &mut Vec<ScreenSource>) {
|
||||
}
|
||||
};
|
||||
for (idx, m) in monitors.iter().enumerate() {
|
||||
let name = monitor_label(m, idx);
|
||||
let width = m.width();
|
||||
let height = m.height();
|
||||
// xcap exposes an OS-level monitor id; pass it through as the
|
||||
// middle component so the JS side can correlate repeat enumerations.
|
||||
let raw_id = m.id();
|
||||
let id = format!("screen:{raw_id}:0");
|
||||
let thumb = capture_monitor_thumbnail(m);
|
||||
out.push(ScreenSource {
|
||||
id,
|
||||
name,
|
||||
id: format!("screen:{}:0", m.id()),
|
||||
name: monitor_label(m, idx),
|
||||
kind: "screen",
|
||||
thumbnail_png: thumb,
|
||||
width,
|
||||
height,
|
||||
thumbnail_png: capture_monitor_thumbnail(m),
|
||||
width: m.width(),
|
||||
height: m.height(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn capture_monitor_by_id(raw: &str) -> Option<String> {
|
||||
let raw_id: u32 = raw.parse().ok()?;
|
||||
let monitors = xcap::Monitor::all().ok()?;
|
||||
let target = monitors.into_iter().find(|m| m.id() == raw_id)?;
|
||||
capture_monitor_thumbnail(&target)
|
||||
}
|
||||
|
||||
fn monitor_label(m: &xcap::Monitor, idx: usize) -> String {
|
||||
let name = m.name();
|
||||
if name.is_empty() {
|
||||
@@ -98,6 +156,34 @@ fn capture_monitor_thumbnail(m: &xcap::Monitor) -> Option<String> {
|
||||
// Windows
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn append_window_metadata(out: &mut Vec<ScreenSource>) {
|
||||
let windows = match xcap::Window::all() {
|
||||
Ok(w) => w,
|
||||
Err(err) => {
|
||||
eprintln!("xcap Window::all failed: {err}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
for w in windows.iter() {
|
||||
if !is_shareable_window(w) {
|
||||
continue;
|
||||
}
|
||||
let title = w.title();
|
||||
if title.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
out.push(ScreenSource {
|
||||
id: format!("window:{}:0", w.id()),
|
||||
name: title.to_string(),
|
||||
kind: "window",
|
||||
thumbnail_png: None,
|
||||
width: w.width(),
|
||||
height: w.height(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn append_windows(out: &mut Vec<ScreenSource>) {
|
||||
let windows = match xcap::Window::all() {
|
||||
Ok(w) => w,
|
||||
@@ -114,22 +200,24 @@ fn append_windows(out: &mut Vec<ScreenSource>) {
|
||||
if title.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let width = w.width();
|
||||
let height = w.height();
|
||||
let raw_id = w.id();
|
||||
let id = format!("window:{raw_id}:0");
|
||||
let thumb = capture_window_thumbnail(w);
|
||||
out.push(ScreenSource {
|
||||
id,
|
||||
id: format!("window:{}:0", w.id()),
|
||||
name: title.to_string(),
|
||||
kind: "window",
|
||||
thumbnail_png: thumb,
|
||||
width,
|
||||
height,
|
||||
thumbnail_png: capture_window_thumbnail(w),
|
||||
width: w.width(),
|
||||
height: w.height(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn capture_window_by_id(raw: &str) -> Option<String> {
|
||||
let raw_id: u32 = raw.parse().ok()?;
|
||||
let windows = xcap::Window::all().ok()?;
|
||||
let target = windows.into_iter().find(|w| w.id() == raw_id)?;
|
||||
capture_window_thumbnail(&target)
|
||||
}
|
||||
|
||||
fn is_shareable_window(w: &xcap::Window) -> bool {
|
||||
if w.is_minimized() {
|
||||
return false;
|
||||
|
||||
Reference in New Issue
Block a user