feat(call): Discord-style screen-source picker with thumbnails

Rust side:
- New src-tauri/src/screen_sources.rs with an enumerate_screen_sources
  command. Uses the xcap crate for cross-platform screen + window
  enumeration and capture; PNG thumbnails are letterbox-scaled to fit
  320x180 and returned as base64.
- Source ids emit Chromium's internal desktopCapturer format
  ("screen:<id>:0", "window:<hwnd>:0") so JS can try passing them
  straight into chromeMediaSourceId.
- Registered in both invoke_handler branches in lib.rs.

Frontend:
- New lib/screenSources.ts — Tauri command wrapper + thumbnailDataUrl
  helper for the picker UI.
- New components/ScreenSourcePicker.tsx — Discord-style grid: sources
  grouped under "Bildschirme" / "Fenster", large thumbnail cards with
  selection state, quality preset + system-audio toggle in the footer.
  "Teilen" button is enabled either way; without a selection it says
  "Ohne Auswahl weiter" and falls through to the OS picker.
- Replaces the old form-style ScreenShareDialog entirely (removed).

CallContext wiring:
- startScreenShare now accepts an optional sourceId. When set, it
  captures that exact source via getUserMedia's legacy
  chromeMediaSourceId constraint and publishes the resulting tracks
  manually (video as ScreenShare, audio as ScreenShareAudio). Falls
  back to setScreenShareEnabled if WebView2 rejects the constraint,
  so users always get a working share even if the direct path fails.
- Track 'ended' listeners unpublish the pub when the OS revokes
  capture (close of shared window, OS "stop sharing" banner).

InCallPanel:
- Left-click on the share button now opens the picker instead of
  starting with last-saved settings; right-click opens it too. The
  picker itself is the 1-click UX.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
byGalax
2026-04-22 20:55:52 +02:00
parent 331b1298f8
commit b44a785d20
9 changed files with 1472 additions and 284 deletions
+3
View File
@@ -1,4 +1,5 @@
mod crypto;
mod screen_sources;
#[cfg(feature = "rust-livekit")]
mod livekit_bridge;
@@ -91,6 +92,7 @@ pub fn run() {
crypto::crypto_box_seal,
crypto::crypto_box_seal_open,
crypto::crypto_pwhash,
screen_sources::enumerate_screen_sources,
])
.plugin(tauri_plugin_notification::init());
@@ -107,6 +109,7 @@ pub fn run() {
crypto::crypto_box_seal,
crypto::crypto_box_seal_open,
crypto::crypto_pwhash,
screen_sources::enumerate_screen_sources,
livekit_bridge::livekit_connect,
livekit_bridge::livekit_disconnect,
livekit_bridge::livekit_send_data,
@@ -0,0 +1,191 @@
// Source enumeration for the Discord-style screen-share picker. Returns a
// flat list of screens + windows with small PNG thumbnails so the JS
// picker can render a grid without the browser's native picker.
//
// The `id` field is emitted in Chromium's internal desktopCapturer format
// ("screen:<id>:0" / "window:<hwnd>:0") so the JS side can try to pass it
// straight into getUserMedia's `chromeMediaSourceId` constraint. That
// bypass may or may not be accepted by WebView2 depending on the version
// — if it isn't, the JS falls back to the ordinary getDisplayMedia flow
// and at least the user has seen an informed preview first.
//
// xcap abstracts the platform-specific capture APIs (Windows GDI + DXGI,
// macOS CoreGraphics/ScreenCaptureKit, X11) so the code here stays flat.
// Thumbnails are captured at native resolution, then letterbox-downscaled
// to fit a 320×180 box to keep the base64 payload small.
use base64::Engine;
use image::{ImageBuffer, Rgba};
use serde::Serialize;
const THUMB_MAX_W: u32 = 320;
const THUMB_MAX_H: u32 = 180;
#[derive(Serialize, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScreenSource {
/// Chromium-format source id — stable within one enumeration call.
pub id: String,
/// Human-readable label for the picker (monitor name or window title).
pub name: String,
/// Discriminator for the grid grouping.
pub kind: &'static str,
/// Base64-encoded PNG, no data-URL prefix. None when the capture
/// fails (minimised window, permission-denied surface, transient
/// race). The UI renders a name-only card in that case.
pub thumbnail_png: Option<String>,
/// Native width of the full-res source — mostly informational, used
/// by the UI for aspect-ratio styling of the card.
pub width: u32,
pub height: u32,
}
#[tauri::command]
pub fn enumerate_screen_sources() -> Result<Vec<ScreenSource>, String> {
let mut out: Vec<ScreenSource> = Vec::new();
append_monitors(&mut out);
append_windows(&mut out);
Ok(out)
}
// ---------------------------------------------------------------------------
// Monitors
// ---------------------------------------------------------------------------
fn append_monitors(out: &mut Vec<ScreenSource>) {
let monitors = match xcap::Monitor::all() {
Ok(m) => m,
Err(err) => {
eprintln!("xcap Monitor::all failed: {err}");
return;
}
};
for (idx, m) in monitors.iter().enumerate() {
let name = monitor_label(m, idx);
let width = m.width();
let height = m.height();
// xcap exposes an OS-level monitor id; pass it through as the
// middle component so the JS side can correlate repeat enumerations.
let raw_id = m.id();
let id = format!("screen:{raw_id}:0");
let thumb = capture_monitor_thumbnail(m);
out.push(ScreenSource {
id,
name,
kind: "screen",
thumbnail_png: thumb,
width,
height,
});
}
}
fn monitor_label(m: &xcap::Monitor, idx: usize) -> String {
let name = m.name();
if name.is_empty() {
format!("Bildschirm {}", idx + 1)
} else {
name.to_string()
}
}
fn capture_monitor_thumbnail(m: &xcap::Monitor) -> Option<String> {
let image = m.capture_image().ok()?;
encode_scaled_png(image)
}
// ---------------------------------------------------------------------------
// Windows
// ---------------------------------------------------------------------------
fn append_windows(out: &mut Vec<ScreenSource>) {
let windows = match xcap::Window::all() {
Ok(w) => w,
Err(err) => {
eprintln!("xcap Window::all failed: {err}");
return;
}
};
for w in windows.iter() {
if !is_shareable_window(w) {
continue;
}
let title = w.title();
if title.trim().is_empty() {
continue;
}
let width = w.width();
let height = w.height();
let raw_id = w.id();
let id = format!("window:{raw_id}:0");
let thumb = capture_window_thumbnail(w);
out.push(ScreenSource {
id,
name: title.to_string(),
kind: "window",
thumbnail_png: thumb,
width,
height,
});
}
}
fn is_shareable_window(w: &xcap::Window) -> bool {
if w.is_minimized() {
return false;
}
let width = w.width();
let height = w.height();
// Tooltips, invisible tray-helpers, etc. sit at or near zero size —
// they'd clutter the picker grid and usually can't be captured anyway.
if width < 80 || height < 60 {
return false;
}
true
}
fn capture_window_thumbnail(w: &xcap::Window) -> Option<String> {
let image = w.capture_image().ok()?;
encode_scaled_png(image)
}
// ---------------------------------------------------------------------------
// Scaling + encoding
// ---------------------------------------------------------------------------
// Letterbox-shrink the captured image so the long edge is at most
// THUMB_MAX_W / THUMB_MAX_H. Keeps aspect ratio, skips upscaling entirely
// (tiny windows stay their captured size). Returns a base64 PNG string or
// None if encoding fails.
fn encode_scaled_png(src: ImageBuffer<Rgba<u8>, Vec<u8>>) -> Option<String> {
let (w, h) = src.dimensions();
if w == 0 || h == 0 {
return None;
}
let scale = (THUMB_MAX_W as f32 / w as f32)
.min(THUMB_MAX_H as f32 / h as f32)
.min(1.0);
let scaled = if scale < 1.0 {
let new_w = ((w as f32) * scale).round().max(1.0) as u32;
let new_h = ((h as f32) * scale).round().max(1.0) as u32;
image::imageops::resize(&src, new_w, new_h, image::imageops::FilterType::Triangle)
} else {
src
};
let mut buf: Vec<u8> = Vec::new();
let encoder = image::codecs::png::PngEncoder::new_with_quality(
&mut buf,
image::codecs::png::CompressionType::Fast,
image::codecs::png::FilterType::Adaptive,
);
use image::ImageEncoder;
encoder
.write_image(
scaled.as_raw(),
scaled.width(),
scaled.height(),
image::ExtendedColorType::Rgba8,
)
.ok()?;
Some(base64::engine::general_purpose::STANDARD.encode(&buf))
}