feat(call): native screen-capture pipeline + faster picker

Picker speed (Phase 1+2):
- screen_sources.rs split into list_screen_sources (metadata only,
  returns in ~10ms) + capture_screen_source_thumbnail (single source,
  by id). ScreenSourcePicker now shows names + placeholders instantly
  and streams thumbnails in as each capture lands. Total wall-clock
  is bounded by the slowest source instead of the serial sum.
- enumerate_screen_sources kept as a dead_code fallback so any
  rollout regression can switch the frontend back without code loss.

Native capture (Phase 3):
- New src-tauri/src/screen_capture.rs. start_screen_capture spawns a
  Rust thread per share that grabs frames via xcap, downscales to the
  user's quality preset, JPEG-encodes at Q72, and streams each frame
  through a Tauri Channel<FramePayload>. stop_screen_capture signals
  the stop flag and joins the worker.
- Worker re-resolves the xcap handle inside the thread because
  xcap::Window holds a !Send HWND — passing the source id string
  across the thread boundary sidesteps that.
- New lib/screenCapture.ts: decodes each frame into an ImageBitmap,
  draws to an offscreen canvas, exposes canvas.captureStream() as the
  MediaStream LiveKit publishes. Latest-wins frame queue drops stale
  frames when the JS side falls behind the Rust producer. 3s first-
  frame timeout so a silently-failing source (locked screen, DRM
  window) surfaces as a clean NativeCaptureUnavailable and we fall
  back to getDisplayMedia.
- CallContext.startScreenShare takes the native path first when the
  picker provided a sourceId and system audio wasn't requested. The
  old chromeMediaSourceId attempt and final setScreenShareEnabled
  fallback stay in place for the audio case + non-Tauri runtimes.
- stopScreenShare kills the native handle first, then unpublishes any
  manually-published ScreenShare/ScreenShareAudio tracks, then falls
  back to setScreenShareEnabled(false). disconnectRoom also stops
  the handle so we don't leak Rust threads across calls.

Scope note: native path is video-only. System-audio capture needs
WASAPI-loopback (Windows) or ScreenCaptureKit-audio (macOS); until
those are wired, requesting audio in the picker falls through to
the getDisplayMedia path and shows the OS picker for that one case.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

apps/desktop/src/components/ScreenSourcePicker.tsx

@@ -10,7 +10,8 @@ import {
   updateScreenShareSettings,
 } from '../lib/screenShareSettings';
 import {
-  enumerateScreenSources,
+  captureScreenSourceThumbnail,
+  listScreenSources,
   type ScreenSource,
   thumbnailDataUrl,
 } from '../lib/screenSources';
@@ -47,9 +48,11 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
   const [busy, setBusy] = useState(false);
   const [error, setError] = useState<string | null>(null);
 
-  // Re-enumerate every time the picker opens so closed windows + new ones
-  // stay accurate. A previous stale list would surface sources the user
-  // can't actually share anymore.
+  // Two-phase load: (1) fast list returns names + placeholders so the grid
+  // paints instantly, (2) fire one thumbnail capture per source in
+  // parallel. Thumbnails fill in as each capture completes — Tauri's
+  // command thread pool runs them concurrently so wall-clock is bounded
+  // by the slowest source, not the sum.
   useEffect(() => {
     if (!open) {
       setSources(null);
@@ -59,9 +62,27 @@ export function ScreenSourcePicker({ open, onClose, onStart }: Props) {
     }
     let cancelled = false;
     void (async () => {
-      const list = await enumerateScreenSources();
+      const list = await listScreenSources();
       if (cancelled) return;
       setSources(list);
+      // Fan out thumbnail captures. No Promise.all — we want each result
+      // to render as it lands, not wait for the full batch. The id-based
+      // setState patch means the slowest source can still be in flight
+      // while the user already picked one of the fast ones.
+      for (const src of list) {
+        void (async () => {
+          const png = await captureScreenSourceThumbnail(src.id);
+          if (cancelled || png === null) return;
+          setSources((prev) => {
+            if (!prev) return prev;
+            const idx = prev.findIndex((s) => s.id === src.id);
+            if (idx === -1) return prev;
+            const next = prev.slice();
+            next[idx] = { ...prev[idx]!, thumbnailPng: png };
+            return next;
+          });
+        })();
+      }
     })();
     return () => {
       cancelled = true;

apps/desktop/src/context/CallContext.tsx

@@ -75,6 +75,11 @@ import {
   setAllPipelinesSinkId,
   setPipelineGain,
 } from '../lib/remoteAudioPipelines';
+import {
+  type NativeCaptureHandle,
+  NativeCaptureUnavailable,
+  startNativeCapture,
+} from '../lib/screenCapture';
 import {
   clearScreenShareVolumes,
   getScreenShareVolume,
@@ -295,6 +300,11 @@ export function CallProvider({ children }: { children: ReactNode }) {
   // to `connected` after a few seconds so the UI doesn't hang in "Verbinde…"
   // indefinitely. The solo-timeout will then cleanly close if nobody arrives.
   const joinFallbackTimerRef = useRef<number | null>(null);
+  // Active native screen-capture handle (Rust side). Set when
+  // startScreenShare takes the xcap path; cleared on stopScreenShare or
+  // on the canvas track's 'ended' event. Not kept in React state because
+  // it never feeds into a render.
+  const nativeCaptureRef = useRef<NativeCaptureHandle | null>(null);
   const roomRef = useRef<Room | null>(null);
   // Web Audio graph that mixes live mic + soundboard sources into a single
   // published track. Created per call in joinRoom, destroyed in disconnectRoom.
@@ -469,6 +479,13 @@ export function CallProvider({ children }: { children: ReactNode }) {
         /* ignore */
       }
     }
+    // Stop any lingering native screen-capture thread so we don't leak
+    // Rust threads when the call ends mid-share.
+    if (nativeCaptureRef.current) {
+      const h = nativeCaptureRef.current;
+      nativeCaptureRef.current = null;
+      await h.stop().catch(() => undefined);
+    }
     roomRef.current = null;
     setRoom(null);
     setRemoteParticipants([]);
@@ -1172,6 +1189,69 @@ export function CallProvider({ children }: { children: ReactNode }) {
       const fps = framerateOverride ?? ssParams.framerate;
       const sourceId = overrides?.sourceId ?? null;
 
+      // Native capture path — tried first when the user came in via our
+      // custom picker. xcap on the Rust side grabs frames, streams them as
+      // JPEG over a Tauri channel, and we draw them onto a canvas whose
+      // captureStream() becomes the MediaStream LiveKit publishes. This
+      // completely skips the OS picker. Video-only (no system audio yet);
+      // if the user asked for audio we fall through to the legacy paths
+      // below so audio still works via getDisplayMedia.
+      if (sourceId && !settings.includeSystemAudio) {
+        try {
+          const { Track: LkTrack } = await import('livekit-client');
+          const maxWidth = ssParams.dims?.width ?? 1920;
+          const maxHeight = ssParams.dims?.height ?? 1080;
+          const handle = await startNativeCapture({
+            sourceId,
+            maxWidth,
+            maxHeight,
+            fps,
+          });
+          nativeCaptureRef.current = handle;
+          const videoMst = handle.stream.getVideoTracks()[0];
+          if (!videoMst) {
+            await handle.stop();
+            nativeCaptureRef.current = null;
+            throw new NativeCaptureUnavailable('canvas stream produced no video track');
+          }
+          const videoPub = await lp.publishTrack(videoMst, {
+            source: LkTrack.Source.ScreenShare,
+            videoCodec: 'vp9',
+          });
+          // Canvas stream 'ended' fires on handle.stop() (we track.stop()
+          // each track) — chain unpublish + native teardown so one ended
+          // event cleans everything up regardless of who triggered it.
+          videoMst.addEventListener('ended', () => {
+            void (async () => {
+              try {
+                if (videoPub.track) await lp.unpublishTrack(videoPub.track);
+              } catch {
+                /* already unpublished */
+              }
+              const active = nativeCaptureRef.current;
+              if (active && active.captureId === handle.captureId) {
+                nativeCaptureRef.current = null;
+                await active.stop().catch(() => undefined);
+              }
+              setIsScreenSharing(false);
+            })();
+          });
+          setIsScreenSharing(true);
+          return;
+        } catch (err: unknown) {
+          // Native path unavailable (non-Tauri runtime, source vanished,
+          // first-frame timeout). Clean up any partial handle and fall
+          // through to the getUserMedia / getDisplayMedia paths.
+          if (nativeCaptureRef.current) {
+            await nativeCaptureRef.current.stop().catch(() => undefined);
+            nativeCaptureRef.current = null;
+          }
+          if (!(err instanceof NativeCaptureUnavailable)) {
+            console.warn('native screen-capture failed, falling back', err);
+          }
+        }
+      }
+
       // Direct-publish path when our custom picker supplied a Chromium-
       // format source id. Bypasses the OS picker so the user shares exactly
       // the window/monitor they clicked in the grid. getUserMedia with the
@@ -1299,16 +1379,42 @@ export function CallProvider({ children }: { children: ReactNode }) {
   );
 
   const stopScreenShare = useCallback(async () => {
+    // Native path first — stopping the handle kills the canvas track,
+    // which fires 'ended' on the MediaStreamTrack, which the start-handler
+    // already listens to for unpublishing and flipping isScreenSharing.
+    if (nativeCaptureRef.current) {
+      const h = nativeCaptureRef.current;
+      nativeCaptureRef.current = null;
+      try {
+        await h.stop();
+      } catch (err: unknown) {
+        console.warn('native capture stop failed', err);
+      }
+    }
     const r = roomRef.current;
     if (!r) return;
     const lp = r.localParticipant;
-    if (!lp.isScreenShareEnabled) return;
-    try {
-      await lp.setScreenShareEnabled(false);
-      setIsScreenSharing(false);
-    } catch (err: unknown) {
-      console.error('stopScreenShare failed', err);
+    // Unpublish any manually-published ScreenShare/ScreenShareAudio tracks
+    // (from the chromeMediaSourceId fallback path). setScreenShareEnabled
+    // only manages LK's own internally-captured tracks.
+    const toUnpublish: import('livekit-client').LocalTrackPublication[] = [];
+    for (const pub of lp.videoTrackPublications.values()) {
+      if (pub.source === Track.Source.ScreenShare && pub.track) toUnpublish.push(pub);
     }
+    for (const pub of lp.audioTrackPublications.values()) {
+      if (pub.source === Track.Source.ScreenShareAudio && pub.track) toUnpublish.push(pub);
+    }
+    for (const pub of toUnpublish) {
+      if (pub.track) await lp.unpublishTrack(pub.track).catch(() => undefined);
+    }
+    if (lp.isScreenShareEnabled) {
+      try {
+        await lp.setScreenShareEnabled(false);
+      } catch (err: unknown) {
+        console.error('stopScreenShare failed', err);
+      }
+    }
+    setIsScreenSharing(false);
   }, []);
 
   // Legacy toggle kept for convenience elsewhere — opens/closes with the

apps/desktop/src/lib/screenCapture.ts

@@ -0,0 +1,184 @@
+// Frontend side of the native screen-capture pipeline. Starts a Rust-side
+// capture thread via `start_screen_capture` and streams JPEG frames back
+// through a Tauri Channel. Each frame is decoded into an ImageBitmap,
+// drawn onto an offscreen canvas, and the canvas' captureStream() is
+// returned as a MediaStream that LiveKit can publishTrack() directly —
+// no OS/browser screen picker is involved.
+//
+// Video-only: system audio would require WASAPI / ScreenCaptureKit hooks
+// that xcap doesn't provide. Callers that request shared audio must
+// either fall back to the browser picker or accept video-without-audio.
+
+import { isTauriRuntime } from './globalShortcut';
+
+export interface NativeCaptureHandle {
+  /** Rust-side capture id. Pass to `stopNativeCapture` to tear down. */
+  captureId: number;
+  /** MediaStream fed by a canvas that's drawing each incoming frame. */
+  stream: MediaStream;
+  /** Cleanup — stops the Rust thread, closes channels, revokes the canvas
+   *  stream. Idempotent. */
+  stop: () => Promise<void>;
+}
+
+interface FramePayload {
+  captureId: number;
+  width: number;
+  height: number;
+  jpegBase64: string;
+}
+
+/** Returned when the runtime can't support native capture (no Tauri, no
+ *  WebAudio, source vanished between enumeration and start, etc.). The
+ *  caller is expected to fall back to the browser's getDisplayMedia path. */
+export class NativeCaptureUnavailable extends Error {
+  constructor(reason: string) {
+    super('native capture unavailable: ' + reason);
+    this.name = 'NativeCaptureUnavailable';
+  }
+}
+
+export async function startNativeCapture(opts: {
+  sourceId: string;
+  maxWidth: number;
+  maxHeight: number;
+  fps: number;
+}): Promise<NativeCaptureHandle> {
+  if (!isTauriRuntime()) {
+    throw new NativeCaptureUnavailable('not a tauri runtime');
+  }
+
+  const canvas = document.createElement('canvas');
+  canvas.width = opts.maxWidth;
+  canvas.height = opts.maxHeight;
+  const ctx = canvas.getContext('2d');
+  if (!ctx) {
+    throw new NativeCaptureUnavailable('canvas 2d context unavailable');
+  }
+
+  // Track whether we got the first frame so we can fail fast if Rust
+  // reports "found" but then produces no output (e.g. screen was locked).
+  let firstFrameResolved = false;
+  let firstFrameResolve!: () => void;
+  let firstFrameReject!: (err: Error) => void;
+  const firstFramePromise = new Promise<void>((resolve, reject) => {
+    firstFrameResolve = resolve;
+    firstFrameReject = reject;
+  });
+
+  const { Channel, invoke } = await import('@tauri-apps/api/core');
+  const channel = new Channel<FramePayload>();
+
+  // Latest-wins frame queue: if the JS side falls behind the Rust producer,
+  // we drop stale frames rather than queue them. Keeps memory flat and
+  // latency sensible for live screenshare.
+  let pendingFrame: FramePayload | null = null;
+  let decoding = false;
+
+  const drainQueue = async () => {
+    if (decoding) return;
+    decoding = true;
+    try {
+      while (pendingFrame) {
+        const frame = pendingFrame;
+        pendingFrame = null;
+        const bytes = base64ToBytes(frame.jpegBase64);
+        // Uint8Array's buffer type is `ArrayBufferLike` (could be a
+        // SharedArrayBuffer in theory); Blob wants plain ArrayBuffer.
+        // Pass the underlying buffer explicitly so the type narrows.
+        const blob = new Blob([bytes.buffer as ArrayBuffer], { type: 'image/jpeg' });
+        let bitmap: ImageBitmap;
+        try {
+          bitmap = await createImageBitmap(blob);
+        } catch (err: unknown) {
+          console.warn('createImageBitmap failed', err);
+          continue;
+        }
+        if (canvas.width !== bitmap.width || canvas.height !== bitmap.height) {
+          canvas.width = bitmap.width;
+          canvas.height = bitmap.height;
+        }
+        ctx.drawImage(bitmap, 0, 0);
+        bitmap.close();
+        if (!firstFrameResolved) {
+          firstFrameResolved = true;
+          firstFrameResolve();
+        }
+      }
+    } finally {
+      decoding = false;
+    }
+  };
+
+  channel.onmessage = (frame: FramePayload) => {
+    pendingFrame = frame;
+    void drainQueue();
+  };
+
+  let captureId: number;
+  try {
+    captureId = await invoke<number>('start_screen_capture', {
+      sourceId: opts.sourceId,
+      maxWidth: opts.maxWidth,
+      maxHeight: opts.maxHeight,
+      fps: opts.fps,
+      channel,
+    });
+  } catch (err: unknown) {
+    throw new NativeCaptureUnavailable(
+      err instanceof Error ? err.message : String(err),
+    );
+  }
+
+  // Bound the wait: the capture thread may fail silently on some sources
+  // (locked screens, protected windows). Fall back to getDisplayMedia in
+  // that case rather than hang the user.
+  const firstFrameTimeout = window.setTimeout(() => {
+    firstFrameReject(new Error('first frame timed out (3s)'));
+  }, 3000);
+  try {
+    await firstFramePromise;
+  } catch (err: unknown) {
+    window.clearTimeout(firstFrameTimeout);
+    try {
+      await invoke('stop_screen_capture', { captureId });
+    } catch {
+      /* ignore */
+    }
+    throw new NativeCaptureUnavailable(
+      err instanceof Error ? err.message : String(err),
+    );
+  }
+  window.clearTimeout(firstFrameTimeout);
+
+  const stream = canvas.captureStream(opts.fps);
+
+  let stopped = false;
+  const stop = async (): Promise<void> => {
+    if (stopped) return;
+    stopped = true;
+    try {
+      await invoke('stop_screen_capture', { captureId });
+    } catch (err: unknown) {
+      console.warn('stop_screen_capture failed', err);
+    }
+    for (const track of stream.getTracks()) {
+      try {
+        track.stop();
+      } catch {
+        /* already stopped */
+      }
+    }
+  };
+
+  return { captureId, stream, stop };
+}
+
+function base64ToBytes(b64: string): Uint8Array {
+  const bin = atob(b64);
+  const bytes = new Uint8Array(bin.length);
+  for (let i = 0; i < bin.length; i++) {
+    bytes[i] = bin.charCodeAt(i);
+  }
+  return bytes;
+}

apps/desktop/src/lib/screenSources.ts

@@ -18,6 +18,44 @@ export interface ScreenSource {
   height: number;
 }
 
+// Fast, metadata-only list. The picker uses this first so names show up
+// immediately; thumbnails stream in via captureScreenSourceThumbnail below.
+export async function listScreenSources(): Promise<ScreenSource[]> {
+  if (!isTauriRuntime()) return [];
+  try {
+    const { invoke } = await import('@tauri-apps/api/core');
+    const raw = await invoke<ScreenSource[]>('list_screen_sources');
+    return raw ?? [];
+  } catch (err: unknown) {
+    console.warn('list_screen_sources failed', err);
+    return [];
+  }
+}
+
+// Single-source thumbnail capture. Called N times in parallel from the
+// picker so Tauri's command thread pool runs captures concurrently — total
+// wall-clock time becomes bounded by the slowest source, not the sum.
+// Returns the base64 PNG or null when the source disappeared / capture
+// permission was denied.
+export async function captureScreenSourceThumbnail(
+  sourceId: string,
+): Promise<string | null> {
+  if (!isTauriRuntime()) return null;
+  try {
+    const { invoke } = await import('@tauri-apps/api/core');
+    const result = await invoke<string | null>('capture_screen_source_thumbnail', {
+      sourceId,
+    });
+    return result ?? null;
+  } catch (err: unknown) {
+    console.warn('capture_screen_source_thumbnail failed', { sourceId, err });
+    return null;
+  }
+}
+
+// Legacy single-shot variant. Captures everything serially on the Rust side
+// before returning. Prefer listScreenSources + captureScreenSourceThumbnail
+// for user-facing flows — they feel 5–10× more responsive in practice.
 export async function enumerateScreenSources(): Promise<ScreenSource[]> {
   if (!isTauriRuntime()) return [];
   try {