feat(call): native screen-capture pipeline + faster picker

Picker speed (Phase 1+2): - screen_sources.rs split into list_screen_sources (metadata only, returns in ~10ms) + capture_screen_source_thumbnail (single source, by id). ScreenSourcePicker now shows names + placeholders instantly and streams thumbnails in as each capture lands. Total wall-clock is bounded by the slowest source instead of the serial sum. - enumerate_screen_sources kept as a dead_code fallback so any rollout regression can switch the frontend back without code loss. Native capture (Phase 3): - New src-tauri/src/screen_capture.rs. start_screen_capture spawns a Rust thread per share that grabs frames via xcap, downscales to the user's quality preset, JPEG-encodes at Q72, and streams each frame through a Tauri Channel<FramePayload>. stop_screen_capture signals the stop flag and joins the worker. - Worker re-resolves the xcap handle inside the thread because xcap::Window holds a !Send HWND — passing the source id string across the thread boundary sidesteps that. - New lib/screenCapture.ts: decodes each frame into an ImageBitmap, draws to an offscreen canvas, exposes canvas.captureStream() as the MediaStream LiveKit publishes. Latest-wins frame queue drops stale frames when the JS side falls behind the Rust producer. 3s first- frame timeout so a silently-failing source (locked screen, DRM window) surfaces as a clean NativeCaptureUnavailable and we fall back to getDisplayMedia. - CallContext.startScreenShare takes the native path first when the picker provided a sourceId and system audio wasn't requested. The old chromeMediaSourceId attempt and final setScreenShareEnabled fallback stay in place for the audio case + non-Tauri runtimes. - stopScreenShare kills the native handle first, then unpublishes any manually-published ScreenShare/ScreenShareAudio tracks, then falls back to setScreenShareEnabled(false). disconnectRoom also stops the handle so we don't leak Rust threads across calls. Scope note: native path is video-only. System-audio capture needs WASAPI-loopback (Windows) or ScreenCaptureKit-audio (macOS); until those are wired, requesting audio in the picker falls through to the getDisplayMedia path and shows the OS picker for that one case. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 21:42:52 +02:00
parent 8f9b823d69
commit c3e0c47d32
7 changed files with 752 additions and 40 deletions
@@ -0,0 +1,184 @@
+// Frontend side of the native screen-capture pipeline. Starts a Rust-side
+// capture thread via `start_screen_capture` and streams JPEG frames back
+// through a Tauri Channel. Each frame is decoded into an ImageBitmap,
+// drawn onto an offscreen canvas, and the canvas' captureStream() is
+// returned as a MediaStream that LiveKit can publishTrack() directly —
+// no OS/browser screen picker is involved.
+//
+// Video-only: system audio would require WASAPI / ScreenCaptureKit hooks
+// that xcap doesn't provide. Callers that request shared audio must
+// either fall back to the browser picker or accept video-without-audio.
+
+import { isTauriRuntime } from './globalShortcut';
+
+export interface NativeCaptureHandle {
+  /** Rust-side capture id. Pass to `stopNativeCapture` to tear down. */
+  captureId: number;
+  /** MediaStream fed by a canvas that's drawing each incoming frame. */
+  stream: MediaStream;
+  /** Cleanup — stops the Rust thread, closes channels, revokes the canvas
+   *  stream. Idempotent. */
+  stop: () => Promise<void>;
+}
+
+interface FramePayload {
+  captureId: number;
+  width: number;
+  height: number;
+  jpegBase64: string;
+}
+
+/** Returned when the runtime can't support native capture (no Tauri, no
+ *  WebAudio, source vanished between enumeration and start, etc.). The
+ *  caller is expected to fall back to the browser's getDisplayMedia path. */
+export class NativeCaptureUnavailable extends Error {
+  constructor(reason: string) {
+    super('native capture unavailable: ' + reason);
+    this.name = 'NativeCaptureUnavailable';
+  }
+}
+
+export async function startNativeCapture(opts: {
+  sourceId: string;
+  maxWidth: number;
+  maxHeight: number;
+  fps: number;
+}): Promise<NativeCaptureHandle> {
+  if (!isTauriRuntime()) {
+    throw new NativeCaptureUnavailable('not a tauri runtime');
+  }
+
+  const canvas = document.createElement('canvas');
+  canvas.width = opts.maxWidth;
+  canvas.height = opts.maxHeight;
+  const ctx = canvas.getContext('2d');
+  if (!ctx) {
+    throw new NativeCaptureUnavailable('canvas 2d context unavailable');
+  }
+
+  // Track whether we got the first frame so we can fail fast if Rust
+  // reports "found" but then produces no output (e.g. screen was locked).
+  let firstFrameResolved = false;
+  let firstFrameResolve!: () => void;
+  let firstFrameReject!: (err: Error) => void;
+  const firstFramePromise = new Promise<void>((resolve, reject) => {
+    firstFrameResolve = resolve;
+    firstFrameReject = reject;
+  });
+
+  const { Channel, invoke } = await import('@tauri-apps/api/core');
+  const channel = new Channel<FramePayload>();
+
+  // Latest-wins frame queue: if the JS side falls behind the Rust producer,
+  // we drop stale frames rather than queue them. Keeps memory flat and
+  // latency sensible for live screenshare.
+  let pendingFrame: FramePayload | null = null;
+  let decoding = false;
+
+  const drainQueue = async () => {
+    if (decoding) return;
+    decoding = true;
+    try {
+      while (pendingFrame) {
+        const frame = pendingFrame;
+        pendingFrame = null;
+        const bytes = base64ToBytes(frame.jpegBase64);
+        // Uint8Array's buffer type is `ArrayBufferLike` (could be a
+        // SharedArrayBuffer in theory); Blob wants plain ArrayBuffer.
+        // Pass the underlying buffer explicitly so the type narrows.
+        const blob = new Blob([bytes.buffer as ArrayBuffer], { type: 'image/jpeg' });
+        let bitmap: ImageBitmap;
+        try {
+          bitmap = await createImageBitmap(blob);
+        } catch (err: unknown) {
+          console.warn('createImageBitmap failed', err);
+          continue;
+        }
+        if (canvas.width !== bitmap.width || canvas.height !== bitmap.height) {
+          canvas.width = bitmap.width;
+          canvas.height = bitmap.height;
+        }
+        ctx.drawImage(bitmap, 0, 0);
+        bitmap.close();
+        if (!firstFrameResolved) {
+          firstFrameResolved = true;
+          firstFrameResolve();
+        }
+      }
+    } finally {
+      decoding = false;
+    }
+  };
+
+  channel.onmessage = (frame: FramePayload) => {
+    pendingFrame = frame;
+    void drainQueue();
+  };
+
+  let captureId: number;
+  try {
+    captureId = await invoke<number>('start_screen_capture', {
+      sourceId: opts.sourceId,
+      maxWidth: opts.maxWidth,
+      maxHeight: opts.maxHeight,
+      fps: opts.fps,
+      channel,
+    });
+  } catch (err: unknown) {
+    throw new NativeCaptureUnavailable(
+      err instanceof Error ? err.message : String(err),
+    );
+  }
+
+  // Bound the wait: the capture thread may fail silently on some sources
+  // (locked screens, protected windows). Fall back to getDisplayMedia in
+  // that case rather than hang the user.
+  const firstFrameTimeout = window.setTimeout(() => {
+    firstFrameReject(new Error('first frame timed out (3s)'));
+  }, 3000);
+  try {
+    await firstFramePromise;
+  } catch (err: unknown) {
+    window.clearTimeout(firstFrameTimeout);
+    try {
+      await invoke('stop_screen_capture', { captureId });
+    } catch {
+      /* ignore */
+    }
+    throw new NativeCaptureUnavailable(
+      err instanceof Error ? err.message : String(err),
+    );
+  }
+  window.clearTimeout(firstFrameTimeout);
+
+  const stream = canvas.captureStream(opts.fps);
+
+  let stopped = false;
+  const stop = async (): Promise<void> => {
+    if (stopped) return;
+    stopped = true;
+    try {
+      await invoke('stop_screen_capture', { captureId });
+    } catch (err: unknown) {
+      console.warn('stop_screen_capture failed', err);
+    }
+    for (const track of stream.getTracks()) {
+      try {
+        track.stop();
+      } catch {
+        /* already stopped */
+      }
+    }
+  };
+
+  return { captureId, stream, stop };
+}
+
+function base64ToBytes(b64: string): Uint8Array {
+  const bin = atob(b64);
+  const bytes = new Uint8Array(bin.length);
+  for (let i = 0; i < bin.length; i++) {
+    bytes[i] = bin.charCodeAt(i);
+  }
+  return bytes;
+}