From c449943b52ca2344f6759bc53dd2ae9147784515 Mon Sep 17 00:00:00 2001 From: byGalax Date: Sun, 17 May 2026 00:12:03 +0200 Subject: [PATCH] perf(P6B.T7): WebP thumbnails for image attachments (320px max, thumb-first render) --- .../src/components/AttachmentImage.tsx | 119 ++++++++++++++++-- apps/desktop/src/lib/imageCompress.ts | 48 +++++++ .../src/lib/useConversationMessages.ts | 8 ++ packages/shared/src/chat/attachments.ts | 92 ++++++++++++++ 4 files changed, 258 insertions(+), 9 deletions(-) diff --git a/apps/desktop/src/components/AttachmentImage.tsx b/apps/desktop/src/components/AttachmentImage.tsx index 9623764..f5c32e7 100644 --- a/apps/desktop/src/components/AttachmentImage.tsx +++ b/apps/desktop/src/components/AttachmentImage.tsx @@ -1,4 +1,8 @@ -import { type AttachmentHandle, downloadAndDecryptAttachment } from '@chat-app/shared/chat'; +import { + type AttachmentHandle, + downloadAndDecryptAttachment, + downloadAndDecryptAttachmentThumb, +} from '@chat-app/shared/chat'; import { useEffect, useState } from 'react'; import { getCachedAttachment, putCachedAttachment } from '../lib/attachmentCache'; @@ -56,6 +60,17 @@ export function AttachmentImage({ handle, mine = false }: Props) { const [error, setError] = useState(null); const [lightboxOpen, setLightboxOpen] = useState(false); + // Whether the sender shipped a pre-built WebP thumb alongside this attachment + // (Phase 6B+). When true we render the bubble from just the thumb and only + // fetch the full blob when the user opens the lightbox or for view-once. + const hasServerThumb = Boolean(handle.thumbStoragePath && handle.thumbNonceB64); + // View-once needs the full blob ready instantly the moment the recipient + // taps (otherwise we'd show a spinner during the burn animation, then race + // the "mark viewed" RPC). Same for the legacy path with no server thumb — + // we have to download the whole thing just to render the client-side + // makeThumbnail fallback. + const needsEagerFull = handle.viewOnce === true || !hasServerThumb; + useEffect(() => { let cancelled = false; const created: string[] = []; @@ -69,9 +84,39 @@ export function AttachmentImage({ handle, mine = false }: Props) { return u; }; - // OPFS cache → decrypt → generate thumbnail for inline display. - // Lightbox swaps to the full blob when opened. + const thumbCacheId = handle.id + '-thumb'; + void (async () => { + // Phase 6B fast path: if the sender shipped a server-side WebP thumb, + // grab it first so the bubble paints from ~20KB instead of waiting on + // the multi-MB full blob. The OPFS cache is keyed separately so the + // thumb survives independent of full-blob eviction. + if (hasServerThumb) { + try { + let thumbBlob: Blob | null = await getCachedAttachment(thumbCacheId); + if (!thumbBlob) { + thumbBlob = await downloadAndDecryptAttachmentThumb({ + client: supabase, + handle, + }); + if (thumbBlob) void putCachedAttachment(thumbCacheId, thumbBlob); + } + if (cancelled) return; + if (thumbBlob) { + setThumbUrl(take(thumbBlob)); + } + } catch (err: unknown) { + // Thumb decrypt failure isn't fatal — fall through to the full + // blob path below so the user still sees the image. + console.warn('thumb load failed', err); + } + } + + // Eagerly resolve the full blob when we need it for view-once or as + // the only render source (no server thumb). For the thumb-first path + // the full blob is deferred until the lightbox opens (see below). + if (!needsEagerFull) return; + const cached = await getCachedAttachment(handle.id); let blob: Blob; if (cached) { @@ -90,10 +135,14 @@ export function AttachmentImage({ handle, mine = false }: Props) { if (cancelled) return; const full = take(blob); setFullUrl(full); - const thumb = await makeThumbnail(blob); - if (cancelled) return; - if (thumb) { - setThumbUrl(take(thumb)); + // Pre-Phase-6B fallback: no server thumb shipped, so re-derive a + // smaller preview on the client to keep memory pressure down. + if (!hasServerThumb) { + const thumb = await makeThumbnail(blob); + if (cancelled) return; + if (thumb) { + setThumbUrl(take(thumb)); + } } })(); @@ -101,7 +150,51 @@ export function AttachmentImage({ handle, mine = false }: Props) { cancelled = true; for (const u of created) URL.revokeObjectURL(u); }; - }, [handle.id, handle.storagePath, handle.keyB64, handle.nonceB64]); + }, [ + handle, + handle.id, + handle.storagePath, + handle.keyB64, + handle.nonceB64, + handle.thumbStoragePath, + handle.thumbNonceB64, + hasServerThumb, + needsEagerFull, + ]); + + // Lazy full-image fetch for click-to-expand (only kicks in when we + // skipped the eager full-blob download above). Resolves into the same + // `fullUrl` state that the Lightbox consumes; the thumb URL keeps + // backing the bubble until the lightbox actually mounts. + useEffect(() => { + if (!lightboxOpen) return; + if (fullUrl) return; + let cancelled = false; + const created: string[] = []; + void (async () => { + const cached = await getCachedAttachment(handle.id); + let blob: Blob; + if (cached) { + blob = cached; + } else { + try { + blob = await downloadAndDecryptAttachment({ client: supabase, handle }); + void putCachedAttachment(handle.id, blob); + } catch (err: unknown) { + if (!cancelled) setError(err instanceof Error ? err.message : 'download failed'); + return; + } + } + if (cancelled) return; + const u = URL.createObjectURL(blob); + created.push(u); + setFullUrl(u); + })(); + return () => { + cancelled = true; + for (const u of created) URL.revokeObjectURL(u); + }; + }, [lightboxOpen, fullUrl, handle]); const blobUrl = thumbUrl ?? fullUrl; @@ -157,7 +250,15 @@ export function AttachmentImage({ handle, mine = false }: Props) { className="block h-auto max-h-80 w-auto max-w-full object-contain" /> - {lightboxOpen && fullUrl && setLightboxOpen(false)} />} + {lightboxOpen && ( + setLightboxOpen(false)} + /> + )} ); } diff --git a/apps/desktop/src/lib/imageCompress.ts b/apps/desktop/src/lib/imageCompress.ts index 4b8ef43..140b9ac 100644 --- a/apps/desktop/src/lib/imageCompress.ts +++ b/apps/desktop/src/lib/imageCompress.ts @@ -55,3 +55,51 @@ function renameToWebp(original: string): string { export async function compressImages(files: File[]): Promise { return Promise.all(files.map((f) => compressImage(f))); } + +// Bandwidth threshold for thumb generation. Below ~50KB the WebP overhead +// of a fresh re-encode can exceed the original; not worth a second upload. +const THUMB_SKIP_BELOW_BYTES = 50 * 1024; +const THUMB_MAX_DIM = 320; +const THUMB_QUALITY = 0.7; +// Animated formats lose motion when redrawn onto a Canvas, so we skip them +// and let the receiver render the full file. GIF is the dominant case; the +// rest stay too (apng/animated-webp). +const THUMB_ANIMATED_MIME = /^image\/(gif|apng)$/; + +// Generates a small WebP preview thumb (max 320×320) from an image file. +// Used by the send path so each image attachment can ship a tiny inline +// preview alongside the encrypted full blob. Returns `null` when: +// - the input isn't an image, +// - the input is animated (GIF/APNG — would lose motion), +// - the input is already small enough that a thumb wouldn't save bandwidth, +// - OffscreenCanvas / createImageBitmap aren't available, or +// - decode/encode threw (corrupt input). +// The caller treats `null` as "skip thumb" and uploads only the full blob. +export async function generateWebPThumb( + file: File, + maxDim: number = THUMB_MAX_DIM, +): Promise { + if (!file.type.startsWith('image/')) return null; + if (THUMB_ANIMATED_MIME.test(file.type)) return null; + if (file.size < THUMB_SKIP_BELOW_BYTES) return null; + if (typeof createImageBitmap !== 'function') return null; + if (typeof OffscreenCanvas !== 'function') return null; + try { + const bitmap = await createImageBitmap(file); + const ratio = Math.min(maxDim / bitmap.width, maxDim / bitmap.height, 1); + const w = Math.max(1, Math.round(bitmap.width * ratio)); + const h = Math.max(1, Math.round(bitmap.height * ratio)); + const canvas = new OffscreenCanvas(w, h); + const ctx = canvas.getContext('2d'); + if (!ctx) { + bitmap.close(); + return null; + } + ctx.drawImage(bitmap, 0, 0, w, h); + bitmap.close(); + return await canvas.convertToBlob({ type: 'image/webp', quality: THUMB_QUALITY }); + } catch (err: unknown) { + console.warn('generateWebPThumb failed', err); + return null; + } +} diff --git a/apps/desktop/src/lib/useConversationMessages.ts b/apps/desktop/src/lib/useConversationMessages.ts index d788c41..e3728d0 100644 --- a/apps/desktop/src/lib/useConversationMessages.ts +++ b/apps/desktop/src/lib/useConversationMessages.ts @@ -16,6 +16,7 @@ import { bytesToPgHex, pgBytesToBytes } from '@chat-app/shared/supabase'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { decryptBatch as decryptBatchWorker } from './decryptWorker'; +import { generateWebPThumb } from './imageCompress'; import { loadCachedMessages, persistMessages, @@ -632,6 +633,12 @@ export function useConversationMessages({ conversationId, userId, deviceId }: Ar throw new Error('attachment exceeds max size (10 MB)'); } const dims = await readImageDimensions(file); + // Phase 6B: generate a small WebP preview thumb so the receiver's + // bubble loads fast (typical 320×240 WebP is 10–30KB vs the full + // image's 1–10MB). `generateWebPThumb` short-circuits to null on + // non-images, animated formats, and small files — and on failure; + // the upload helper then just skips the second upload. + const thumbBlob = await generateWebPThumb(file); const res = await encryptAndUploadAttachment({ client: supabase, conversationId, @@ -640,6 +647,7 @@ export function useConversationMessages({ conversationId, userId, deviceId }: Ar sizeBytes: file.size, ...(dims.width !== undefined ? { width: dims.width } : {}), ...(dims.height !== undefined ? { height: dims.height } : {}), + ...(thumbBlob ? { thumbBlob } : {}), }); // Stamp the view-once flag on each handle the caller requested it // for. The flag rides inside the encrypted payload (so peers can diff --git a/packages/shared/src/chat/attachments.ts b/packages/shared/src/chat/attachments.ts index e8e45ca..b675c00 100644 --- a/packages/shared/src/chat/attachments.ts +++ b/packages/shared/src/chat/attachments.ts @@ -35,6 +35,14 @@ export interface AttachmentHandle { /** UUID of the first non-sender who viewed. Populated alongside `viewedAt` * by the mark-viewed RPC so the renderer can render attribution. */ viewedBy?: string | null; + /** Storage path of the encrypted WebP preview thumb (max 320×320). The + * thumb shares the per-attachment symmetric key with the full blob but + * uses its own nonce. Absent on pre-Phase-6B messages — the receiver + * falls through to downloading the full blob in that case. */ + thumbStoragePath?: string; + /** Base-64 nonce that decrypts `-thumb.bin`. Always present iff + * `thumbStoragePath` is set. */ + thumbNonceB64?: string; } export type CallEventStatus = 'ended' | 'missed' | 'declined'; @@ -241,6 +249,13 @@ export interface EncryptedAttachmentResult { // Encrypt + upload a blob. Does NOT insert the message_attachments row — // the caller combines this with a message insert so everything commits // atomically at the application layer. +// +// When `thumbBlob` is supplied (Phase 6B image-thumbnail path) a second +// ciphertext is uploaded to `/-thumb.bin` encrypted +// with the SAME per-attachment key + a fresh nonce. The handle's +// `thumbStoragePath` / `thumbNonceB64` get populated so the receiver +// can prefer the thumb for inline preview. Thumb-upload failure is +// non-fatal — we log and fall through so the full image still posts. export async function encryptAndUploadAttachment(params: { client: AppSupabaseClient; conversationId: string; @@ -249,6 +264,7 @@ export async function encryptAndUploadAttachment(params: { sizeBytes: number; width?: number; height?: number; + thumbBlob?: Blob | null; }): Promise { const backend = getCryptoBackend(); @@ -268,6 +284,35 @@ export async function encryptAndUploadAttachment(params: { }); if (error) throw error; + let thumbStoragePath: string | undefined; + let thumbNonceB64: string | undefined; + if (params.thumbBlob) { + try { + const thumbBytes = new Uint8Array(await params.thumbBlob.arrayBuffer()); + const thumbNonce = backend.randomBytes(backend.secretboxNonceLength); + const thumbCipher = backend.secretbox(thumbBytes, thumbNonce, key); + const thumbPath = params.conversationId + '/' + id + '-thumb.bin'; + const { error: thumbErr } = await params.client.storage + .from(ATTACHMENT_BUCKET) + .upload(thumbPath, thumbCipher, { + contentType: 'application/octet-stream', + upsert: false, + }); + if (thumbErr) { + // Non-fatal: log and continue with full-only handle. Receiver will + // fall back to fetching the full blob. + console.warn('thumb upload failed', thumbErr); + } else { + thumbStoragePath = thumbPath; + thumbNonceB64 = await toBase64(thumbNonce); + } + // Wipe nonce buffer. + for (let i = 0; i < thumbNonce.length; i++) thumbNonce[i] = 0; + } catch (err: unknown) { + console.warn('thumb encrypt/upload failed', err); + } + } + const handle: AttachmentHandle = { id, storagePath, @@ -277,6 +322,8 @@ export async function encryptAndUploadAttachment(params: { ...(params.height !== undefined ? { height: params.height } : {}), keyB64: await toBase64(key), nonceB64: await toBase64(nonce), + ...(thumbStoragePath !== undefined ? { thumbStoragePath } : {}), + ...(thumbNonceB64 !== undefined ? { thumbNonceB64 } : {}), }; return { handle, key, nonce }; @@ -309,6 +356,51 @@ export async function downloadAndDecryptAttachment(params: { return new Blob([copy.buffer], { type: params.handle.mimeType }); } +// Download + decrypt the small WebP preview thumb that the sender uploaded +// alongside an image attachment (Phase 6B optimisation). Returns `null` if +// the handle has no thumb metadata (pre-Phase-6B message) or if the thumb +// blob is missing from storage — caller falls back to the full image. +// +// We deliberately swallow ANY download error (missing object, transient +// 5xx) so the receiver gracefully degrades to the full-blob path; only a +// successful decrypt-failure throws, since that signals a real corruption. +export async function downloadAndDecryptAttachmentThumb(params: { + client: AppSupabaseClient; + handle: AttachmentHandle; +}): Promise { + if (!params.handle.thumbStoragePath || !params.handle.thumbNonceB64) { + return null; + } + const backend = getCryptoBackend(); + + let data: Blob | null = null; + try { + const res = await params.client.storage + .from(ATTACHMENT_BUCKET) + .download(params.handle.thumbStoragePath); + if (res.error) { + // Likely 404 — sender failed to upload thumb, or it's been GC'd. + // Receiver falls back to full image. + return null; + } + data = res.data; + } catch { + return null; + } + if (!data) return null; + + const ciphertext = new Uint8Array(await data.arrayBuffer()); + const key = await fromBase64(params.handle.keyB64); + const nonce = await fromBase64(params.handle.thumbNonceB64); + const plainBytes = backend.secretboxOpen(ciphertext, nonce, key); + for (let i = 0; i < key.length; i++) key[i] = 0; + for (let i = 0; i < nonce.length; i++) nonce[i] = 0; + const copy = new Uint8Array(plainBytes.byteLength); + copy.set(plainBytes); + // Thumbs are always image/webp regardless of original mime. + return new Blob([copy.buffer], { type: 'image/webp' }); +} + // Insert the public metadata row for an attachment. The ciphertext itself has // already been uploaded to storage under `handle.storagePath`. export async function insertAttachmentRow(