From 51e067e872f326754c7b2b0d4c92c10bd354f571 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:09:17 +0000 Subject: [PATCH 1/3] perf(desktop): skip duplicate browser updates --- apps/desktop/src/preview/Manager.test.ts | 49 ++++++++++++++++++++++-- apps/desktop/src/preview/Manager.ts | 31 +++++++++++++-- apps/web/src/previewStateStore.test.ts | 33 ++++++++++++++++ apps/web/src/previewStateStore.ts | 21 ++++++++++ 4 files changed, 127 insertions(+), 7 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 3bf6d63051af..fd9d58c45ef6 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -2448,9 +2448,7 @@ describe("PreviewManager", () => { const framesBeforePictureInPictureOnlyTick = pictureInPictureSend.mock.calls.length; yield* TestClock.adjust(100); expect(capturePage).toHaveBeenCalledTimes(3); - expect(pictureInPictureSend.mock.calls.length).toBeGreaterThan( - framesBeforePictureInPictureOnlyTick, - ); + expect(pictureInPictureSend.mock.calls.length).toBe(framesBeforePictureInPictureOnlyTick); expect(recordingFrames).toHaveLength(1); setBackgroundThrottling.mockImplementationOnce(() => { @@ -2467,6 +2465,51 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("delivers recording frames only when pixels change", () => + withManager((manager) => + Effect.gen(function* () { + const unchanged = Buffer.from("unchanged-preview-frame"); + const changed = Buffer.from("changed-preview-frame"); + let captureIndex = 0; + const capturePage = vi.fn(async () => { + const jpeg = captureIndex < 2 ? unchanged : changed; + captureIndex += 1; + return { + toJPEG: () => jpeg, + getSize: () => ({ width: 1280, height: 720 }), + }; + }); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + const frames: DesktopPreviewRecordingFrame[] = []; + + yield* manager.subscribeRecordingFrames((frame) => + Effect.sync(() => { + frames.push(frame); + }), + ); + yield* manager.createTab("tab_unchanged_frame"); + yield* manager.registerWebview("tab_unchanged_frame", 42); + yield* manager.startRecording("tab_unchanged_frame"); + + expect(frames.map((frame) => frame.data)).toEqual([unchanged.toString("base64")]); + + yield* TestClock.adjust(100); + + expect(capturePage).toHaveBeenCalledTimes(2); + expect(frames.map((frame) => frame.data)).toEqual([unchanged.toString("base64")]); + + yield* TestClock.adjust(100); + + expect(capturePage).toHaveBeenCalledTimes(3); + expect(frames.map((frame) => frame.data)).toEqual([ + unchanged.toString("base64"), + changed.toString("base64"), + ]); + yield* manager.stopRecording("tab_unchanged_frame"); + }), + ), + ); + effectIt.effect("retries a cold hidden-tab capture without dropping recording", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 0d90e0175fe3..181989d4581c 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -381,6 +381,7 @@ type FrameCaptureConsumer = "picture-in-picture" | "recording"; interface FrameCaptureSession { readonly scope: Scope.Closeable; readonly consumers: ReadonlySet; + readonly lastFrame: Buffer | null; } interface PictureInPictureSession { @@ -2549,18 +2550,38 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId, webContentsId: wc.id, }, - () => image.toJPEG(RECORDING_JPEG_QUALITY).toString("base64"), + () => image.toJPEG(RECORDING_JPEG_QUALITY), ); + const deliveryCaptureSession = yield* SynchronizedRef.modify( + frameCaptureSessionsRef, + (sessions) => { + const current = sessions.get(tabId); + if ( + current?.scope !== captureSession.scope || + current.lastFrame?.equals(encoded) === true + ) { + return [undefined, sessions] as const; + } + const next = { ...current, lastFrame: encoded }; + return [ + next, + replaceMap(sessions, (copy) => { + copy.set(tabId, next); + }), + ] as const; + }, + ); + if (!deliveryCaptureSession) return; const receivedAt = yield* currentIso; const frame: DesktopPreviewRecordingFrame = { tabId, - data: encoded, + data: encoded.toString("base64"), width: size.width, height: size.height, receivedAt, }; const deliveries: Array> = []; - if (currentCaptureSession.consumers.has("recording")) { + if (deliveryCaptureSession.consumers.has("recording")) { const listeners = yield* Ref.get(recordingFrameListenersRef); deliveries.push( Effect.forEach( @@ -2570,7 +2591,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ), ); } - if (currentCaptureSession.consumers.has("picture-in-picture")) { + if (deliveryCaptureSession.consumers.has("picture-in-picture")) { const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get( tabId, )?.window; @@ -2672,6 +2693,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function copy.set(tabId, { ...current, consumers: new Set([...current.consumers, consumer]), + lastFrame: null, }); }), ] as const; @@ -2687,6 +2709,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function copy.set(tabId, { scope, consumers: new Set([consumer]), + lastFrame: null, }); }), ] as const; diff --git a/apps/web/src/previewStateStore.test.ts b/apps/web/src/previewStateStore.test.ts index bfe5d46b1877..cc6c20b0cd54 100644 --- a/apps/web/src/previewStateStore.test.ts +++ b/apps/web/src/previewStateStore.test.ts @@ -20,6 +20,7 @@ import { rememberPreviewUrl, removePreviewThread, resetPreviewStateForTests, + subscribeThreadPreviewState, setActivePreviewTab, updatePreviewServerSnapshot, } from "./previewStateStore"; @@ -331,6 +332,38 @@ describe("previewStateStore (single-tab)", () => { expect(state.snapshot?.canGoBack).toBe(false); }); + it("does not publish duplicate desktop browser state", () => { + const snapshot = makeSnapshot(); + applyPreviewServerSnapshot(ref, snapshot); + const overlay = { + hasWebContents: true, + canGoBack: true, + canGoForward: false, + loading: false, + zoomFactor: 1, + pictureInPicture: false, + colorScheme: "system" as const, + audioMuted: false, + audible: false, + controller: "none" as const, + favicon: { + dataUrl: "data:image/png;base64,AA==", + pageUrl: "https://example.com", + capturedAt: 1, + }, + }; + const updates: unknown[] = []; + const unsubscribe = subscribeThreadPreviewState(ref, (state) => { + updates.push(state); + }); + + applyPreviewDesktopState(ref, snapshot.tabId, overlay); + applyPreviewDesktopState(ref, snapshot.tabId, { ...overlay, favicon: { ...overlay.favicon } }); + unsubscribe(); + + expect(updates).toHaveLength(1); + }); + it("retains multiple tabs and switches active desktop state", () => { const first = makeSnapshot(); const second = { ...makeSnapshot(), tabId: "tab_2", updatedAt: "2026-01-02T00:00:00.000Z" }; diff --git a/apps/web/src/previewStateStore.ts b/apps/web/src/previewStateStore.ts index a40e65fbc6a8..ced0e406559d 100644 --- a/apps/web/src/previewStateStore.ts +++ b/apps/web/src/previewStateStore.ts @@ -374,6 +374,27 @@ export function applyPreviewDesktopState( overlay: DesktopPreviewOverlay | null, ): void { updateThreadPreviewState(ref, (current) => { + const previous = current.desktopByTabId[tabId] ?? null; + if ( + previous === overlay || + (previous !== null && + overlay !== null && + previous.hasWebContents === overlay.hasWebContents && + previous.canGoBack === overlay.canGoBack && + previous.canGoForward === overlay.canGoForward && + previous.loading === overlay.loading && + previous.zoomFactor === overlay.zoomFactor && + previous.pictureInPicture === overlay.pictureInPicture && + previous.colorScheme === overlay.colorScheme && + previous.audioMuted === overlay.audioMuted && + previous.audible === overlay.audible && + previous.controller === overlay.controller && + previous.favicon?.dataUrl === overlay.favicon?.dataUrl && + previous.favicon?.pageUrl === overlay.favicon?.pageUrl && + previous.favicon?.capturedAt === overlay.favicon?.capturedAt) + ) { + return current; + } const desktopByTabId = { ...current.desktopByTabId }; if (overlay) desktopByTabId[tabId] = overlay; else delete desktopByTabId[tabId]; From 93e8a2526784524621684c373fd8acee2bae3f1b Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:28:21 +0000 Subject: [PATCH 2/3] fix(desktop): dedupe frames per consumer --- apps/desktop/src/preview/Manager.test.ts | 42 +++++++++++++++ apps/desktop/src/preview/Manager.ts | 65 ++++++++++++++---------- apps/web/src/previewStateStore.test.ts | 8 +-- 3 files changed, 85 insertions(+), 30 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index fd9d58c45ef6..f47d201a70bb 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -2434,6 +2434,7 @@ describe("PreviewManager", () => { ); expect(states.at(-1)?.pictureInPicture).toBe(true); expect(capturePage).toHaveBeenCalledOnce(); + const pictureInPictureFramesBeforeRecording = pictureInPictureSend.mock.calls.length; yield* manager.startRecording("tab_pip"); expect(capturePage).toHaveBeenCalledOnce(); @@ -2441,6 +2442,7 @@ describe("PreviewManager", () => { yield* TestClock.adjust(100); expect(capturePage).toHaveBeenCalledTimes(2); + expect(pictureInPictureSend).toHaveBeenCalledTimes(pictureInPictureFramesBeforeRecording); expect(recordingFrames).toHaveLength(1); yield* manager.stopRecording("tab_pip"); @@ -2465,6 +2467,46 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("delivers unchanged frames only to a new picture-in-picture consumer", () => + withManager((manager) => + Effect.gen(function* () { + const jpeg = Buffer.from("shared-preview-frame"); + const capturePage = vi.fn(async () => ({ + toJPEG: () => jpeg, + getSize: () => ({ width: 1280, height: 720 }), + })); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + const { pictureInPictureWindow, send } = makeTestPictureInPictureWindow(); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + const recordingFrames: DesktopPreviewRecordingFrame[] = []; + + yield* manager.subscribeRecordingFrames((frame) => + Effect.sync(() => { + recordingFrames.push(frame); + }), + ); + yield* manager.createTab("tab_recording_then_pip"); + yield* manager.registerWebview("tab_recording_then_pip", 42); + yield* manager.startRecording("tab_recording_then_pip"); + + expect(recordingFrames).toHaveLength(1); + yield* manager.openPictureInPicture("tab_recording_then_pip"); + expect(capturePage).toHaveBeenCalledOnce(); + expect(send).not.toHaveBeenCalled(); + + yield* TestClock.adjust(100); + + expect(capturePage).toHaveBeenCalledTimes(2); + expect(recordingFrames).toHaveLength(1); + expect(send).toHaveBeenCalledOnce(); + yield* manager.closePictureInPicture("tab_recording_then_pip"); + yield* manager.stopRecording("tab_recording_then_pip"); + }), + ), + ); + effectIt.effect("delivers recording frames only when pixels change", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 181989d4581c..96f65ccba7d0 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -381,7 +381,8 @@ type FrameCaptureConsumer = "picture-in-picture" | "recording"; interface FrameCaptureSession { readonly scope: Scope.Closeable; readonly consumers: ReadonlySet; - readonly lastFrame: Buffer | null; + readonly lastPictureInPictureFrame: Buffer | null; + readonly lastRecordingFrame: Buffer | null; } interface PictureInPictureSession { @@ -617,7 +618,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return [ undefined, replaceMap(sessions, (copy) => { - copy.set(tabId, { ...current, consumers }); + copy.set(tabId, { + ...current, + consumers, + lastPictureInPictureFrame: + consumer === "picture-in-picture" ? null : current.lastPictureInPictureFrame, + lastRecordingFrame: consumer === "recording" ? null : current.lastRecordingFrame, + }); }), ] as const; } @@ -2552,26 +2559,32 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }, () => image.toJPEG(RECORDING_JPEG_QUALITY), ); - const deliveryCaptureSession = yield* SynchronizedRef.modify( - frameCaptureSessionsRef, - (sessions) => { - const current = sessions.get(tabId); - if ( - current?.scope !== captureSession.scope || - current.lastFrame?.equals(encoded) === true - ) { - return [undefined, sessions] as const; - } - const next = { ...current, lastFrame: encoded }; - return [ - next, - replaceMap(sessions, (copy) => { - copy.set(tabId, next); - }), - ] as const; - }, - ); - if (!deliveryCaptureSession) return; + const frameConsumers = yield* SynchronizedRef.modify(frameCaptureSessionsRef, (sessions) => { + const current = sessions.get(tabId); + if (current?.scope !== captureSession.scope) { + return [undefined, sessions] as const; + } + const recording = + current.consumers.has("recording") && current.lastRecordingFrame?.equals(encoded) !== true; + const pictureInPicture = + current.consumers.has("picture-in-picture") && + current.lastPictureInPictureFrame?.equals(encoded) !== true; + if (!recording && !pictureInPicture) { + return [undefined, sessions] as const; + } + const next = { + ...current, + lastPictureInPictureFrame: pictureInPicture ? encoded : current.lastPictureInPictureFrame, + lastRecordingFrame: recording ? encoded : current.lastRecordingFrame, + }; + return [ + { pictureInPicture, recording }, + replaceMap(sessions, (copy) => { + copy.set(tabId, next); + }), + ] as const; + }); + if (!frameConsumers) return; const receivedAt = yield* currentIso; const frame: DesktopPreviewRecordingFrame = { tabId, @@ -2581,7 +2594,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function receivedAt, }; const deliveries: Array> = []; - if (deliveryCaptureSession.consumers.has("recording")) { + if (frameConsumers.recording) { const listeners = yield* Ref.get(recordingFrameListenersRef); deliveries.push( Effect.forEach( @@ -2591,7 +2604,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ), ); } - if (deliveryCaptureSession.consumers.has("picture-in-picture")) { + if (frameConsumers.pictureInPicture) { const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get( tabId, )?.window; @@ -2693,7 +2706,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function copy.set(tabId, { ...current, consumers: new Set([...current.consumers, consumer]), - lastFrame: null, }); }), ] as const; @@ -2709,7 +2721,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function copy.set(tabId, { scope, consumers: new Set([consumer]), - lastFrame: null, + lastPictureInPictureFrame: null, + lastRecordingFrame: null, }); }), ] as const; diff --git a/apps/web/src/previewStateStore.test.ts b/apps/web/src/previewStateStore.test.ts index cc6c20b0cd54..321dd68c09aa 100644 --- a/apps/web/src/previewStateStore.test.ts +++ b/apps/web/src/previewStateStore.test.ts @@ -352,16 +352,16 @@ describe("previewStateStore (single-tab)", () => { capturedAt: 1, }, }; - const updates: unknown[] = []; - const unsubscribe = subscribeThreadPreviewState(ref, (state) => { - updates.push(state); + let updateCount = 0; + const unsubscribe = subscribeThreadPreviewState(ref, () => { + updateCount += 1; }); applyPreviewDesktopState(ref, snapshot.tabId, overlay); applyPreviewDesktopState(ref, snapshot.tabId, { ...overlay, favicon: { ...overlay.favicon } }); unsubscribe(); - expect(updates).toHaveLength(1); + expect(updateCount).toBe(1); }); it("retains multiple tabs and switches active desktop state", () => { From 1279cf80161ac4a2bf48d194cc33b5bf4e0d01e4 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:36:43 +0000 Subject: [PATCH 3/3] fix(desktop): retry failed picture-in-picture frames --- apps/desktop/src/preview/Manager.test.ts | 31 ++++++++++++++++++++++++ apps/desktop/src/preview/Manager.ts | 25 ++++++++++++------- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index f47d201a70bb..d03a55d197f8 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -2507,6 +2507,37 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("retries an unchanged picture-in-picture frame after delivery fails", () => + withManager((manager) => + Effect.gen(function* () { + const jpeg = Buffer.from("retry-preview-frame"); + const capturePage = vi.fn(async () => ({ + toJPEG: () => jpeg, + getSize: () => ({ width: 1280, height: 720 }), + })); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + const { pictureInPictureWindow, send } = makeTestPictureInPictureWindow(); + send.mockImplementationOnce(() => { + throw new Error("picture-in-picture delivery failed"); + }); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + + yield* manager.createTab("tab_pip_delivery_retry"); + yield* manager.registerWebview("tab_pip_delivery_retry", 42); + yield* manager.openPictureInPicture("tab_pip_delivery_retry"); + expect(send).toHaveBeenCalledOnce(); + + yield* TestClock.adjust(100); + + expect(capturePage).toHaveBeenCalledTimes(2); + expect(send).toHaveBeenCalledTimes(2); + yield* manager.closePictureInPicture("tab_pip_delivery_retry"); + }), + ), + ); + effectIt.effect("delivers recording frames only when pixels change", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 96f65ccba7d0..be2f73c476ec 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2572,16 +2572,14 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (!recording && !pictureInPicture) { return [undefined, sessions] as const; } - const next = { - ...current, - lastPictureInPictureFrame: pictureInPicture ? encoded : current.lastPictureInPictureFrame, - lastRecordingFrame: recording ? encoded : current.lastRecordingFrame, - }; + const next = recording ? { ...current, lastRecordingFrame: encoded } : current; return [ - { pictureInPicture, recording }, - replaceMap(sessions, (copy) => { - copy.set(tabId, next); - }), + { pictureInPicture, recording, session: next }, + recording + ? replaceMap(sessions, (copy) => { + copy.set(tabId, next); + }) + : sessions, ] as const; }); if (!frameConsumers) return; @@ -2654,6 +2652,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); }, ); + yield* SynchronizedRef.update(frameCaptureSessionsRef, (sessions) => { + if (sessions.get(tabId) !== frameConsumers.session) return sessions; + return replaceMap(sessions, (copy) => { + copy.set(tabId, { + ...frameConsumers.session, + lastPictureInPictureFrame: encoded, + }); + }); + }); }).pipe( Effect.catch((error) => Effect.logWarning("Picture-in-picture frame delivery failed.", {