From eb9590191c96e3a53d75b702d2487964ec5be4e1 Mon Sep 17 00:00:00 2001 From: mweinbach Date: Mon, 24 Aug 2026 21:17:13 -0400 Subject: [PATCH 1/3] chore(web): add browser-side HEIC image decoder --- apps/web/package.json | 1 + pnpm-lock.yaml | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/apps/web/package.json b/apps/web/package.json index 598feaec0ce9..ea6b00effb3b 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -34,6 +34,7 @@ "class-variance-authority": "^0.7.1", "culori": "^4.0.2", "effect": "catalog:", + "heic-to": "^1.5.2", "jose": "catalog:", "jsonc-parser": "3.3.1", "jszip": "3.10.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6f456d7e65f9..02dda6303a64 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -590,6 +590,9 @@ importers: effect: specifier: 4.0.0-beta.103 version: 4.0.0-beta.103(patch_hash=af36b7948b6f9c56623074662b51dade5699880c1a7c71245de73e13c3185fb6) + heic-to: + specifier: ^1.5.2 + version: 1.5.2 jose: specifier: 'catalog:' version: 6.2.2 @@ -7149,6 +7152,9 @@ packages: headers-polyfill@4.0.3: resolution: {integrity: sha512-IScLbePpkvO846sIwOtOTDjutRMWdXdJmXdMvk6gCBHxFO8d+QKOQedyZSxFTTFYRSmlgSTDtXqqq4pcenBXLQ==} + heic-to@1.5.2: + resolution: {integrity: sha512-8Fns+lZHAWmz5U5IUxDeXKwIf3foBoKNPLxxFY4B0MkLjNuomEIHCoDbDE+x/llFK3NCEO1cu4+n3iUKY+Svmw==} + hermes-compiler@250829098.0.10: resolution: {integrity: sha512-TcRlZ0/TlyfJqquRFAWoyElVNnkdYRi/sEp4/Qy8/GYxjg8j2cS9D4MjuaQ+qimkmLN7AmO+44IznRf06mAr0w==} @@ -17820,6 +17826,8 @@ snapshots: headers-polyfill@4.0.3: {} + heic-to@1.5.2: {} + hermes-compiler@250829098.0.10: {} hermes-estree@0.33.3: {} From d30992fc21e73b0de4d9af5292819de84b2938ac Mon Sep 17 00:00:00 2001 From: mweinbach Date: Mon, 24 Aug 2026 21:23:25 -0400 Subject: [PATCH 2/3] feat(web): convert HEIC photo attachments to JPEG --- apps/web/src/components/chat/ChatComposer.tsx | 22 +++-- apps/web/src/lib/imageCompression.test.ts | 93 +++++++++++++++++++ apps/web/src/lib/imageCompression.ts | 64 +++++++++++-- docs/user/composer.md | 3 + 4 files changed, 170 insertions(+), 12 deletions(-) diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 1ec58e0de702..841bbb8f4011 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -73,7 +73,11 @@ import { type ComposerTaskStep, type ComposerTasksProgress, } from "./ComposerTasksBadge"; -import { compressImageForStash, compressImageToByteLimit } from "../../lib/imageCompression"; +import { + compressImageForStash, + isHeicImageFile, + prepareImageForAttachment, +} from "../../lib/imageCompression"; import { releaseAttachmentUpload, retryAttachmentUpload, @@ -2537,12 +2541,13 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) const acceptedFiles: File[] = []; let error: string | null = null; for (const file of files) { - if (!file.type.startsWith("image/")) { + const isHeicImage = isHeicImageFile(file); + if (!file.type.startsWith("image/") && !isHeicImage) { error = `Unsupported file type for '${file.name}'. Please attach image files only.`; continue; } - if (!isProviderSendTurnSupportedImageMimeType(file.type)) { - error = `'${file.name}' is not a supported image type. Attach GIF, JPEG, PNG, or WebP images.`; + if (!isHeicImage && !isProviderSendTurnSupportedImageMimeType(file.type)) { + error = `'${file.name}' is not a supported image type. Attach GIF, HEIC, HEIF, JPEG, PNG, or WebP images.`; continue; } if (reservedCount >= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) { @@ -2562,7 +2567,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) for (const file of acceptedFiles) { // Images over the wire cap are downscaled to fit rather than // refused; files already within it pass through byte-for-byte. - const compressed = await compressImageToByteLimit(file, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES); + const compressed = await prepareImageForAttachment( + file, + PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, + ); if (!compressed.ok) { compressionError = compressed.reason === "unreadable" @@ -2615,7 +2623,9 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) const onComposerPaste = (event: React.ClipboardEvent) => { const files = Array.from(event.clipboardData.files); if (files.length === 0) return; - const imageFiles = files.filter((file) => file.type.startsWith("image/")); + const imageFiles = files.filter( + (file) => file.type.startsWith("image/") || isHeicImageFile(file), + ); if (imageFiles.length === 0) return; event.preventDefault(); void addComposerImages(imageFiles); diff --git a/apps/web/src/lib/imageCompression.test.ts b/apps/web/src/lib/imageCompression.test.ts index 63712ca7e295..a8da3d4b27fd 100644 --- a/apps/web/src/lib/imageCompression.test.ts +++ b/apps/web/src/lib/imageCompression.test.ts @@ -3,10 +3,20 @@ import { afterEach, describe, expect, it, vi } from "vite-plus/test"; import { compressImageForStash, compressImageToByteLimit, + isHeicImageFile, MAX_COMPRESSIBLE_SOURCE_BYTES, MAX_STASH_IMAGE_DATA_URL_CHARS, + prepareImageForAttachment, } from "./imageCompression"; +const mocks = vi.hoisted(() => ({ + heicTo: vi.fn(), +})); + +vi.mock("heic-to/csp", () => ({ + heicTo: mocks.heicTo, +})); + /** * jsdom has no real canvas/codec, so the re-encode path is exercised with * stubbed `createImageBitmap` + `OffscreenCanvas`. The encoder stub returns a @@ -63,6 +73,7 @@ function stubCanvasPipeline( } afterEach(() => { + mocks.heicTo.mockReset(); vi.unstubAllGlobals(); globalThis.createImageBitmap = originalCreateImageBitmap; globalThis.OffscreenCanvas = originalOffscreenCanvas; @@ -251,3 +262,85 @@ describe("compressImageForStash", () => { expect(smallestRequested).toBeLessThan(800); }); }); + +describe("HEIC attachment preparation", () => { + it("recognizes HEIC and HEIF MIME types and case-insensitive file extensions", () => { + expect(isHeicImageFile({ name: "photo.bin", type: "image/heic" })).toBe(true); + expect(isHeicImageFile({ name: "photo.bin", type: "image/heif-sequence" })).toBe(true); + expect(isHeicImageFile({ name: "IMG_1234.HEIC", type: "" })).toBe(true); + expect(isHeicImageFile({ name: "photo.heif", type: "application/octet-stream" })).toBe(true); + expect(isHeicImageFile({ name: "photo.png", type: "image/png" })).toBe(false); + }); + + it("converts a HEIC photo with a missing MIME type into a named JPEG", async () => { + const original = new File([new Uint8Array([1, 2, 3])], "IMG_1234.HEIC", { + lastModified: 123, + }); + mocks.heicTo.mockResolvedValueOnce( + new Blob([new Uint8Array([4, 5, 6, 7])], { type: "image/jpeg" }), + ); + + const result = await prepareImageForAttachment(original, 1024); + + expect(mocks.heicTo).toHaveBeenCalledWith({ + blob: original, + type: "image/jpeg", + quality: 0.92, + }); + expect(result.ok && result.file.name).toBe("IMG_1234.jpg"); + expect(result.ok && result.file.type).toBe("image/jpeg"); + expect(result.ok && result.file.size).toBe(4); + expect(result.ok && result.file.lastModified).toBe(123); + expect(result.ok && result.recompressed).toBe(true); + }); + + it("keeps oversized converted photos in JPEG format while shrinking them", async () => { + const original = new File([new Uint8Array([1, 2, 3])], "photo.heif", { + type: "image/heif", + }); + mocks.heicTo.mockResolvedValueOnce( + new Blob([new Uint8Array(2_000_000)], { type: "image/jpeg" }), + ); + const { fillRect } = stubCanvasPipeline(() => 200_000); + + const result = await prepareImageForAttachment(original, 1_000_000); + + expect(result.ok && result.file.name).toBe("photo.jpg"); + expect(result.ok && result.file.type).toBe("image/jpeg"); + expect(result.ok && result.file.size).toBeLessThanOrEqual(1_000_000); + expect(fillRect).toHaveBeenCalled(); + }); + + it("reports unreadable when HEIC decoding fails", async () => { + const original = new File([new Uint8Array([1, 2, 3])], "broken.heic", { + type: "image/heic", + }); + mocks.heicTo.mockRejectedValueOnce(new Error("Invalid HEIC image")); + + expect(await prepareImageForAttachment(original, 1024)).toEqual({ + ok: false, + reason: "unreadable", + }); + }); + + it("rejects unsafe HEIC sources before loading the decoder", async () => { + const original = new File(["photo"], "large.heic", { type: "image/heic" }); + Object.defineProperty(original, "size", { value: MAX_COMPRESSIBLE_SOURCE_BYTES + 1 }); + + expect(await prepareImageForAttachment(original, 1024)).toEqual({ + ok: false, + reason: "too-large", + }); + expect(mocks.heicTo).not.toHaveBeenCalled(); + }); + + it("leaves supported images untouched without loading the HEIC decoder", async () => { + const original = makeFile(1024); + + const result = await prepareImageForAttachment(original, 2048); + + expect(result.ok && result.file).toBe(original); + expect(result.ok && result.recompressed).toBe(false); + expect(mocks.heicTo).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/lib/imageCompression.ts b/apps/web/src/lib/imageCompression.ts index be45024f38c4..453e6c6e0bd2 100644 --- a/apps/web/src/lib/imageCompression.ts +++ b/apps/web/src/lib/imageCompression.ts @@ -8,7 +8,8 @@ * `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit * via `compressImageToByteLimit` instead of rejecting the paste. * - * Images already within budget pass through untouched. + * Supported images already within budget pass through untouched. HEIC/HEIF + * photos are decoded to JPEG first because providers cannot consume them. */ /** @@ -32,6 +33,8 @@ export const MAX_COMPRESSIBLE_SOURCE_BYTES = 50 * 1024 * 1024; const QUALITY_STEPS = [0.92, 0.85, 0.78, 0.68] as const; /** Extra downscale passes applied when even the lowest quality overflows. */ const FALLBACK_SCALE_STEPS = [0.75, 0.55] as const; +const HEIC_IMAGE_MIME_TYPE = /^image\/hei(?:c|f)(?:-sequence)?$/i; +const HEIC_IMAGE_EXTENSION = /\.(?:heic|heif)$/i; export interface CompressedStashImage { dataUrl: string; @@ -55,6 +58,11 @@ export type CompressImageFileResult = | { ok: true; file: File; recompressed: boolean } | { ok: false; reason: ImageCompressionFailureReason }; +/** Finder and some browsers omit the MIME type when dragging HEIC photos. */ +export function isHeicImageFile(file: Pick): boolean { + return HEIC_IMAGE_MIME_TYPE.test(file.type) || HEIC_IMAGE_EXTENSION.test(file.name); +} + /** Chunked so a large image can't blow the argument limit of `fromCharCode`. */ const BASE64_CHUNK_SIZE = 0x8000; @@ -167,6 +175,7 @@ async function encodeWithinBudget( bitmap: ImageBitmap, maxDimension: number, budgetChars: number, + preferredMimeType?: "image/jpeg", ): Promise<{ dataUrl: string; mimeType: string } | null> { const scale = Math.min(1, maxDimension / Math.max(bitmap.width, bitmap.height)); const width = Math.max(1, Math.round(bitmap.width * scale)); @@ -176,8 +185,11 @@ async function encodeWithinBudget( // Probe WebP once; JPEG (no alpha) needs a white matte, so the fill has to // happen before drawing and depends on which codec we end up using. - const probe = await encodeCanvas(target.canvas, QUALITY_STEPS[0], "image/webp", 0); - const mimeType = probe ? "image/webp" : "image/jpeg"; + const mimeType = + preferredMimeType ?? + ((await encodeCanvas(target.canvas, QUALITY_STEPS[0], "image/webp", 0)) + ? "image/webp" + : "image/jpeg"); if (mimeType === "image/jpeg") { target.context.fillStyle = "#ffffff"; @@ -203,7 +215,11 @@ type ReencodeResult = * Shared re-encode loop: decodes `file`, then walks the quality ladder and * fallback downscale passes until an encoding fits `budgetChars`. */ -async function reencodeWithinBudget(file: File, budgetChars: number): Promise { +async function reencodeWithinBudget( + file: File, + budgetChars: number, + preferredMimeType?: "image/jpeg", +): Promise { if (!canRecompress()) { return { ok: false, reason: "too-large" }; } @@ -229,7 +245,7 @@ async function reencodeWithinBudget(file: File, budgetChars: number): Promise { if (file.size <= maxBytes) { return { ok: true, file, recompressed: false }; @@ -316,7 +333,7 @@ export async function compressImageToByteLimit( // into 4 chars; flooring keeps the budget a hair conservative instead of // admitting an encoding right at the byte cap. const budgetChars = Math.floor(maxBytes / 3) * 4; - const reencoded = await reencodeWithinBudget(file, budgetChars); + const reencoded = await reencodeWithinBudget(file, budgetChars, options?.preferredMimeType); if (!reencoded.ok) { return reencoded; } @@ -330,3 +347,38 @@ export async function compressImageToByteLimit( recompressed: true, }; } + +/** + * Converts HEIC/HEIF photos to provider-compatible JPEG before applying the + * attachment size limit. The decoder is loaded only when such a photo arrives. + */ +export async function prepareImageForAttachment( + file: File, + maxBytes: number, +): Promise { + if (!isHeicImageFile(file)) { + return compressImageToByteLimit(file, maxBytes); + } + + if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) { + return { ok: false, reason: "too-large" }; + } + + let converted: Blob; + try { + const { heicTo } = await import("heic-to/csp"); + converted = await heicTo({ blob: file, type: "image/jpeg", quality: QUALITY_STEPS[0] }); + } catch { + return { ok: false, reason: "unreadable" }; + } + + const jpeg = new File([converted], fileNameForMimeType(file.name || "image", "image/jpeg"), { + type: "image/jpeg", + lastModified: file.lastModified, + }); + const result = await compressImageToByteLimit(jpeg, maxBytes, { + preferredMimeType: "image/jpeg", + }); + + return result.ok ? { ...result, recompressed: true } : result; +} diff --git a/docs/user/composer.md b/docs/user/composer.md index b7ef57a56015..35d634556d88 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -7,6 +7,9 @@ multiple messages, then send again in the same thread. On servers that support direct uploads, images upload as soon as you add them. The send button becomes available after every upload finishes. Failed uploads can be retried or removed. +On web and desktop, HEIC and HEIF photos are automatically converted to JPEG when you drag them into +the composer or paste them into a message. + ## Commands and skills Type `/` to open the command menu. Type `$` to find and add a skill. Skill rows show their source, From ffcebada711ce18548ce6d5e5bfd884355413607 Mon Sep 17 00:00:00 2001 From: mweinbach Date: Mon, 24 Aug 2026 22:06:27 -0400 Subject: [PATCH 3/3] fix(web): compress oversized HEIC conversion outputs --- apps/web/src/lib/imageCompression.test.ts | 19 +++++++++++++++++++ apps/web/src/lib/imageCompression.ts | 9 ++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/apps/web/src/lib/imageCompression.test.ts b/apps/web/src/lib/imageCompression.test.ts index a8da3d4b27fd..728224323faf 100644 --- a/apps/web/src/lib/imageCompression.test.ts +++ b/apps/web/src/lib/imageCompression.test.ts @@ -311,6 +311,25 @@ describe("HEIC attachment preparation", () => { expect(fillRect).toHaveBeenCalled(); }); + it("compresses JPEG intermediates above the source safety ceiling", async () => { + const original = new File([new Uint8Array([1, 2, 3])], "large.heic", { + type: "image/heic", + }); + mocks.heicTo.mockResolvedValueOnce( + new Blob([new Uint8Array(MAX_COMPRESSIBLE_SOURCE_BYTES + 1)], { + type: "image/jpeg", + }), + ); + const { close } = stubCanvasPipeline(() => 200_000); + + const result = await prepareImageForAttachment(original, 1_000_000); + + expect(result.ok && result.file.name).toBe("large.jpg"); + expect(result.ok && result.file.type).toBe("image/jpeg"); + expect(result.ok && result.file.size).toBeLessThanOrEqual(1_000_000); + expect(close).toHaveBeenCalled(); + }); + it("reports unreadable when HEIC decoding fails", async () => { const original = new File([new Uint8Array([1, 2, 3])], "broken.heic", { type: "image/heic", diff --git a/apps/web/src/lib/imageCompression.ts b/apps/web/src/lib/imageCompression.ts index 453e6c6e0bd2..2fc701e72aca 100644 --- a/apps/web/src/lib/imageCompression.ts +++ b/apps/web/src/lib/imageCompression.ts @@ -316,17 +316,19 @@ export async function compressImageForStash( * `File` (WebP or JPEG). Files already within the limit pass through * untouched, preserving their exact bytes and format. Sources above * `MAX_COMPRESSIBLE_SOURCE_BYTES` are refused outright — decoding them is - * the risk, so no amount of output budget makes them safe. + * the risk, so no amount of output budget makes them safe. An internally + * converted image can provide its original source size when the intermediate + * format expands beyond that ceiling. */ export async function compressImageToByteLimit( file: File, maxBytes: number, - options?: { preferredMimeType?: "image/jpeg" }, + options?: { preferredMimeType?: "image/jpeg"; sourceSizeBytes?: number }, ): Promise { if (file.size <= maxBytes) { return { ok: true, file, recompressed: false }; } - if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) { + if ((options?.sourceSizeBytes ?? file.size) > MAX_COMPRESSIBLE_SOURCE_BYTES) { return { ok: false, reason: "too-large" }; } // The re-encode loop budgets in data-URL characters. Base64 turns 3 bytes @@ -378,6 +380,7 @@ export async function prepareImageForAttachment( }); const result = await compressImageToByteLimit(jpeg, maxBytes, { preferredMimeType: "image/jpeg", + sourceSizeBytes: file.size, }); return result.ok ? { ...result, recompressed: true } : result;