feat(web): convert HEIC photo attachments to JPEG

apps/web/package.json

@@ -34,6 +34,7 @@
     "class-variance-authority": "^0.7.1",
     "culori": "^4.0.2",
     "effect": "catalog:",
+    "heic-to": "^1.5.2",
     "jose": "catalog:",
     "jsonc-parser": "3.3.1",
     "jszip": "3.10.1",

apps/web/src/components/chat/ChatComposer.tsx

@@ -73,7 +73,11 @@ import {
   type ComposerTaskStep,
   type ComposerTasksProgress,
 } from "./ComposerTasksBadge";
-import { compressImageForStash, compressImageToByteLimit } from "../../lib/imageCompression";
+import {
+  compressImageForStash,
+  isHeicImageFile,
+  prepareImageForAttachment,
+} from "../../lib/imageCompression";
 import {
   releaseAttachmentUpload,
   retryAttachmentUpload,
@@ -2537,12 +2541,13 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
     const acceptedFiles: File[] = [];
     let error: string | null = null;
     for (const file of files) {
-      if (!file.type.startsWith("image/")) {
+      const isHeicImage = isHeicImageFile(file);
+      if (!file.type.startsWith("image/") && !isHeicImage) {
         error = `Unsupported file type for '${file.name}'. Please attach image files only.`;
         continue;
       }
-      if (!isProviderSendTurnSupportedImageMimeType(file.type)) {
-        error = `'${file.name}' is not a supported image type. Attach GIF, JPEG, PNG, or WebP images.`;
+      if (!isHeicImage && !isProviderSendTurnSupportedImageMimeType(file.type)) {
+        error = `'${file.name}' is not a supported image type. Attach GIF, HEIC, HEIF, JPEG, PNG, or WebP images.`;
         continue;
       }
       if (reservedCount >= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) {
@@ -2562,7 +2567,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
       for (const file of acceptedFiles) {
         // Images over the wire cap are downscaled to fit rather than
         // refused; files already within it pass through byte-for-byte.
-        const compressed = await compressImageToByteLimit(file, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES);
+        const compressed = await prepareImageForAttachment(
+          file,
+          PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
+        );
         if (!compressed.ok) {
           compressionError =
             compressed.reason === "unreadable"
@@ -2615,7 +2623,9 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
   const onComposerPaste = (event: React.ClipboardEvent<HTMLElement>) => {
     const files = Array.from(event.clipboardData.files);
     if (files.length === 0) return;
-    const imageFiles = files.filter((file) => file.type.startsWith("image/"));
+    const imageFiles = files.filter(
+      (file) => file.type.startsWith("image/") || isHeicImageFile(file),
+    );
     if (imageFiles.length === 0) return;
     event.preventDefault();
     void addComposerImages(imageFiles);

apps/web/src/lib/imageCompression.test.ts

@@ -3,10 +3,20 @@ import { afterEach, describe, expect, it, vi } from "vite-plus/test";
 import {
   compressImageForStash,
   compressImageToByteLimit,
+  isHeicImageFile,
   MAX_COMPRESSIBLE_SOURCE_BYTES,
   MAX_STASH_IMAGE_DATA_URL_CHARS,
+  prepareImageForAttachment,
 } from "./imageCompression";
 
+const mocks = vi.hoisted(() => ({
+  heicTo: vi.fn(),
+}));
+
+vi.mock("heic-to/csp", () => ({
+  heicTo: mocks.heicTo,
+}));
+
 /**
  * jsdom has no real canvas/codec, so the re-encode path is exercised with
  * stubbed `createImageBitmap` + `OffscreenCanvas`. The encoder stub returns a
@@ -63,6 +73,7 @@ function stubCanvasPipeline(
 }
 
 afterEach(() => {
+  mocks.heicTo.mockReset();
   vi.unstubAllGlobals();
   globalThis.createImageBitmap = originalCreateImageBitmap;
   globalThis.OffscreenCanvas = originalOffscreenCanvas;
@@ -251,3 +262,104 @@ describe("compressImageForStash", () => {
     expect(smallestRequested).toBeLessThan(800);
   });
 });
+
+describe("HEIC attachment preparation", () => {
+  it("recognizes HEIC and HEIF MIME types and case-insensitive file extensions", () => {
+    expect(isHeicImageFile({ name: "photo.bin", type: "image/heic" })).toBe(true);
+    expect(isHeicImageFile({ name: "photo.bin", type: "image/heif-sequence" })).toBe(true);
+    expect(isHeicImageFile({ name: "IMG_1234.HEIC", type: "" })).toBe(true);
+    expect(isHeicImageFile({ name: "photo.heif", type: "application/octet-stream" })).toBe(true);
+    expect(isHeicImageFile({ name: "photo.png", type: "image/png" })).toBe(false);
+  });
+
+  it("converts a HEIC photo with a missing MIME type into a named JPEG", async () => {
+    const original = new File([new Uint8Array([1, 2, 3])], "IMG_1234.HEIC", {
+      lastModified: 123,
+    });
+    mocks.heicTo.mockResolvedValueOnce(
+      new Blob([new Uint8Array([4, 5, 6, 7])], { type: "image/jpeg" }),
+    );
+
+    const result = await prepareImageForAttachment(original, 1024);
+
+    expect(mocks.heicTo).toHaveBeenCalledWith({
+      blob: original,
+      type: "image/jpeg",
+      quality: 0.92,
+    });
+    expect(result.ok && result.file.name).toBe("IMG_1234.jpg");
+    expect(result.ok && result.file.type).toBe("image/jpeg");
+    expect(result.ok && result.file.size).toBe(4);
+    expect(result.ok && result.file.lastModified).toBe(123);
+    expect(result.ok && result.recompressed).toBe(true);
+  });
+
+  it("keeps oversized converted photos in JPEG format while shrinking them", async () => {
+    const original = new File([new Uint8Array([1, 2, 3])], "photo.heif", {
+      type: "image/heif",
+    });
+    mocks.heicTo.mockResolvedValueOnce(
+      new Blob([new Uint8Array(2_000_000)], { type: "image/jpeg" }),
+    );
+    const { fillRect } = stubCanvasPipeline(() => 200_000);
+
+    const result = await prepareImageForAttachment(original, 1_000_000);
+
+    expect(result.ok && result.file.name).toBe("photo.jpg");
+    expect(result.ok && result.file.type).toBe("image/jpeg");
+    expect(result.ok && result.file.size).toBeLessThanOrEqual(1_000_000);
+    expect(fillRect).toHaveBeenCalled();
+  });
+
+  it("compresses JPEG intermediates above the source safety ceiling", async () => {
+    const original = new File([new Uint8Array([1, 2, 3])], "large.heic", {
+      type: "image/heic",
+    });
+    mocks.heicTo.mockResolvedValueOnce(
+      new Blob([new Uint8Array(MAX_COMPRESSIBLE_SOURCE_BYTES + 1)], {
+        type: "image/jpeg",
+      }),
+    );
+    const { close } = stubCanvasPipeline(() => 200_000);
+
+    const result = await prepareImageForAttachment(original, 1_000_000);
+
+    expect(result.ok && result.file.name).toBe("large.jpg");
+    expect(result.ok && result.file.type).toBe("image/jpeg");
+    expect(result.ok && result.file.size).toBeLessThanOrEqual(1_000_000);
+    expect(close).toHaveBeenCalled();
+  });
+
+  it("reports unreadable when HEIC decoding fails", async () => {
+    const original = new File([new Uint8Array([1, 2, 3])], "broken.heic", {
+      type: "image/heic",
+    });
+    mocks.heicTo.mockRejectedValueOnce(new Error("Invalid HEIC image"));
+
+    expect(await prepareImageForAttachment(original, 1024)).toEqual({
+      ok: false,
+      reason: "unreadable",
+    });
+  });
+
+  it("rejects unsafe HEIC sources before loading the decoder", async () => {
+    const original = new File(["photo"], "large.heic", { type: "image/heic" });
+    Object.defineProperty(original, "size", { value: MAX_COMPRESSIBLE_SOURCE_BYTES + 1 });
+
+    expect(await prepareImageForAttachment(original, 1024)).toEqual({
+      ok: false,
+      reason: "too-large",
+    });
+    expect(mocks.heicTo).not.toHaveBeenCalled();
+  });
+
+  it("leaves supported images untouched without loading the HEIC decoder", async () => {
+    const original = makeFile(1024);
+
+    const result = await prepareImageForAttachment(original, 2048);
+
+    expect(result.ok && result.file).toBe(original);
+    expect(result.ok && result.recompressed).toBe(false);
+    expect(mocks.heicTo).not.toHaveBeenCalled();
+  });
+});

apps/web/src/lib/imageCompression.ts

@@ -8,7 +8,8 @@
  *   `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit
  *   via `compressImageToByteLimit` instead of rejecting the paste.
  *
- * Images already within budget pass through untouched.
+ * Supported images already within budget pass through untouched. HEIC/HEIF
+ * photos are decoded to JPEG first because providers cannot consume them.
  */
 
 /**
@@ -32,6 +33,8 @@ export const MAX_COMPRESSIBLE_SOURCE_BYTES = 50 * 1024 * 1024;
 const QUALITY_STEPS = [0.92, 0.85, 0.78, 0.68] as const;
 /** Extra downscale passes applied when even the lowest quality overflows. */
 const FALLBACK_SCALE_STEPS = [0.75, 0.55] as const;
+const HEIC_IMAGE_MIME_TYPE = /^image\/hei(?:c|f)(?:-sequence)?$/i;
+const HEIC_IMAGE_EXTENSION = /\.(?:heic|heif)$/i;
 
 export interface CompressedStashImage {
   dataUrl: string;
@@ -55,6 +58,11 @@ export type CompressImageFileResult =
   | { ok: true; file: File; recompressed: boolean }
   | { ok: false; reason: ImageCompressionFailureReason };
 
+/** Finder and some browsers omit the MIME type when dragging HEIC photos. */
+export function isHeicImageFile(file: Pick<File, "name" | "type">): boolean {
+  return HEIC_IMAGE_MIME_TYPE.test(file.type) || HEIC_IMAGE_EXTENSION.test(file.name);
+}
+
 /** Chunked so a large image can't blow the argument limit of `fromCharCode`. */
 const BASE64_CHUNK_SIZE = 0x8000;
 
@@ -167,6 +175,7 @@ async function encodeWithinBudget(
   bitmap: ImageBitmap,
   maxDimension: number,
   budgetChars: number,
+  preferredMimeType?: "image/jpeg",
 ): Promise<{ dataUrl: string; mimeType: string } | null> {
   const scale = Math.min(1, maxDimension / Math.max(bitmap.width, bitmap.height));
   const width = Math.max(1, Math.round(bitmap.width * scale));
@@ -176,8 +185,11 @@ async function encodeWithinBudget(
 
   // Probe WebP once; JPEG (no alpha) needs a white matte, so the fill has to
   // happen before drawing and depends on which codec we end up using.
-  const probe = await encodeCanvas(target.canvas, QUALITY_STEPS[0], "image/webp", 0);
-  const mimeType = probe ? "image/webp" : "image/jpeg";
+  const mimeType =
+    preferredMimeType ??
+    ((await encodeCanvas(target.canvas, QUALITY_STEPS[0], "image/webp", 0))
+      ? "image/webp"
+      : "image/jpeg");
 
   if (mimeType === "image/jpeg") {
     target.context.fillStyle = "#ffffff";
@@ -203,7 +215,11 @@ type ReencodeResult =
  * Shared re-encode loop: decodes `file`, then walks the quality ladder and
  * fallback downscale passes until an encoding fits `budgetChars`.
  */
-async function reencodeWithinBudget(file: File, budgetChars: number): Promise<ReencodeResult> {
+async function reencodeWithinBudget(
+  file: File,
+  budgetChars: number,
+  preferredMimeType?: "image/jpeg",
+): Promise<ReencodeResult> {
   if (!canRecompress()) {
     return { ok: false, reason: "too-large" };
   }
@@ -229,7 +245,7 @@ async function reencodeWithinBudget(file: File, budgetChars: number): Promise<Re
       const targetDimension = Math.max(1, Math.round(baseDimension * dimensionScale));
       let encoded: { dataUrl: string; mimeType: string } | null;
       try {
-        encoded = await encodeWithinBudget(bitmap, targetDimension, budgetChars);
+        encoded = await encodeWithinBudget(bitmap, targetDimension, budgetChars, preferredMimeType);
       } catch {
         // Canvas allocation, drawing, or the codec itself can throw — often
         // precisely *because* the target is too big (OOM on a large bitmap).
@@ -300,23 +316,26 @@ export async function compressImageForStash(
  * `File` (WebP or JPEG). Files already within the limit pass through
  * untouched, preserving their exact bytes and format. Sources above
  * `MAX_COMPRESSIBLE_SOURCE_BYTES` are refused outright — decoding them is
- * the risk, so no amount of output budget makes them safe.
+ * the risk, so no amount of output budget makes them safe. An internally
+ * converted image can provide its original source size when the intermediate
+ * format expands beyond that ceiling.
  */
 export async function compressImageToByteLimit(
   file: File,
   maxBytes: number,
+  options?: { preferredMimeType?: "image/jpeg"; sourceSizeBytes?: number },
 ): Promise<CompressImageFileResult> {
   if (file.size <= maxBytes) {
     return { ok: true, file, recompressed: false };
   }
-  if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) {
+  if ((options?.sourceSizeBytes ?? file.size) > MAX_COMPRESSIBLE_SOURCE_BYTES) {
     return { ok: false, reason: "too-large" };
   }
   // The re-encode loop budgets in data-URL characters. Base64 turns 3 bytes
   // into 4 chars; flooring keeps the budget a hair conservative instead of
   // admitting an encoding right at the byte cap.
   const budgetChars = Math.floor(maxBytes / 3) * 4;
-  const reencoded = await reencodeWithinBudget(file, budgetChars);
+  const reencoded = await reencodeWithinBudget(file, budgetChars, options?.preferredMimeType);
   if (!reencoded.ok) {
     return reencoded;
   }
@@ -330,3 +349,39 @@ export async function compressImageToByteLimit(
     recompressed: true,
   };
 }
+
+/**
+ * Converts HEIC/HEIF photos to provider-compatible JPEG before applying the
+ * attachment size limit. The decoder is loaded only when such a photo arrives.
+ */
+export async function prepareImageForAttachment(
+  file: File,
+  maxBytes: number,
+): Promise<CompressImageFileResult> {
+  if (!isHeicImageFile(file)) {
+    return compressImageToByteLimit(file, maxBytes);
+  }
+
+  if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) {
+    return { ok: false, reason: "too-large" };
+  }
+
+  let converted: Blob;
+  try {
+    const { heicTo } = await import("heic-to/csp");
+    converted = await heicTo({ blob: file, type: "image/jpeg", quality: QUALITY_STEPS[0] });
+  } catch {
+    return { ok: false, reason: "unreadable" };
+  }
+
+  const jpeg = new File([converted], fileNameForMimeType(file.name || "image", "image/jpeg"), {
+    type: "image/jpeg",
+    lastModified: file.lastModified,
+  });
+  const result = await compressImageToByteLimit(jpeg, maxBytes, {
+    preferredMimeType: "image/jpeg",
+    sourceSizeBytes: file.size,
+  });
+
+  return result.ok ? { ...result, recompressed: true } : result;
+}

docs/user/composer.md

@@ -7,6 +7,9 @@ multiple messages, then send again in the same thread.
 On servers that support direct uploads, images upload as soon as you add them. The send button
 becomes available after every upload finishes. Failed uploads can be retried or removed.
 
+On web and desktop, HEIC and HEIF photos are automatically converted to JPEG when you drag them into
+the composer or paste them into a message.
+
 ## Commands and skills
 
 Type `/` to open the command menu. Type `$` to find and add a skill. Skill rows show their source,