diff --git a/apps/roam/src/utils/__tests__/mirrorAssetToRoamStorage.test.ts b/apps/roam/src/utils/__tests__/mirrorAssetToRoamStorage.test.ts new file mode 100644 index 000000000..6e607ea1c --- /dev/null +++ b/apps/roam/src/utils/__tests__/mirrorAssetToRoamStorage.test.ts @@ -0,0 +1,344 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import { MAX_IMPORTED_ASSET_BYTES } from "@repo/database/lib/assetLimits"; +import { + extractUploadedUrl, + mirrorAssetToRoamStorage, + mirroredAssetFileName, +} from "../mirrorAssetToRoamStorage"; +import { readMirroredAssetUrl, recordMirroredAsset } from "../assetRegistry"; + +vi.mock("../assetRegistry", () => ({ + readMirroredAssetUrl: vi.fn(), + recordMirroredAsset: vi.fn(), +})); + +const HASH = "a".repeat(64); +const UPLOADED_URL = + "https://firebasestorage.googleapis.com/v0/b/firescript-577a2.appspot.com/o/imgs%2Fapp%2FMAPLab%2FZr4mWpN70c.png?alt=media&token=9f1c07a4"; + +/** + * A stand-in for the registry that behaves like graph props: what is recorded is what a + * later read returns, so a second call for the same hash sees the first call's upload. + */ +const useInMemoryRegistry = () => { + const entries = new Map(); + vi.mocked(readMirroredAssetUrl).mockImplementation((hash) => + entries.get(hash), + ); + vi.mocked(recordMirroredAsset).mockImplementation(({ contentHash, url }) => { + entries.set(contentHash, url); + return Promise.resolve(); + }); + return entries; +}; + +const makeClient = ({ + size = 1024, + contentType = "image/png", +}: { size?: number; contentType?: string } = {}) => { + const blob = new Blob([new Uint8Array(size)], { type: contentType }); + const info = vi.fn().mockResolvedValue({ + data: { size, contentType }, + error: null, + }); + const download = vi.fn().mockResolvedValue({ data: blob, error: null }); + const client = { + storage: { from: vi.fn(() => ({ info, download })) }, + } as unknown as DGSupabaseClient; + return { client, info, download }; +}; + +/** The suite runs in node, so `window` is stubbed the way the other Roam tests stub it. */ +const setRoamUpload = (upload: ReturnType): void => { + (globalThis as { window: unknown }).window = { + roamAlphaAPI: { file: { upload } }, + }; +}; + +const mockUpload = (returnValue: string) => { + const upload = vi.fn().mockResolvedValue(returnValue); + setRoamUpload(upload); + return upload; +}; + +beforeEach(() => { + useInMemoryRegistry(); +}); + +afterEach(() => { + vi.clearAllMocks(); +}); + +describe("extractUploadedUrl", () => { + // `file.upload` returns a media-type-dependent block string, not a URL. + it.each([ + ["an image", `![](${UPLOADED_URL})`], + ["an image with alt text", `![diagram](${UPLOADED_URL})`], + ["a pdf", `{{[[pdf]]: ${UPLOADED_URL}}}`], + ["audio", `{{[[audio]]: ${UPLOADED_URL}}}`], + ["video", `{{[[video]]: ${UPLOADED_URL}}}`], + ["anything else, as a bare URL", UPLOADED_URL], + ["a media embed written without brackets", `{{pdf: ${UPLOADED_URL}}}`], + ["a labelled link", `[report.docx](${UPLOADED_URL})`], + ])("unwraps %s", (_label, uploadReturn) => { + expect(extractUploadedUrl(uploadReturn)).toBe(UPLOADED_URL); + }); + + // Every branch requires the scheme. Recording a non-URL would poison the registry for + // that hash permanently, since nothing revisits an entry once written. + it.each([ + ["an image embed", "![](not-a-url)"], + ["a media embed", "{{[[pdf]]: not-a-url}}"], + ["a labelled link", "[report.docx](../attachments/report.docx)"], + ["a bare token", "not-a-url"], + ])("refuses %s carrying no scheme", (_label, uploadReturn) => { + expect(extractUploadedUrl(uploadReturn)).toBeUndefined(); + }); + + it("reports a shape it cannot read rather than guessing a URL", () => { + expect( + extractUploadedUrl("{{[[roam/render]]: something}}"), + ).toBeUndefined(); + expect(extractUploadedUrl("")).toBeUndefined(); + // A URL embedded in prose is not an upload result, so it is not taken. + expect( + extractUploadedUrl(`see ${UPLOADED_URL} for details`), + ).toBeUndefined(); + }); +}); + +describe("mirroredAssetFileName", () => { + it("carries the hash, so a lost registry can be rebuilt from the file's own name", () => { + expect( + mirroredAssetFileName({ contentHash: HASH, sourcePath: "diagram.png" }), + ).toBe(`imported-${HASH}.png`); + }); + + it("falls back to the mime subtype when the recorded name has no extension", () => { + expect( + mirroredAssetFileName({ + contentHash: HASH, + sourcePath: "scan", + mimetype: "application/pdf", + }), + ).toBe(`imported-${HASH}.pdf`); + }); + + it("takes the extension from a recorded name that is a path", () => { + expect( + mirroredAssetFileName({ + contentHash: HASH, + sourcePath: "attachments/notes/report.docx", + }), + ).toBe(`imported-${HASH}.docx`); + }); + + it("ignores a dot in the stem, which is not an extension", () => { + expect( + mirroredAssetFileName({ + contentHash: HASH, + sourcePath: "notes/2024.06.01 meeting", + mimetype: "application/pdf", + }), + ).toBe(`imported-${HASH}.pdf`); + }); + + it("invents no extension when neither source has one", () => { + expect(mirroredAssetFileName({ contentHash: HASH })).toBe( + `imported-${HASH}`, + ); + expect( + mirroredAssetFileName({ contentHash: HASH, mimetype: "image/svg+xml" }), + ).toBe(`imported-${HASH}.svg`); + }); +}); + +describe("mirrorAssetToRoamStorage", () => { + it("uploads the bytes and records the URL against the hash", async () => { + const { client } = makeClient(); + const upload = mockUpload(`![](${UPLOADED_URL})`); + + const result = await mirrorAssetToRoamStorage({ + client, + contentHash: HASH, + sourcePath: "diagram.png", + }); + + expect(result).toEqual({ + status: "mirrored", + contentHash: HASH, + url: UPLOADED_URL, + }); + expect(upload).toHaveBeenCalledTimes(1); + const uploaded = upload.mock.calls[0]?.[0] as { file: File }; + expect(uploaded.file.name).toBe(`imported-${HASH}.png`); + expect(recordMirroredAsset).toHaveBeenCalledWith({ + contentHash: HASH, + url: UPLOADED_URL, + }); + }); + + it("reports a copy that succeeded even when the registry write fails", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { client } = makeClient(); + mockUpload(`![](${UPLOADED_URL})`); + vi.mocked(recordMirroredAsset).mockRejectedValue( + new Error("block props write refused"), + ); + + // The bytes are in this graph's storage and the URL is in hand. Failing here would + // leave the page on the published token and re-upload the same file next run. + await expect( + mirrorAssetToRoamStorage({ client, contentHash: HASH }), + ).resolves.toEqual({ + status: "mirrored", + contentHash: HASH, + url: UPLOADED_URL, + }); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("uploaded again"), + ); + }); + + it("hides Roam's per-upload toast, since one import mirrors many files", async () => { + const { client } = makeClient(); + const upload = mockUpload(`![](${UPLOADED_URL})`); + + await mirrorAssetToRoamStorage({ client, contentHash: HASH }); + + expect(upload).toHaveBeenCalledWith( + expect.objectContaining({ toast: { hide: true } }), + ); + }); + + it("mirrors on when the metadata cannot be read, since the cap is enforced after the download", async () => { + const { client, info, download } = makeClient(); + info.mockResolvedValue({ + data: null, + error: { message: "object info unavailable" }, + }); + const upload = mockUpload(`![](${UPLOADED_URL})`); + + const result = await mirrorAssetToRoamStorage({ + client, + contentHash: HASH, + }); + + expect(download).toHaveBeenCalledTimes(1); + expect(upload).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ status: "mirrored", url: UPLOADED_URL }); + }); + + it("still refuses an over-cap asset when the metadata could not be read", async () => { + const { client, info } = makeClient({ size: MAX_IMPORTED_ASSET_BYTES + 1 }); + info.mockResolvedValue({ data: null, error: { message: "unavailable" } }); + const upload = mockUpload(`![](${UPLOADED_URL})`); + + const result = await mirrorAssetToRoamStorage({ + client, + contentHash: HASH, + }); + + expect(result).toMatchObject({ status: "skipped", reason: "too-large" }); + expect(upload).not.toHaveBeenCalled(); + }); + + it("uploads once for the same hash, however many nodes reference it", async () => { + const { client, download } = makeClient(); + const upload = mockUpload(`![](${UPLOADED_URL})`); + + const first = await mirrorAssetToRoamStorage({ client, contentHash: HASH }); + const second = await mirrorAssetToRoamStorage({ + client, + contentHash: HASH, + }); + + expect(first.status).toBe("mirrored"); + expect(second).toEqual({ + status: "reused", + contentHash: HASH, + url: UPLOADED_URL, + }); + expect(upload).toHaveBeenCalledTimes(1); + // The reused branch costs no network at all, not merely no upload. + expect(download).toHaveBeenCalledTimes(1); + }); + + it("records the URL only after the upload returns", async () => { + const { client } = makeClient(); + const order: string[] = []; + const upload = vi.fn().mockImplementation(() => { + order.push("upload"); + return Promise.resolve(`![](${UPLOADED_URL})`); + }); + setRoamUpload(upload); + vi.mocked(recordMirroredAsset).mockImplementation(() => { + order.push("record"); + return Promise.resolve(); + }); + + await mirrorAssetToRoamStorage({ client, contentHash: HASH }); + + expect(order).toEqual(["upload", "record"]); + }); + + it("skips an asset at or above the cap, reading the size before downloading it", async () => { + const { client, download } = makeClient({ size: MAX_IMPORTED_ASSET_BYTES }); + const upload = mockUpload(`![](${UPLOADED_URL})`); + + const result = await mirrorAssetToRoamStorage({ + client, + contentHash: HASH, + }); + + expect(result).toEqual({ + status: "skipped", + contentHash: HASH, + reason: "too-large", + size: MAX_IMPORTED_ASSET_BYTES, + limit: MAX_IMPORTED_ASSET_BYTES, + }); + // The cap exists to keep these bytes off the wire, so the skip has to precede both. + expect(download).not.toHaveBeenCalled(); + expect(upload).not.toHaveBeenCalled(); + expect(recordMirroredAsset).not.toHaveBeenCalled(); + }); + + it("skips an oversized asset whose metadata omits its size", async () => { + const { client, info } = makeClient({ size: MAX_IMPORTED_ASSET_BYTES + 1 }); + const upload = mockUpload(`![](${UPLOADED_URL})`); + // Roam-origin objects predate the metadata this reads, so the size can be absent. + info.mockResolvedValue({ data: { contentType: "image/png" }, error: null }); + + const result = await mirrorAssetToRoamStorage({ + client, + contentHash: HASH, + }); + + expect(result).toMatchObject({ status: "skipped", reason: "too-large" }); + expect(upload).not.toHaveBeenCalled(); + }); + + it("reports an unreadable upload result rather than recording a wrong URL", async () => { + const { client } = makeClient(); + mockUpload("{{[[roam/render]]: something}}"); + + await expect( + mirrorAssetToRoamStorage({ client, contentHash: HASH }), + ).rejects.toThrow(/cannot read a URL from/); + expect(recordMirroredAsset).not.toHaveBeenCalled(); + }); + + it("records nothing when the bytes cannot be downloaded", async () => { + const { client, download } = makeClient(); + const upload = mockUpload(`![](${UPLOADED_URL})`); + download.mockResolvedValue({ data: null, error: new Error("not found") }); + + await expect( + mirrorAssetToRoamStorage({ client, contentHash: HASH }), + ).rejects.toThrow(); + expect(upload).not.toHaveBeenCalled(); + expect(recordMirroredAsset).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/roam/src/utils/mirrorAssetToRoamStorage.ts b/apps/roam/src/utils/mirrorAssetToRoamStorage.ts new file mode 100644 index 000000000..1c8c1f784 --- /dev/null +++ b/apps/roam/src/utils/mirrorAssetToRoamStorage.ts @@ -0,0 +1,234 @@ +import { + MAX_IMPORTED_ASSET_BYTES, + isAssetTooLarge, +} from "@repo/database/lib/assetLimits"; +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import { readMirroredAssetUrl, recordMirroredAsset } from "./assetRegistry"; + +/** + * Copies one asset out of shared storage and into this graph's own Roam storage. + * + * Roam renders an embed by issuing an anonymous cross-origin GET, with no credentials, + * so whatever an imported page points at has to be fetchable without our auth. Shared + * storage is private, so the bytes have to land somewhere public: this graph's own + * Firebase storage, through `file.upload`. + * + * Every asset takes this path, whatever platform published it. A Roam-origin asset + * imported into a second graph is copied too, rather than pointing at the origin graph's + * URL: recognising a Firebase URL in order to skip the copy would put origin detection + * back into the destination, and the shortcut would leave this graph's page depending on + * a blob the origin graph's owner can delete. + * + * The copy is irrevocable, and that is deliberate. See design.md Decision 3. + * + * **Call this one asset at a time.** What makes it idempotent is a synchronous registry + * read separated from the matching write by a download and an upload, so callers running + * it under `Promise.all` all see an empty registry for the same hash: the bytes upload + * once per call, the registry keeps one URL, and the rest are permanent orphans in the + * user's Roam storage. `assetRegistry.ts` prices in a write lost between two tabs, which + * costs one redundant upload; this is the larger cost, and the mitigation is the caller's + * sequential loop rather than a lock here. Parallelising a caller means adding an + * in-flight map of hash to promise in this module first. + */ + +const getErrorMessage = (error: unknown): string => + error instanceof Error ? error.message : String(error); + +/** The bucket `addFile` writes to, keyed by content hash. */ +const SHARED_ASSET_BUCKET = "assets"; + +export type MirroredAsset = { + /** The bytes were uploaded to this graph's storage by this call. */ + status: "mirrored"; + contentHash: string; + url: string; +}; + +export type ReusedAsset = { + /** This graph already held a copy, so nothing was uploaded. */ + status: "reused"; + contentHash: string; + url: string; +}; + +export type SkippedMirror = { + status: "skipped"; + contentHash: string; + reason: "too-large"; + size: number; + limit: number; +}; + +export type AssetMirrorResult = MirroredAsset | ReusedAsset | SkippedMirror; + +/** + * The forms `file.upload` resolves to. It returns a block string rather than a URL, and + * which one depends on the media type, so the URL has to be unwrapped before it can be + * recorded or written into a link. + * + * Anchored, and every branch requires the scheme: the input is one upload's return value + * rather than page content, so a pattern that matched loosely would take a non-URL out of + * a string this code does not recognise, and recording the wrong URL is worse than + * reporting an unknown shape. + * + * Two branches are tolerance rather than observation. `file.upload` has been seen to + * return `![](url)` for an image, the bracketed `{{[[pdf]]: url}}` family for those types, + * and a bare URL for everything else (design.md Decision 4); the bracket-less embed and + * the labelled link are shapes Roam writes elsewhere and could plausibly return. They + * stay because the trade is asymmetric, and worse than asymmetric: an unreadable return + * arrives *after* the bytes are uploaded, so it costs an orphaned file in the user's + * storage as well as an unresolved asset, and every retry orphans another. + */ +const UPLOADED_URL = String.raw`(https?://\S+?)`; + +const UPLOAD_RETURN_PATTERNS = [ + new RegExp(String.raw`^!\[[^\]]*\]\(${UPLOADED_URL}\)$`), // an image + new RegExp( + String.raw`^\{\{\[\[(?:pdf|audio|video)\]\]:\s*${UPLOADED_URL}\s*\}\}$`, + ), + new RegExp(String.raw`^\{\{(?:pdf|audio|video):\s*${UPLOADED_URL}\s*\}\}$`), + new RegExp(String.raw`^\[[^\]]*\]\(${UPLOADED_URL}\)$`), // a labelled link + new RegExp(String.raw`^${UPLOADED_URL}$`), // everything else, as a bare URL +]; + +export const extractUploadedUrl = ( + uploadReturn: string, +): string | undefined => { + const trimmed = uploadReturn.trim(); + for (const pattern of UPLOAD_RETURN_PATTERNS) { + const url = trimmed.match(pattern)?.[1]; + if (url) return url; + } + return undefined; +}; + +/** + * An extension for the uploaded file, so Roam serves it as the type it is. + * + * Taken from the recorded name where there is one, since that is what the publisher saw, + * and derived from the MIME subtype otherwise. Neither is guaranteed, and an asset with + * no extension still uploads and still renders for the types Roam sniffs. + */ +const EXTENSION_CHARACTERS = /^[a-z0-9]+$/i; + +const extensionFor = ({ + sourcePath, + mimetype, +}: { + sourcePath?: string | null; + mimetype?: string; +}): string => { + const leaf = sourcePath?.split("/").pop() ?? ""; + const dot = leaf.lastIndexOf("."); + // A dot in the stem is not an extension: `2024.06.01 meeting` would otherwise upload as + // `.06.01 meeting`. Anything that does not look like an extension falls through to the + // MIME type, which is the better guess precisely in that case. + const named = dot > 0 ? leaf.slice(dot + 1) : ""; + if (EXTENSION_CHARACTERS.test(named)) return `.${named}`; + + // `image/svg+xml` carries its useful part before the `+`; anything else non-alphanumeric + // is not an extension we should be inventing. + const subtype = mimetype?.split("/")[1]?.split("+")[0] ?? ""; + return EXTENSION_CHARACTERS.test(subtype) ? `.${subtype}` : ""; +}; + +/** + * `imported-`, which is what makes the registry a cache rather than a source of + * truth: the hash travels with the file, so a lost registry can be rebuilt by reading + * the names of the files a graph already holds. See design.md Decision 4. + */ +export const mirroredAssetFileName = ({ + contentHash, + sourcePath, + mimetype, +}: { + contentHash: string; + sourcePath?: string | null; + mimetype?: string; +}): string => + `imported-${contentHash}${extensionFor({ sourcePath, mimetype })}`; + +export const mirrorAssetToRoamStorage = async ({ + client, + contentHash, + sourcePath, +}: { + client: DGSupabaseClient; + contentHash: string; + /** The name the reference records, used only to give the upload an extension. */ + sourcePath?: string | null; +}): Promise => { + const alreadyMirrored = readMirroredAssetUrl(contentHash); + if (alreadyMirrored) + return { status: "reused", contentHash, url: alreadyMirrored }; + + const storage = client.storage.from(SHARED_ASSET_BUCKET); + + // Size is read from the object's metadata rather than from the downloaded bytes, so an + // oversized asset costs one small request instead of the download this cap exists to + // prevent. Where the metadata omits it, the check moves after the download. + // + // An info failure is metadata this object does not carry, not a reason to abort: the + // pre-check is only an optimisation, and the post-download check enforces the cap on + // its own. Where info fails for a reason that matters — a missing object, a policy that + // denies reads — the download fails next and reports it there. + const { data: info } = await storage.info(contentHash); + const tooLarge = (size: number): SkippedMirror => ({ + status: "skipped", + contentHash, + reason: "too-large", + size, + limit: MAX_IMPORTED_ASSET_BYTES, + }); + if ( + info?.size !== undefined && + isAssetTooLarge({ size: info.size, limit: MAX_IMPORTED_ASSET_BYTES }) + ) + return tooLarge(info.size); + + const { data: blob, error: downloadError } = + await storage.download(contentHash); + if (downloadError) throw downloadError; + if (!blob) + throw new Error(`No bytes in shared storage for asset ${contentHash}`); + if (isAssetTooLarge({ size: blob.size, limit: MAX_IMPORTED_ASSET_BYTES })) + return tooLarge(blob.size); + + const mimetype = info?.contentType || blob.type || undefined; + const file = new File( + [blob], + mirroredAssetFileName({ contentHash, sourcePath, mimetype }), + mimetype ? { type: mimetype } : undefined, + ); + + // Upload first, then record. A placeholder written beforehand would carry no URL, so a + // later run could neither reuse the upload nor find it, and Roam exposes no way to list + // a graph's files. A crash between the two leaves one orphaned file, which is the + // residue Roam's own documentation treats as normal. See design.md Decision 5. + // One import mirrors one file per distinct hash across every node it brings in, so the + // default per-upload toast would fire dozens of times at a user who asked for one + // import. The import reports what it did; Roam does not need to narrate each file. + const uploadReturn = await window.roamAlphaAPI.file.upload({ + file, + toast: { hide: true }, + }); + const url = extractUploadedUrl(uploadReturn); + if (!url) + throw new Error( + `Roam returned an upload result this code cannot read a URL from: ${uploadReturn}`, + ); + + // Best-effort, and never fatal. The bytes are in this graph's storage and the URL is in + // hand, so failing here would report a copy that plainly succeeded as a failure, leave + // the page on the published token, and re-upload the same file on the next run. What a + // failed write actually costs is one redundant upload later, which is the price of the + // registry being a cache rather than a source of truth. + try { + await recordMirroredAsset({ contentHash, url }); + } catch (error) { + console.warn( + `Copied an asset into this graph but could not record it in the asset registry, so it will be uploaded again on the next import: ${getErrorMessage(error)}`, + ); + } + return { status: "mirrored", contentHash, url }; +};