From 2ae1e0e11b2aadf8a9396afb73f5223a4cae1f62 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 3 Sep 2026 13:26:08 -0400 Subject: [PATCH 1/2] eng-1864 Define CrossApp Asset reference contract --- packages/database/src/crossAppContracts.ts | 26 +++++++++++++ .../src/crossAppNodeContract.example.ts | 37 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/packages/database/src/crossAppContracts.ts b/packages/database/src/crossAppContracts.ts index df7911ff3..00b571f6a 100644 --- a/packages/database/src/crossAppContracts.ts +++ b/packages/database/src/crossAppContracts.ts @@ -74,6 +74,28 @@ type InlineCrossAppTypedContent = InlineCrossAppContent & { contentType: ContentType; }; +// An asset (an image or attachment) that a node's full content references. +export type CrossAppAsset = { + // What the node's full content refers to, exactly as it wrote it: a path on a + // platform that addresses assets by path, a URL on one that addresses them by URL. + // Publication never rewrites it, so this is what a destination matches on, and it + // is unique within one node's `assets`: it maps to `FileReference.filepath`, which + // is part of that table's primary key. + sourceRef: string; + // The SHA-256 of the stored bytes as 64 lowercase hex characters, which is also + // their object name in shared storage. A destination looks the bytes up by this + // string exactly, so any other encoding fails as a not-found rather than a type error. + // Required: an asset whose bytes were not stored is represented by its absence + // from `assets`, not by an entry with nothing to resolve. Its `sourceRef` stays in + // the content, and the failure is reported by whichever transfer hit it. + contentHash: string; + // Where the source kept the asset: a name on a platform with a flat asset namespace, + // a path on one with folders. A destination derives the name and placement of its + // local copy by decomposing this as a path, and a bare name decomposes to itself. + // Absent when the source recorded nothing beyond `sourceRef`. + sourcePath?: string; +}; + // A node instance export type CrossAppNode = CrossAppBase & { nodeType: LocalId; @@ -86,6 +108,10 @@ export type CrossAppNode = CrossAppBase & { direct: InlineCrossAppContent; full?: InlineCrossAppTypedContent; }; + // The assets referenced by `content.full` whose bytes are stored. An asset that + // could not be stored is not listed: a destination leaves its token untouched, + // which is the same thing it does for a link that was never an asset. + assets?: CrossAppAsset[]; }; // A relation instance diff --git a/packages/database/src/crossAppNodeContract.example.ts b/packages/database/src/crossAppNodeContract.example.ts index e32ec31d2..31f5b1a5e 100644 --- a/packages/database/src/crossAppNodeContract.example.ts +++ b/packages/database/src/crossAppNodeContract.example.ts @@ -4,11 +4,25 @@ import type { CrossAppNode } from "./crossAppContracts"; // const ROAM_SOURCE_SPACE_ID = "https://roamresearch.com/#/app/MAPLab"; const ROAM_SOURCE_NODE_ID = "tgWb6JozF"; +// Roam addresses assets by URL. Both of these are the tokens the source page holds, +// and publication leaves them exactly as they are. +const ROAM_STORED_ASSET_URL = + "https://firebasestorage.googleapis.com/v0/b/firescript-577a2.appspot.com/o/imgs%2Fapp%2FMAPLab%2FlqP2ioVNC3.png?alt=media&token=9f1c07a4-2b3e-4c5d-8a91-6e0f2d7b4c13"; +// Bytes that could not be copied at publication. It is embedded in the markdown below +// with Roam's own PDF syntax — so a destination scanning for embeds does find it — but +// is deliberately absent from `assets`: an unresolvable asset has no recorded reference +// at all, so the destination finds no entry for this token and leaves it in place. +const ROAM_UNRESOLVABLE_ASSET_URL = + "https://firebasestorage.googleapis.com/v0/b/firescript-577a2.appspot.com/o/imgs%2Fapp%2FMAPLab%2FGVfB6XBcMR.pdf?alt=media&token=3a5d81b6-7c24-4e19-b0f8-52ca9e3d1f07"; + const roamFullMarkdown = `# Sleep improves memory consolidation Multiple studies show that sleep after learning strengthens memory traces. +![](${ROAM_STORED_ASSET_URL}) + - Supported by [[EVD]] - Rasch & Born 2013 +- Protocol: {{[[pdf]]: ${ROAM_UNRESOLVABLE_ASSET_URL}}} `; export const roamOriginNodeExample: CrossAppNode = { @@ -26,6 +40,16 @@ export const roamOriginNodeExample: CrossAppNode = { authorId: "someone", }, }, + assets: [ + { + sourceRef: ROAM_STORED_ASSET_URL, + contentHash: + "e030fe745078ef6ea92f5cf4f65a0d93755ba9abe1bb53653da5f4b7cdb91a57", + // Roam keeps the uploaded name in Firebase custom metadata; the publisher reads + // it from there, because the URL itself carries only a random storage uid. + sourcePath: "CleanShot 2025-11-16 at 17.14.44@2x.png", + }, + ], createdAt: new Date("2026-06-12T14:00:00.000Z"), modifiedAt: new Date("2026-06-12T15:00:00.000Z"), authorId: "maparent", @@ -34,6 +58,8 @@ export const roamOriginNodeExample: CrossAppNode = { // const OBSIDIAN_SOURCE_SPACE_ID = "obsidian:9a8b7c6d5e4f3210"; const OBSIDIAN_SOURCE_NODE_ID = "0192f1a0-7b3c-7e2a-9f10-1a2b3c4d5e6f"; const OBSIDIAN_SOURCE_NODE_TYPE_ID = "evd-7c1f9a2b"; +// Obsidian addresses assets by vault path, and publication carries the path unchanged. +const OBSIDIAN_ASSET_PATH = "attachments/rem-sleep-recall.png"; const obsidianFullMarkdown = `--- nodeTypeId: ${OBSIDIAN_SOURCE_NODE_TYPE_ID} @@ -43,6 +69,8 @@ nodeInstanceId: ${OBSIDIAN_SOURCE_NODE_ID} # REM sleep correlates with recall Participants with more REM sleep showed better next-day recall. + +![[${OBSIDIAN_ASSET_PATH}]] `; export const obsidianOriginNodeExample: CrossAppNode = { @@ -60,6 +88,15 @@ export const obsidianOriginNodeExample: CrossAppNode = { authorId: "someone", }, }, + assets: [ + { + sourceRef: OBSIDIAN_ASSET_PATH, + contentHash: + "b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78", + // No sourcePath: an Obsidian sourceRef is already a vault path, which a + // destination can decompose on its own. + }, + ], createdAt: new Date("2026-06-14T10:30:00.000Z"), modifiedAt: new Date("2026-06-14T15:00:00.000Z"), authorId: "maparent", From a243f7cabf9f48bbf34ecb387f8b460b46cfce0c Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 3 Sep 2026 14:34:16 -0400 Subject: [PATCH 2/2] Add assetLimit constants --- .../src/utils/syncDgNodesToSupabase.ts | 11 ++++- .../src/lib/__tests__/assetLimits.test.ts | 37 +++++++++++++++ packages/database/src/lib/assetLimits.ts | 47 +++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 packages/database/src/lib/__tests__/assetLimits.test.ts create mode 100644 packages/database/src/lib/assetLimits.ts diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index 8d5de8f0f..d2b484716 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -1,5 +1,9 @@ import { Notice, TFile } from "obsidian"; import { addFile } from "@repo/database/lib/files"; +import { + MAX_PUBLISHED_ASSET_BYTES, + isAssetTooLarge, +} from "@repo/database/lib/assetLimits"; import mime from "mime-types"; import { ensureNodeInstanceId } from "~/utils/nodeInstanceId"; import type { DGSupabaseClient } from "@repo/database/lib/client"; @@ -718,7 +722,12 @@ export const syncPublishedNodeAssets = async ({ const mimetype = mime.lookup(attachment.path) || "application/octet-stream"; if (mimetype.startsWith("text/")) continue; // Do not use standard upload for large files - if (attachment.stat.size >= 6 * 1024 * 1024) { + if ( + isAssetTooLarge({ + size: attachment.stat.size, + limit: MAX_PUBLISHED_ASSET_BYTES, + }) + ) { new Notice( `Asset file ${attachment.path} is larger than 6Mb and will not be uploaded`, ); diff --git a/packages/database/src/lib/__tests__/assetLimits.test.ts b/packages/database/src/lib/__tests__/assetLimits.test.ts new file mode 100644 index 000000000..becadc122 --- /dev/null +++ b/packages/database/src/lib/__tests__/assetLimits.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { + MAX_IMPORTED_ASSET_BYTES, + MAX_PUBLISHED_ASSET_BYTES, + isAssetTooLarge, +} from "../assetLimits"; + +const MIB = 1024 * 1024; + +describe("asset size caps", () => { + it("holds each direction as its own named constant", () => { + expect(MAX_PUBLISHED_ASSET_BYTES).toBe(6 * MIB); + expect(MAX_IMPORTED_ASSET_BYTES).toBe(6 * MIB); + }); +}); + +describe.each([ + ["publish", MAX_PUBLISHED_ASSET_BYTES], + ["import", MAX_IMPORTED_ASSET_BYTES], +])("the %s cap", (_direction, limit) => { + it("skips an asset above it, reporting rather than throwing", () => { + let verdict: boolean | undefined; + expect(() => { + verdict = isAssetTooLarge({ size: limit + 1, limit }); + }).not.toThrow(); + expect(verdict).toBe(true); + }); + + it("skips an asset exactly at it", () => { + expect(isAssetTooLarge({ size: limit, limit })).toBe(true); + }); + + it("transfers an asset below it", () => { + expect(isAssetTooLarge({ size: limit - 1, limit })).toBe(false); + expect(isAssetTooLarge({ size: 0, limit })).toBe(false); + }); +}); diff --git a/packages/database/src/lib/assetLimits.ts b/packages/database/src/lib/assetLimits.ts new file mode 100644 index 000000000..c2fdb1fee --- /dev/null +++ b/packages/database/src/lib/assetLimits.ts @@ -0,0 +1,47 @@ +/** + * Size caps for moving asset bytes across the shared-storage boundary. + * + * The two directions are separate constants on purpose. They happen to hold the same + * number today, but they answer different questions and are expected to diverge: the + * publish cap is about what we will store and pay for, the import cap about what a + * browser should shuttle and what it costs a user's quota on the destination platform. + * Neither may be derived from the other, and neither may be imported from a caller's + * own limit. + */ + +/** + * The largest asset, in bytes, that a platform will copy into shared storage when + * publishing a node. + * + * 6 MiB is Supabase's threshold for a standard upload; above it an uploader is expected + * to switch to a resumable one, which `addFile` does not implement. It is deliberately + * not read from the bucket's own `file_size_limit` (50 MiB), because the constraint is + * the upload method rather than what the bucket would accept. + */ +export const MAX_PUBLISHED_ASSET_BYTES = 6 * 1024 * 1024; + +/** + * The largest asset, in bytes, that a destination will copy out of shared storage into + * its own storage when importing a node. + * + * Same number as the publish cap for now, so there is one figure to reason about, but + * held separately because the constraint is different: the destination client pulls the + * bytes down and pushes them back up, in the browser, once per importing graph, spending + * that user's storage quota. The destination platform's own ceiling is unknown to us. + */ +export const MAX_IMPORTED_ASSET_BYTES = 6 * 1024 * 1024; + +/** + * Whether an asset is too large to transfer under the given cap. + * + * Returns a verdict rather than throwing: an oversized asset is a skip, and the node it + * belongs to still publishes or imports with its content intact. The boundary is + * inclusive, so an asset exactly at the cap is skipped, matching the guard this replaced. + */ +export const isAssetTooLarge = ({ + size, + limit, +}: { + size: number; + limit: number; +}): boolean => size >= limit;