From 68ffafb263f5661763153de8378531f1a7787106 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 15:14:06 -0700 Subject: [PATCH 01/12] fix(files): archive rendered document bytes instead of generator source Generated docs (docx/pptx/pdf/xlsx) store their generation source as the primary file; the rendered binary lives in a separate content-addressed artifact store. resolveServableDocBytes is the shared chokepoint that swaps one for the other, and the serve route, single-file download and ~50 tool routes all go through it. The bulk zip route and file_compress read raw bytes instead, so every generated document landed in the archive as source text under a .docx name and Word reported it as corrupt. Both now resolve through the servable helper, with bounded concurrency, a size cap re-checked against the rendered bytes, and a 409 naming any document whose artifact is still compiling. --- apps/sim/app/api/tools/file/manage/route.ts | 16 +- .../[id]/files/download/route.test.ts | 147 ++++++++++++++++++ .../workspaces/[id]/files/download/route.ts | 75 ++++++++- 3 files changed, 232 insertions(+), 6 deletions(-) create mode 100644 apps/sim/app/api/workspaces/[id]/files/download/route.test.ts diff --git a/apps/sim/app/api/tools/file/manage/route.ts b/apps/sim/app/api/tools/file/manage/route.ts index 248002656ee..b38afb77735 100644 --- a/apps/sim/app/api/tools/file/manage/route.ts +++ b/apps/sim/app/api/tools/file/manage/route.ts @@ -685,9 +685,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) if (denied) return denied - const buffer = await downloadFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_COMPRESS_FILE_BYTES, - }) + // Generated docs store their generation source, not the rendered binary, so + // the archive must carry the servable bytes instead of the raw source text. + let buffer: Buffer + try { + const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_COMPRESS_FILE_BYTES, + }) + buffer = servable.buffer + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + throw error + } totalBytes += buffer.length if (totalBytes > MAX_COMPRESS_TOTAL_BYTES) { return NextResponse.json( diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts new file mode 100644 index 00000000000..0c8dd40227c --- /dev/null +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -0,0 +1,147 @@ +/** + * @vitest-environment node + */ +import { createMockRequest } from '@sim/testing' +import JSZip from 'jszip' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockGetSession, + mockVerifyWorkspaceMembership, + mockListWorkspaceFiles, + mockListWorkspaceFileFolders, + mockDownloadServableFileFromStorage, +} = vi.hoisted(() => ({ + mockGetSession: vi.fn(), + mockVerifyWorkspaceMembership: vi.fn(), + mockListWorkspaceFiles: vi.fn(), + mockListWorkspaceFileFolders: vi.fn(), + mockDownloadServableFileFromStorage: vi.fn(), +})) + +vi.mock('@/lib/auth', () => ({ + auth: { api: { getSession: vi.fn() } }, + getSession: mockGetSession, +})) + +vi.mock('@/app/api/workflows/utils', () => ({ + verifyWorkspaceMembership: mockVerifyWorkspaceMembership, +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + listWorkspaceFiles: mockListWorkspaceFiles, + listWorkspaceFileFolders: mockListWorkspaceFileFolders, + buildWorkspaceFileFolderPathMap: (folders: Array<{ id: string; name: string }>) => + new Map(folders.map((folder) => [folder.id, folder.name])), +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mockDownloadServableFileFromStorage, +})) + +vi.mock('@sim/audit', () => ({ + recordAudit: vi.fn(), + AuditAction: { FILE_DOWNLOADED: 'file.downloaded' }, + AuditResourceType: { FILE: 'file' }, +})) + +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: vi.fn() })) + +import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile' +import { GET } from '@/app/api/workspaces/[id]/files/download/route' + +const WORKSPACE_ID = 'ws-1' +const context = { params: Promise.resolve({ id: WORKSPACE_ID }) } + +function workspaceFile(id: string, name: string, folderId: string | null) { + return { + id, + name, + key: `workspace/${WORKSPACE_ID}/${id}`, + path: `/serve/${id}`, + size: 100, + type: 'application/octet-stream', + folderId, + } +} + +function requestFor(query: string) { + return createMockRequest( + 'GET', + undefined, + {}, + `http://localhost:3000/api/workspaces/${WORKSPACE_ID}/files/download?${query}` + ) +} + +describe('workspace files download route', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) + mockVerifyWorkspaceMembership.mockResolvedValue({ role: 'member' }) + mockListWorkspaceFileFolders.mockResolvedValue([ + { id: 'folder-1', name: 'Reports', parentId: null }, + ]) + }) + + it('zips the rendered bytes for a generated doc, not its stored source', async () => { + mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'overview.docx', 'folder-1')]) + // A real .docx is a ZIP; the stored source would be plain JS text. + const rendered = Buffer.from('PKrendered-docx') + mockDownloadServableFileFromStorage.mockResolvedValue({ + buffer: rendered, + contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + }) + + const response = await GET(requestFor('fileIds=f1'), context) + + expect(response.status).toBe(200) + expect(mockDownloadServableFileFromStorage).toHaveBeenCalledTimes(1) + + const zip = await JSZip.loadAsync(Buffer.from(await response.arrayBuffer())) + const entry = zip.file('Reports/overview.docx') + expect(entry).not.toBeNull() + expect(Buffer.from(await entry!.async('uint8array'))).toEqual(rendered) + }) + + it('returns 409 naming the documents whose artifacts are still compiling', async () => { + mockListWorkspaceFiles.mockResolvedValue([ + workspaceFile('f1', 'ready.md', 'folder-1'), + workspaceFile('f2', 'pending.docx', 'folder-1'), + ]) + mockDownloadServableFileFromStorage.mockImplementation(async (file: { name: string }) => { + if (file.name === 'pending.docx') + throw new DocCompileUserError('Document is still being generated') + return { buffer: Buffer.from('ok'), contentType: 'text/markdown' } + }) + + const response = await GET(requestFor('fileIds=f1&fileIds=f2'), context) + + expect(response.status).toBe(409) + const body = await response.json() + expect(body.error).toContain('pending.docx') + expect(body.error).not.toContain('ready.md') + }) + + it('rejects when rendered documents exceed the download size limit', async () => { + mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'huge.docx', 'folder-1')]) + mockDownloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.alloc(251 * 1024 * 1024), + contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + }) + + const response = await GET(requestFor('fileIds=f1'), context) + + expect(response.status).toBe(400) + expect((await response.json()).error).toContain('exceeds') + }) + + it('surfaces a storage failure as a 500 rather than zipping a placeholder', async () => { + mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'a.txt', 'folder-1')]) + mockDownloadServableFileFromStorage.mockRejectedValue(new Error('storage down')) + + const response = await GET(requestFor('fileIds=f1'), context) + + expect(response.status).toBe(500) + }) +}) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index 577f0e7b2dc..dd495979d9c 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -5,22 +5,40 @@ import { type NextRequest, NextResponse } from 'next/server' import { downloadWorkspaceFileItemsContract } from '@/lib/api/contracts/workspace-file-folders' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' +import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile' +import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { buildWorkspaceFileFolderPathMap, - fetchWorkspaceFileBuffer, listWorkspaceFileFolders, listWorkspaceFiles, } from '@/lib/uploads/contexts/workspace' import { formatFileSize } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' +import type { UserFile } from '@/executor/types' const logger = createLogger('WorkspaceFilesDownloadAPI') const MAX_ZIP_DOWNLOAD_FILES = 100 const MAX_ZIP_DOWNLOAD_BYTES = 250 * 1024 * 1024 +/** Shape `downloadServableFileFromStorage` needs to locate and label the stored object. */ +function toServableInput(file: WorkspaceFileRecord): UserFile { + return { + id: file.id, + name: file.name, + url: file.path, + size: file.size, + type: file.type, + key: file.key, + context: 'workspace', + } +} + function collectDescendantFolderIds( selectedFolderIds: string[], folders: Array<{ id: string; parentId: string | null }> @@ -92,7 +110,58 @@ export const GET = withRouteHandler( ) } - const buffers = await Promise.all(filesToZip.map((file) => fetchWorkspaceFileBuffer(file))) + // Generated docs (docx/pptx/pdf/xlsx) store their generation source, not the + // rendered binary, so the bytes must be resolved through the shared servable + // helper — a raw read ships source text under a `.docx` name. + const requestId = generateRequestId() + const downloads = await mapWithConcurrency( + filesToZip, + MATERIALIZE_CONCURRENCY, + async (file) => { + try { + const servable = await downloadServableFileFromStorage( + toServableInput(file), + requestId, + logger, + { maxBytes: MAX_ZIP_DOWNLOAD_BYTES } + ) + return { ok: true as const, buffer: servable.buffer } + } catch (error) { + return { ok: false as const, file, error } + } + } + ) + + const pending = downloads.filter( + (result) => !result.ok && result.error instanceof DocCompileUserError + ) + if (pending.length > 0) { + const names = pending.map((result) => (result.ok ? '' : result.file.name)) + return NextResponse.json( + { + error: `${pending.length} document${pending.length === 1 ? ' is' : 's are'} still being generated: ${names.join(', ')}. Wait for them to finish, then try again.`, + }, + { status: 409 } + ) + } + + // Any other failure is a real storage error: fail the request rather than + // handing back an archive with a file silently missing or empty. + const buffers = downloads.map((result) => { + if (!result.ok) throw result.error + return result.buffer + }) + + // The pre-download cap used declared source sizes; generated docs resolve larger. + const resolvedBytes = buffers.reduce((sum, buffer) => sum + buffer.length, 0) + if (resolvedBytes > MAX_ZIP_DOWNLOAD_BYTES) { + return NextResponse.json( + { + error: `Selected files total ${formatFileSize(resolvedBytes)} once documents are rendered, which exceeds the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit.`, + }, + { status: 400 } + ) + } // Entry paths stay workspace-root-relative so a mixed selection of folders and // loose files keeps the layout the user sees in the files list. @@ -116,7 +185,7 @@ export const GET = withRouteHandler( action: AuditAction.FILE_DOWNLOADED, resourceType: AuditResourceType.FILE, description: `Downloaded ${filesToZip.length} file${filesToZip.length === 1 ? '' : 's'} as zip`, - metadata: { fileCount: filesToZip.length, totalBytes }, + metadata: { fileCount: filesToZip.length, totalBytes: resolvedBytes }, request, }) captureServerEvent( From 9da5816097f5968a9082b1662b5ab150f4f36ffc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 15:28:32 -0700 Subject: [PATCH 02/12] improvement(files): centralize servable-byte reads and bound the zip byte budget Adds fetchServableWorkspaceFileBuffer beside fetchWorkspaceFileBuffer so the record -> UserFile mapping (including storageContext, which the hand-rolled adapter dropped) lives in one place, and the raw reader's doc comment says it returns generation source. The v1 file download served source bytes for the same reason and now uses it too. Download route: the rendered-byte budget is tracked as files land and each read is capped at what is left, so the request is bounded by the limit rather than by 100 files each allowed the full limit; an oversized artifact returns the 400 size response instead of a 500; a hard failure outranks a pending artifact so clients are not told to retry something that cannot succeed; and queued reads stop once the request is doomed. docNotReadyResponse now takes the pending file names so the 409 copy and body shape stay in the shared helper. Compress drops its inner catch, which the handler's own catch already covered. --- apps/sim/app/api/tools/file/manage/route.ts | 16 +- apps/sim/app/api/v1/files/[fileId]/route.ts | 8 +- .../[id]/files/download/route.test.ts | 61 +++++--- .../workspaces/[id]/files/download/route.ts | 145 +++++++++--------- .../workspace/workspace-file-manager.ts | 38 ++++- .../uploads/utils/servable-file-response.ts | 28 +++- 6 files changed, 185 insertions(+), 111 deletions(-) diff --git a/apps/sim/app/api/tools/file/manage/route.ts b/apps/sim/app/api/tools/file/manage/route.ts index b38afb77735..7bca7f82d63 100644 --- a/apps/sim/app/api/tools/file/manage/route.ts +++ b/apps/sim/app/api/tools/file/manage/route.ts @@ -687,17 +687,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => { // Generated docs store their generation source, not the rendered binary, so // the archive must carry the servable bytes instead of the raw source text. - let buffer: Buffer - try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_COMPRESS_FILE_BYTES, - }) - buffer = servable.buffer - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - throw error - } + // A still-compiling artifact throws, and the handler's catch turns that into + // the shared 409 via `docNotReadyResponse`. + const { buffer } = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_COMPRESS_FILE_BYTES, + }) totalBytes += buffer.length if (totalBytes > MAX_COMPRESS_TOTAL_BYTES) { return NextResponse.json( diff --git a/apps/sim/app/api/v1/files/[fileId]/route.ts b/apps/sim/app/api/v1/files/[fileId]/route.ts index 258e36b9ffb..c9972cc8573 100644 --- a/apps/sim/app/api/v1/files/[fileId]/route.ts +++ b/apps/sim/app/api/v1/files/[fileId]/route.ts @@ -6,7 +6,10 @@ import { parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' -import { fetchWorkspaceFileBuffer, getWorkspaceFile } from '@/lib/uploads/contexts/workspace' +import { + fetchServableWorkspaceFileBuffer, + getWorkspaceFile, +} from '@/lib/uploads/contexts/workspace' import { performDeleteWorkspaceFileItems } from '@/lib/workspace-files/orchestration' import { checkRateLimit, @@ -48,7 +51,8 @@ export const GET = withRouteHandler(async (request: NextRequest, context: FileRo return NextResponse.json({ error: 'File not found' }, { status: 404 }) } - const buffer = await fetchWorkspaceFileBuffer(fileRecord) + // Generated docs store their generation source; serve the rendered artifact. + const { buffer } = await fetchServableWorkspaceFileBuffer(fileRecord) recordAudit({ workspaceId, diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts index 0c8dd40227c..22313e588ca 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -10,13 +10,13 @@ const { mockVerifyWorkspaceMembership, mockListWorkspaceFiles, mockListWorkspaceFileFolders, - mockDownloadServableFileFromStorage, + mockFetchServableWorkspaceFileBuffer, } = vi.hoisted(() => ({ mockGetSession: vi.fn(), mockVerifyWorkspaceMembership: vi.fn(), mockListWorkspaceFiles: vi.fn(), mockListWorkspaceFileFolders: vi.fn(), - mockDownloadServableFileFromStorage: vi.fn(), + mockFetchServableWorkspaceFileBuffer: vi.fn(), })) vi.mock('@/lib/auth', () => ({ @@ -33,10 +33,7 @@ vi.mock('@/lib/uploads/contexts/workspace', () => ({ listWorkspaceFileFolders: mockListWorkspaceFileFolders, buildWorkspaceFileFolderPathMap: (folders: Array<{ id: string; name: string }>) => new Map(folders.map((folder) => [folder.id, folder.name])), -})) - -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadServableFileFromStorage: mockDownloadServableFileFromStorage, + fetchServableWorkspaceFileBuffer: mockFetchServableWorkspaceFileBuffer, })) vi.mock('@sim/audit', () => ({ @@ -48,6 +45,7 @@ vi.mock('@sim/audit', () => ({ vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: vi.fn() })) import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { GET } from '@/app/api/workspaces/[id]/files/download/route' const WORKSPACE_ID = 'ws-1' @@ -88,7 +86,7 @@ describe('workspace files download route', () => { mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'overview.docx', 'folder-1')]) // A real .docx is a ZIP; the stored source would be plain JS text. const rendered = Buffer.from('PKrendered-docx') - mockDownloadServableFileFromStorage.mockResolvedValue({ + mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ buffer: rendered, contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', }) @@ -96,7 +94,7 @@ describe('workspace files download route', () => { const response = await GET(requestFor('fileIds=f1'), context) expect(response.status).toBe(200) - expect(mockDownloadServableFileFromStorage).toHaveBeenCalledTimes(1) + expect(mockFetchServableWorkspaceFileBuffer).toHaveBeenCalledTimes(1) const zip = await JSZip.loadAsync(Buffer.from(await response.arrayBuffer())) const entry = zip.file('Reports/overview.docx') @@ -109,7 +107,7 @@ describe('workspace files download route', () => { workspaceFile('f1', 'ready.md', 'folder-1'), workspaceFile('f2', 'pending.docx', 'folder-1'), ]) - mockDownloadServableFileFromStorage.mockImplementation(async (file: { name: string }) => { + mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { if (file.name === 'pending.docx') throw new DocCompileUserError('Document is still being generated') return { buffer: Buffer.from('ok'), contentType: 'text/markdown' } @@ -123,12 +121,11 @@ describe('workspace files download route', () => { expect(body.error).not.toContain('ready.md') }) - it('rejects when rendered documents exceed the download size limit', async () => { + it('rejects with 400, not 500, when a rendered document blows the byte budget', async () => { mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'huge.docx', 'folder-1')]) - mockDownloadServableFileFromStorage.mockResolvedValue({ - buffer: Buffer.alloc(251 * 1024 * 1024), - contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - }) + mockFetchServableWorkspaceFileBuffer.mockRejectedValue( + new PayloadSizeLimitError('servable file download exceeds limit') + ) const response = await GET(requestFor('fileIds=f1'), context) @@ -136,12 +133,40 @@ describe('workspace files download route', () => { expect((await response.json()).error).toContain('exceeds') }) - it('surfaces a storage failure as a 500 rather than zipping a placeholder', async () => { - mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'a.txt', 'folder-1')]) - mockDownloadServableFileFromStorage.mockRejectedValue(new Error('storage down')) + it('stops issuing reads once one hard-fails instead of draining the selection', async () => { + const files = Array.from({ length: 60 }, (_, index) => + workspaceFile(`f${index}`, `doc${index}.txt`, 'folder-1') + ) + mockListWorkspaceFiles.mockResolvedValue(files) + mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { + if (file.name === 'doc0.txt') throw new Error('storage down') + return { buffer: Buffer.from('ok'), contentType: 'text/plain' } + }) - const response = await GET(requestFor('fileIds=f1'), context) + const response = await GET( + requestFor(files.map((file) => `fileIds=${file.id}`).join('&')), + context + ) + + expect(response.status).toBe(500) + // Reads already in flight finish, but the queued remainder is never started. + expect(mockFetchServableWorkspaceFileBuffer.mock.calls.length).toBeLessThan(files.length) + }) + + it('surfaces a storage failure as a 500 even when another document is pending', async () => { + mockListWorkspaceFiles.mockResolvedValue([ + workspaceFile('f1', 'pending.docx', 'folder-1'), + workspaceFile('f2', 'broken.txt', 'folder-1'), + ]) + mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { + if (file.name === 'pending.docx') + throw new DocCompileUserError('Document is still being generated') + throw new Error('storage down') + }) + + const response = await GET(requestFor('fileIds=f1&fileIds=f2'), context) + // A 409 would tell the client to retry something that can never succeed. expect(response.status).toBe(500) }) }) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index dd495979d9c..12ccd1692e2 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -5,38 +5,32 @@ import { type NextRequest, NextResponse } from 'next/server' import { downloadWorkspaceFileItemsContract } from '@/lib/api/contracts/workspace-file-folders' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' -import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile' import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' -import { generateRequestId } from '@/lib/core/utils/request' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' -import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { buildWorkspaceFileFolderPathMap, + fetchServableWorkspaceFileBuffer, listWorkspaceFileFolders, listWorkspaceFiles, } from '@/lib/uploads/contexts/workspace' import { formatFileSize } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/servable-file-response' import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' -import type { UserFile } from '@/executor/types' const logger = createLogger('WorkspaceFilesDownloadAPI') const MAX_ZIP_DOWNLOAD_FILES = 100 const MAX_ZIP_DOWNLOAD_BYTES = 250 * 1024 * 1024 -/** Shape `downloadServableFileFromStorage` needs to locate and label the stored object. */ -function toServableInput(file: WorkspaceFileRecord): UserFile { - return { - id: file.id, - name: file.name, - url: file.path, - size: file.size, - type: file.type, - key: file.key, - context: 'workspace', - } +function overLimitResponse(bytes: number, qualifier = ''): NextResponse { + return NextResponse.json( + { + error: `Selected files total ${formatFileSize(bytes)}${qualifier}, which exceeds the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit.`, + }, + { status: 400 } + ) } function collectDescendantFolderIds( @@ -100,67 +94,66 @@ export const GET = withRouteHandler( ) } - const totalBytes = filesToZip.reduce((sum, file) => sum + file.size, 0) - if (totalBytes > MAX_ZIP_DOWNLOAD_BYTES) { - return NextResponse.json( - { - error: `Selected files total ${formatFileSize(totalBytes)}, which exceeds the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit.`, - }, - { status: 400 } - ) + const declaredBytes = filesToZip.reduce((sum, file) => sum + file.size, 0) + if (declaredBytes > MAX_ZIP_DOWNLOAD_BYTES) { + return overLimitResponse(declaredBytes) } // Generated docs (docx/pptx/pdf/xlsx) store their generation source, not the - // rendered binary, so the bytes must be resolved through the shared servable - // helper — a raw read ships source text under a `.docx` name. - const requestId = generateRequestId() + // rendered binary, so bytes are resolved through the servable reader — a raw + // read ships source text under a `.docx` name. The rendered artifact can be far + // larger than the declared source size, so the budget is tracked as files land + // and each read is capped at what is left: the request is bounded by the limit + // rather than by 100 files each allowed the full limit. Once the budget is blown + // or a read hard-fails, the abort flag stops reads that have not started yet. + const controller = new AbortController() + let renderedBytes = 0 + let overLimit = false + const downloads = await mapWithConcurrency( filesToZip, MATERIALIZE_CONCURRENCY, async (file) => { + if (controller.signal.aborted) return { buffer: null, pendingName: null, error: null } try { - const servable = await downloadServableFileFromStorage( - toServableInput(file), - requestId, - logger, - { maxBytes: MAX_ZIP_DOWNLOAD_BYTES } - ) - return { ok: true as const, buffer: servable.buffer } + const { buffer } = await fetchServableWorkspaceFileBuffer(file, { + maxBytes: Math.max(0, MAX_ZIP_DOWNLOAD_BYTES - renderedBytes), + signal: controller.signal, + }) + renderedBytes += buffer.length + if (renderedBytes > MAX_ZIP_DOWNLOAD_BYTES) { + overLimit = true + controller.abort() + } + return { buffer, pendingName: null, error: null } } catch (error) { - return { ok: false as const, file, error } + // A file bigger than the remaining budget is a size rejection, not a fault. + if (error instanceof PayloadSizeLimitError) { + overLimit = true + controller.abort() + return { buffer: null, pendingName: null, error: null } + } + // A pending artifact is worth reporting in full, so keep resolving the + // rest of the selection; anything else dooms the request. + const pending = isDocNotReadyError(error) + if (!pending) controller.abort() + return { buffer: null, pendingName: pending ? file.name : null, error } } } ) - const pending = downloads.filter( - (result) => !result.ok && result.error instanceof DocCompileUserError - ) - if (pending.length > 0) { - const names = pending.map((result) => (result.ok ? '' : result.file.name)) - return NextResponse.json( - { - error: `${pending.length} document${pending.length === 1 ? ' is' : 's are'} still being generated: ${names.join(', ')}. Wait for them to finish, then try again.`, - }, - { status: 409 } - ) - } + // A hard failure outranks a pending artifact: waiting cannot fix it, so a 409 + // would send the client into a retry loop that never succeeds. + const failure = downloads.find((result) => result.error && !result.pendingName) + if (failure?.error) throw failure.error - // Any other failure is a real storage error: fail the request rather than - // handing back an archive with a file silently missing or empty. - const buffers = downloads.map((result) => { - if (!result.ok) throw result.error - return result.buffer - }) + if (overLimit || renderedBytes > MAX_ZIP_DOWNLOAD_BYTES) { + return overLimitResponse(renderedBytes, ' once documents are rendered') + } - // The pre-download cap used declared source sizes; generated docs resolve larger. - const resolvedBytes = buffers.reduce((sum, buffer) => sum + buffer.length, 0) - if (resolvedBytes > MAX_ZIP_DOWNLOAD_BYTES) { - return NextResponse.json( - { - error: `Selected files total ${formatFileSize(resolvedBytes)} once documents are rendered, which exceeds the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit.`, - }, - { status: 400 } - ) + const pendingNames = downloads.flatMap((result) => result.pendingName ?? []) + if (pendingNames.length > 0) { + return NextResponse.json({ error: docNotReadyMessage(pendingNames) }, { status: 409 }) } // Entry paths stay workspace-root-relative so a mixed selection of folders and @@ -172,10 +165,11 @@ export const GET = withRouteHandler( })) ) + // Indexed off `downloads` so entries stay aligned with `filesToZip` by construction. const zip = new JSZip() - for (const [index, buffer] of buffers.entries()) { - zip.file(entryPaths[index], buffer) - } + downloads.forEach((result, index) => { + if (result.buffer) zip.file(entryPaths[index], result.buffer) + }) const zipBuffer = await zip.generateAsync({ type: 'nodebuffer' }) @@ -185,7 +179,7 @@ export const GET = withRouteHandler( action: AuditAction.FILE_DOWNLOADED, resourceType: AuditResourceType.FILE, description: `Downloaded ${filesToZip.length} file${filesToZip.length === 1 ? '' : 's'} as zip`, - metadata: { fileCount: filesToZip.length, totalBytes: resolvedBytes }, + metadata: { fileCount: filesToZip.length, totalBytes: renderedBytes }, request, }) captureServerEvent( @@ -195,13 +189,18 @@ export const GET = withRouteHandler( { groups: { workspace: workspaceId } } ) - return new NextResponse(new Uint8Array(zipBuffer), { - headers: { - 'Content-Type': 'application/zip', - 'Content-Disposition': 'attachment; filename="workspace-files.zip"', - 'Cache-Control': 'no-store', - }, - }) + return new NextResponse( + // View, not copy — a 250 MB archive would otherwise cost 500 MB. Node buffers + // are never backed by a SharedArrayBuffer, so the narrowing is sound. + new Uint8Array(zipBuffer.buffer as ArrayBuffer, zipBuffer.byteOffset, zipBuffer.byteLength), + { + headers: { + 'Content-Type': 'application/zip', + 'Content-Disposition': 'attachment; filename="workspace-files.zip"', + 'Cache-Control': 'no-store', + }, + } + ) } catch (error) { logger.error('Failed to download workspace file selection:', error) return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 6967a7d313f..5b490c8168e 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -21,6 +21,7 @@ import { normalizeVfsSegment } from '@/lib/copilot/vfs/normalize-segment' import { canonicalWorkspaceFilePath, decodeVfsPathSegments } from '@/lib/copilot/vfs/path-utils' import { resolveWorkflowAliasForWorkspace } from '@/lib/copilot/vfs/workflow-alias-resolver' import { isReservedWorkflowAliasBackingDisplayPath } from '@/lib/copilot/vfs/workflow-aliases' +import { generateRequestId } from '@/lib/core/utils/request' import { generateRestoreName } from '@/lib/core/utils/restore-name' import type { DbOrTx } from '@/lib/db/types' import { getServePathPrefix } from '@/lib/uploads' @@ -968,7 +969,42 @@ export async function getWorkspaceFile( } /** - * Download workspace file content + * Download the bytes a user should actually receive for a workspace file. + * + * Generated docs (docx/pptx/pdf/xlsx) store their GENERATION SOURCE as the primary + * file, so {@link fetchWorkspaceFileBuffer} hands back JavaScript/Python text under + * a `.docx` name. This resolves the rendered artifact instead, and is what every + * download/attachment surface should call. Reach for the raw reader only when the + * source itself is wanted (style extraction, compile checks, the copilot VFS). + * + * Throws `DocCompileUserError` when a generated doc's artifact is still compiling — + * callers surface a retryable 409 via `docNotReadyResponse` rather than shipping source. + */ +export async function fetchServableWorkspaceFileBuffer( + fileRecord: WorkspaceFileRecord, + options: { maxBytes?: number; signal?: AbortSignal } = {} +): Promise<{ buffer: Buffer; contentType: string }> { + const { downloadServableFileFromStorage } = await import('@/lib/uploads/utils/file-utils.server') + + return downloadServableFileFromStorage( + { + id: fileRecord.id, + name: fileRecord.name, + url: fileRecord.url ?? fileRecord.path, + size: fileRecord.size, + type: fileRecord.type, + key: fileRecord.key, + context: fileRecord.storageContext ?? 'workspace', + }, + generateRequestId(), + logger, + options + ) +} + +/** + * Download raw workspace file content. For generated docs this is the GENERATION + * SOURCE, not the rendered document — see {@link fetchServableWorkspaceFileBuffer}. */ export async function fetchWorkspaceFileBuffer( fileRecord: WorkspaceFileRecord, diff --git a/apps/sim/lib/uploads/utils/servable-file-response.ts b/apps/sim/lib/uploads/utils/servable-file-response.ts index 1e7d1a6124b..c4c07cad922 100644 --- a/apps/sim/lib/uploads/utils/servable-file-response.ts +++ b/apps/sim/lib/uploads/utils/servable-file-response.ts @@ -1,6 +1,23 @@ import { NextResponse } from 'next/server' import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile' +/** True when `error` means a generated document's artifact is still compiling. */ +export function isDocNotReadyError(error: unknown): error is DocCompileUserError { + return error instanceof DocCompileUserError +} + +/** + * Message for a still-compiling generated document. Batch callers pass the names + * they resolved so the copy says which documents to wait on. + */ +export function docNotReadyMessage(fileNames?: string[]): string { + if (!fileNames || fileNames.length === 0) { + return 'A document is still being generated. Wait for it to finish, then try again.' + } + const subject = fileNames.length === 1 ? 'A document is' : `${fileNames.length} documents are` + return `${subject} still being generated: ${fileNames.join(', ')}. Wait for them to finish, then try again.` +} + /** * Canonical retryable response for an attachment/upload whose generated-document * artifact is still compiling. Returns the 409 when `error` is a @@ -8,14 +25,13 @@ import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compil * otherwise `null` so the caller falls through to its own error handling. Shared * by every tool route that downloads workspace files so the status, body shape, * and user-facing copy stay identical instead of being re-typed per route. + * + * Batch callers pass `fileNames` so the message names the pending documents. */ -export function docNotReadyResponse(error: unknown): NextResponse | null { - if (error instanceof DocCompileUserError) { +export function docNotReadyResponse(error: unknown, fileNames?: string[]): NextResponse | null { + if (isDocNotReadyError(error)) { return NextResponse.json( - { - success: false, - error: 'A document is still being generated. Wait for it to finish, then try again.', - }, + { success: false, error: docNotReadyMessage(fileNames) }, { status: 409 } ) } From 81a7e92535050d1870d1449ee91c605e11829ea6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 15:37:14 -0700 Subject: [PATCH 03/12] improvement(files): bound rendered documents with a per-entry byte ceiling Concurrent reads all sample the same running total before any of them land, so capping each at the remaining budget still let every in-flight read be granted the whole 250 MiB. Ordinary uploads serve exactly their declared size, and the pre-download check already bounds their sum to the request budget, so only documents that render from a generation source can exceed what they declared. Those now carry a per-entry ceiling, which is what actually bounds peak memory. The renderable extension set moves into file-utils so the read path and this cap agree on which files can expand. --- .../[id]/files/download/route.test.ts | 22 ++++++++++++++++ .../workspaces/[id]/files/download/route.ts | 25 ++++++++++++++----- .../lib/uploads/utils/file-utils.server.ts | 5 ++-- apps/sim/lib/uploads/utils/file-utils.ts | 12 +++++++++ 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts index 22313e588ca..921aa16fbc9 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -133,6 +133,28 @@ describe('workspace files download route', () => { expect((await response.json()).error).toContain('exceeds') }) + it('caps rendered documents per entry so concurrent reads cannot each claim the budget', async () => { + mockListWorkspaceFiles.mockResolvedValue([ + workspaceFile('f1', 'report.docx', 'folder-1'), + workspaceFile('f2', 'clip.mp4', 'folder-1'), + ]) + mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ + buffer: Buffer.from('ok'), + contentType: 'application/octet-stream', + }) + + await GET(requestFor('fileIds=f1&fileIds=f2'), context) + + const maxBytesFor = (name: string) => + mockFetchServableWorkspaceFileBuffer.mock.calls.find( + (call: [{ name: string }, { maxBytes: number }]) => call[0].name === name + )?.[1].maxBytes + + // Only the source-backed document can render larger than it declares. + expect(maxBytesFor('report.docx')).toBe(50 * 1024 * 1024) + expect(maxBytesFor('clip.mp4')).toBe(250 * 1024 * 1024) + }) + it('stops issuing reads once one hard-fails instead of draining the selection', async () => { const files = Array.from({ length: 60 }, (_, index) => workspaceFile(`f${index}`, `doc${index}.txt`, 'folder-1') diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index 12ccd1692e2..8d711f2a4cf 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -15,7 +15,7 @@ import { listWorkspaceFileFolders, listWorkspaceFiles, } from '@/lib/uploads/contexts/workspace' -import { formatFileSize } from '@/lib/uploads/utils/file-utils' +import { formatFileSize, isRenderableDocumentName } from '@/lib/uploads/utils/file-utils' import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/servable-file-response' import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' @@ -23,6 +23,15 @@ import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' const logger = createLogger('WorkspaceFilesDownloadAPI') const MAX_ZIP_DOWNLOAD_FILES = 100 const MAX_ZIP_DOWNLOAD_BYTES = 250 * 1024 * 1024 +/** + * Per-entry ceiling for documents that render from a generation source. Ordinary + * uploads serve exactly their declared size, which the pre-download check already + * bounds to the request budget; only rendered documents can exceed what they + * declared. Capping them individually is what bounds peak memory, because up to + * `MATERIALIZE_CONCURRENCY` reads are in flight against the same budget and none + * of them can see the others' bytes until they land. + */ +const MAX_RENDERED_DOCUMENT_BYTES = 50 * 1024 * 1024 function overLimitResponse(bytes: number, qualifier = ''): NextResponse { return NextResponse.json( @@ -102,10 +111,11 @@ export const GET = withRouteHandler( // Generated docs (docx/pptx/pdf/xlsx) store their generation source, not the // rendered binary, so bytes are resolved through the servable reader — a raw // read ships source text under a `.docx` name. The rendered artifact can be far - // larger than the declared source size, so the budget is tracked as files land - // and each read is capped at what is left: the request is bounded by the limit - // rather than by 100 files each allowed the full limit. Once the budget is blown - // or a read hard-fails, the abort flag stops reads that have not started yet. + // larger than the declared source size, so each read is capped at whatever is + // left of the budget and rendered documents additionally at a per-entry ceiling + // — concurrent reads cannot see each other's bytes until they land, so the + // per-entry cap is what bounds peak memory. Once the budget is blown or a read + // hard-fails, the abort flag stops reads that have not started yet. const controller = new AbortController() let renderedBytes = 0 let overLimit = false @@ -115,9 +125,12 @@ export const GET = withRouteHandler( MATERIALIZE_CONCURRENCY, async (file) => { if (controller.signal.aborted) return { buffer: null, pendingName: null, error: null } + const remaining = Math.max(0, MAX_ZIP_DOWNLOAD_BYTES - renderedBytes) try { const { buffer } = await fetchServableWorkspaceFileBuffer(file, { - maxBytes: Math.max(0, MAX_ZIP_DOWNLOAD_BYTES - renderedBytes), + maxBytes: isRenderableDocumentName(file.name) + ? Math.min(remaining, MAX_RENDERED_DOCUMENT_BYTES) + : remaining, signal: controller.signal, }) renderedBytes += buffer.length diff --git a/apps/sim/lib/uploads/utils/file-utils.server.ts b/apps/sim/lib/uploads/utils/file-utils.server.ts index cc230df8108..97013439e77 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.ts @@ -20,6 +20,7 @@ import { getMimeTypeFromExtension, inferContextFromKey, isInternalFileUrl, + isRenderableDocumentName, processSingleFileToUserFile, type RawFileInput, resolveTrustedFileContext, @@ -375,8 +376,8 @@ export async function downloadServableFileFromStorage( // Cheap pre-filter so only generated-doc candidates pay for the heavier resolver // import below. - const ext = getFileExtension(userFile.name) - if (ext !== 'pdf' && ext !== 'docx' && ext !== 'pptx' && ext !== 'xlsx') { + if (!isRenderableDocumentName(userFile.name)) { + const ext = getFileExtension(userFile.name) return { buffer, contentType: userFile.type || getMimeTypeFromExtension(ext) } } diff --git a/apps/sim/lib/uploads/utils/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts index 7837617a0cc..442d52d324f 100644 --- a/apps/sim/lib/uploads/utils/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -210,6 +210,18 @@ export function getFileExtension(filename: string): string { return lastDot !== -1 ? filename.slice(lastDot + 1).toLowerCase() : '' } +/** + * Extensions whose stored bytes may be a generation source that renders to a larger + * binary. Everything else stores exactly what it serves, so its declared size is + * an accurate byte budget. + */ +const RENDERABLE_DOCUMENT_EXTENSIONS = new Set(['pdf', 'docx', 'pptx', 'xlsx']) + +/** True when `fileName` may be backed by a generation source rather than final bytes. */ +export function isRenderableDocumentName(fileName: string): boolean { + return RENDERABLE_DOCUMENT_EXTENSIONS.has(getFileExtension(fileName)) +} + const ARCHIVE_EXTENSIONS = new Set(SUPPORTED_ARCHIVE_EXTENSIONS) /** From 84d174be81f5ef36ba57a29bb473c1ecc0078554 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 15:44:07 -0700 Subject: [PATCH 04/12] fix(files): keep size and cancellation outcomes distinct across archive surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cancelling the shared controller made other in-flight reads reject, and those rejections were being recorded as hard failures — so an oversized selection surfaced as a generic 500 instead of the descriptive 400. A read that finds the controller already aborted now reports nothing, checked before the failing worker aborts so the real cause is still recorded, and the size check runs first. The compress handler can also hit the per-file cap now that rendered artifacts can exceed their stored source, so the handler maps PayloadSizeLimitError to 413 alongside the existing combined-size response. The v1 download resolved rendered bytes but still labelled them with the record's source MIME, handing clients .docx bytes typed as text/x-docxjs. --- apps/sim/app/api/tools/file/manage/route.ts | 6 ++++++ apps/sim/app/api/v1/files/[fileId]/route.ts | 5 +++-- .../[id]/files/download/route.test.ts | 21 +++++++++++++++++++ .../workspaces/[id]/files/download/route.ts | 16 ++++++++++---- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/apps/sim/app/api/tools/file/manage/route.ts b/apps/sim/app/api/tools/file/manage/route.ts index 7bca7f82d63..8f1cca412f6 100644 --- a/apps/sim/app/api/tools/file/manage/route.ts +++ b/apps/sim/app/api/tools/file/manage/route.ts @@ -11,6 +11,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { splitWorkspaceFilePath } from '@/lib/copilot/tools/server/files/workspace-file' import { acquireLock, releaseLock } from '@/lib/core/config/redis' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { ensureAbsoluteUrl } from '@/lib/core/utils/urls' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { isSupportedFileType, parseBuffer } from '@/lib/file-parsers' @@ -868,6 +869,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const notReady = docNotReadyResponse(error) if (notReady) return notReady + // A file over its per-file cap is a size rejection, not a fault. Rendered + // documents can cross it even when the stored source was well under. + if (isPayloadSizeLimitError(error)) { + return NextResponse.json({ success: false, error: error.message }, { status: 413 }) + } if (error instanceof ShareValidationError) { return NextResponse.json({ success: false, error: error.message }, { status: 400 }) } diff --git a/apps/sim/app/api/v1/files/[fileId]/route.ts b/apps/sim/app/api/v1/files/[fileId]/route.ts index c9972cc8573..8befb1c2462 100644 --- a/apps/sim/app/api/v1/files/[fileId]/route.ts +++ b/apps/sim/app/api/v1/files/[fileId]/route.ts @@ -52,7 +52,8 @@ export const GET = withRouteHandler(async (request: NextRequest, context: FileRo } // Generated docs store their generation source; serve the rendered artifact. - const { buffer } = await fetchServableWorkspaceFileBuffer(fileRecord) + // Its content type is the rendered one, not the source MIME on the record. + const { buffer, contentType } = await fetchServableWorkspaceFileBuffer(fileRecord) recordAudit({ workspaceId, @@ -80,7 +81,7 @@ export const GET = withRouteHandler(async (request: NextRequest, context: FileRo return new Response(new Uint8Array(buffer), { status: 200, headers: { - 'Content-Type': fileRecord.type || 'application/octet-stream', + 'Content-Type': contentType || fileRecord.type || 'application/octet-stream', 'Content-Disposition': `attachment; filename="${fileRecord.name.replace(/[^\w.-]/g, '_')}"; filename*=UTF-8''${encodeURIComponent(fileRecord.name)}`, 'Content-Length': String(buffer.length), 'X-File-Id': fileRecord.id, diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts index 921aa16fbc9..d74ae2e639e 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -155,6 +155,27 @@ describe('workspace files download route', () => { expect(maxBytesFor('clip.mp4')).toBe(250 * 1024 * 1024) }) + it('reports an oversized selection as 400 even when the abort cancels other reads', async () => { + const files = Array.from({ length: 40 }, (_, index) => + workspaceFile(`f${index}`, `doc${index}.docx`, 'folder-1') + ) + mockListWorkspaceFiles.mockResolvedValue(files) + mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { + if (file.name === 'doc0.docx') + throw new PayloadSizeLimitError({ label: 'servable file download', maxBytes: 1 }) + // Everything else fails the way a cancelled read would. + throw new DOMException('The operation was aborted', 'AbortError') + }) + + const response = await GET( + requestFor(files.map((file) => `fileIds=${file.id}`).join('&')), + context + ) + + // Cancellation noise must not turn the size rejection into a generic 500. + expect(response.status).toBe(400) + }) + it('stops issuing reads once one hard-fails instead of draining the selection', async () => { const files = Array.from({ length: 60 }, (_, index) => workspaceFile(`f${index}`, `doc${index}.txt`, 'folder-1') diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index 8d711f2a4cf..49e72be7628 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -140,6 +140,12 @@ export const GET = withRouteHandler( } return { buffer, pendingName: null, error: null } } catch (error) { + // Checked before this worker aborts anything, so the worker that actually + // failed still records its error while reads cancelled as a consequence + // report nothing — otherwise cancellation noise masks the real outcome. + if (controller.signal.aborted) { + return { buffer: null, pendingName: null, error: null } + } // A file bigger than the remaining budget is a size rejection, not a fault. if (error instanceof PayloadSizeLimitError) { overLimit = true @@ -155,15 +161,17 @@ export const GET = withRouteHandler( } ) + // Size first: the request cannot succeed at any size-adjacent retry, and a + // descriptive 400 beats an opaque 500 raised by whatever the abort cancelled. + if (overLimit || renderedBytes > MAX_ZIP_DOWNLOAD_BYTES) { + return overLimitResponse(renderedBytes, ' once documents are rendered') + } + // A hard failure outranks a pending artifact: waiting cannot fix it, so a 409 // would send the client into a retry loop that never succeeds. const failure = downloads.find((result) => result.error && !result.pendingName) if (failure?.error) throw failure.error - if (overLimit || renderedBytes > MAX_ZIP_DOWNLOAD_BYTES) { - return overLimitResponse(renderedBytes, ' once documents are rendered') - } - const pendingNames = downloads.flatMap((result) => result.pendingName ?? []) if (pendingNames.length > 0) { return NextResponse.json({ error: docNotReadyMessage(pendingNames) }, { status: 409 }) From 257abfee7efde8012641908442a9890dd5abe572 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 15:51:56 -0700 Subject: [PATCH 05/12] fix(files): size the per-entry allowance off the declared size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The office-extension cap rejected ordinary uploads: the per-file limit is enforced on the stored object before the magic-byte passthrough decides whether it is a generated source, so a legitimate 80 MiB deck failed the zip at the 50 MiB render ceiling. Uploads and source-backed docs share those extensions and cannot be told apart before the bytes are read, so the allowance is now the larger of the declared size and the render headroom — real uploads download at their true size while a small generator source still cannot render unbounded. Zip materialization also drops to its own concurrency limit, since each in-flight entry is a whole file held in memory. --- .../[id]/files/download/route.test.ts | 15 +++++++ .../workspaces/[id]/files/download/route.ts | 40 +++++++++++-------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts index d74ae2e639e..cd473ecb1c6 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -133,6 +133,21 @@ describe('workspace files download route', () => { expect((await response.json()).error).toContain('exceeds') }) + it('lets an uploaded office file larger than the render headroom through', async () => { + const big = { ...workspaceFile('f1', 'deck.pptx', 'folder-1'), size: 80 * 1024 * 1024 } + mockListWorkspaceFiles.mockResolvedValue([big]) + mockFetchServableWorkspaceFileBuffer.mockResolvedValue({ + buffer: Buffer.from('ok'), + contentType: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + }) + + const response = await GET(requestFor('fileIds=f1'), context) + + expect(response.status).toBe(200) + // Capped at the declared size, not the smaller render headroom. + expect(mockFetchServableWorkspaceFileBuffer.mock.calls[0][1].maxBytes).toBe(80 * 1024 * 1024) + }) + it('caps rendered documents per entry so concurrent reads cannot each claim the budget', async () => { mockListWorkspaceFiles.mockResolvedValue([ workspaceFile('f1', 'report.docx', 'folder-1'), diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index 49e72be7628..7802779ccf3 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -5,7 +5,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { downloadWorkspaceFileItemsContract } from '@/lib/api/contracts/workspace-file-folders' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' -import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' @@ -24,14 +24,20 @@ const logger = createLogger('WorkspaceFilesDownloadAPI') const MAX_ZIP_DOWNLOAD_FILES = 100 const MAX_ZIP_DOWNLOAD_BYTES = 250 * 1024 * 1024 /** - * Per-entry ceiling for documents that render from a generation source. Ordinary - * uploads serve exactly their declared size, which the pre-download check already - * bounds to the request budget; only rendered documents can exceed what they - * declared. Capping them individually is what bounds peak memory, because up to - * `MATERIALIZE_CONCURRENCY` reads are in flight against the same budget and none - * of them can see the others' bytes until they land. + * Headroom a document may render to beyond what it declared. Office extensions + * cover both ordinary uploads — which serve exactly their declared size — and + * source-backed generated docs, and the two are indistinguishable before the bytes + * are read. So the allowance is the larger of the declared size and this ceiling: + * an uploaded 80 MiB deck still downloads, while a 6 KiB generator source cannot + * quietly render into hundreds of megabytes. */ -const MAX_RENDERED_DOCUMENT_BYTES = 50 * 1024 * 1024 +const RENDERED_DOCUMENT_HEADROOM_BYTES = 50 * 1024 * 1024 +/** + * Reads in flight cannot see each other's bytes until they land, so peak memory + * scales with this. Kept well below the shared default: the entries here are whole + * files held in memory at once, not rows. + */ +const ZIP_MATERIALIZE_CONCURRENCY = 5 function overLimitResponse(bytes: number, qualifier = ''): NextResponse { return NextResponse.json( @@ -112,25 +118,27 @@ export const GET = withRouteHandler( // rendered binary, so bytes are resolved through the servable reader — a raw // read ships source text under a `.docx` name. The rendered artifact can be far // larger than the declared source size, so each read is capped at whatever is - // left of the budget and rendered documents additionally at a per-entry ceiling - // — concurrent reads cannot see each other's bytes until they land, so the - // per-entry cap is what bounds peak memory. Once the budget is blown or a read - // hard-fails, the abort flag stops reads that have not started yet. + // left of the budget, with office extensions additionally held to their declared + // size plus render headroom. Concurrent reads cannot see each other's bytes until + // they land, so that per-entry allowance and the concurrency limit are together + // what bound peak memory. Once the budget is blown or a read hard-fails, the + // abort flag stops reads that have not started yet. const controller = new AbortController() let renderedBytes = 0 let overLimit = false const downloads = await mapWithConcurrency( filesToZip, - MATERIALIZE_CONCURRENCY, + ZIP_MATERIALIZE_CONCURRENCY, async (file) => { if (controller.signal.aborted) return { buffer: null, pendingName: null, error: null } const remaining = Math.max(0, MAX_ZIP_DOWNLOAD_BYTES - renderedBytes) try { + const allowance = isRenderableDocumentName(file.name) + ? Math.max(file.size, RENDERED_DOCUMENT_HEADROOM_BYTES) + : remaining const { buffer } = await fetchServableWorkspaceFileBuffer(file, { - maxBytes: isRenderableDocumentName(file.name) - ? Math.min(remaining, MAX_RENDERED_DOCUMENT_BYTES) - : remaining, + maxBytes: Math.min(remaining, allowance), signal: controller.signal, }) renderedBytes += buffer.length From d9b9a3047f3f945a451cfe3dae5741bc262ad5e8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 15:59:13 -0700 Subject: [PATCH 06/12] fix(files): record a size rejection even when another read aborted first The cancellation guard sat ahead of the size check, so a worker that hit its byte allowance after someone else's hard failure had already aborted the shared controller was treated as cancellation noise. overLimit never got set and an actionable 400 came back as an opaque 500. A size rejection describes the file that raised it regardless of who aborted, so it is recorded first; every other error from an already-aborted read stays suppressed as a consequence of the cancellation. --- .../[id]/files/download/route.test.ts | 18 ++++++++++++++++++ .../workspaces/[id]/files/download/route.ts | 16 +++++++++------- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts index cd473ecb1c6..8ef7e7376c9 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -191,6 +191,24 @@ describe('workspace files download route', () => { expect(response.status).toBe(400) }) + it('keeps the size rejection when a hard failure aborted the read first', async () => { + mockListWorkspaceFiles.mockResolvedValue([ + workspaceFile('f1', 'broken.txt', 'folder-1'), + workspaceFile('f2', 'huge.docx', 'folder-1'), + ]) + mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { + if (file.name === 'broken.txt') throw new Error('storage down') + // Lands after the hard failure has already aborted the shared controller. + await new Promise((resolve) => setTimeout(resolve, 1)) + throw new PayloadSizeLimitError({ label: 'servable file download', maxBytes: 1 }) + }) + + const response = await GET(requestFor('fileIds=f1&fileIds=f2'), context) + + // "Select fewer files" is actionable; a generic 500 is not. + expect(response.status).toBe(400) + }) + it('stops issuing reads once one hard-fails instead of draining the selection', async () => { const files = Array.from({ length: 60 }, (_, index) => workspaceFile(`f${index}`, `doc${index}.txt`, 'folder-1') diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index 7802779ccf3..06bf254d93e 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -148,18 +148,20 @@ export const GET = withRouteHandler( } return { buffer, pendingName: null, error: null } } catch (error) { - // Checked before this worker aborts anything, so the worker that actually - // failed still records its error while reads cancelled as a consequence - // report nothing — otherwise cancellation noise masks the real outcome. - if (controller.signal.aborted) { - return { buffer: null, pendingName: null, error: null } - } - // A file bigger than the remaining budget is a size rejection, not a fault. + // Recorded even when another worker already aborted: a size rejection + // describes this file, so losing it to someone else's cancellation would + // downgrade an actionable 400 into an opaque 500. if (error instanceof PayloadSizeLimitError) { overLimit = true controller.abort() return { buffer: null, pendingName: null, error: null } } + // Any other error from an already-aborted read is a consequence of the + // cancellation, not a cause. Checked before this worker aborts anything so + // the worker that actually failed still records its own error. + if (controller.signal.aborted) { + return { buffer: null, pendingName: null, error: null } + } // A pending artifact is worth reporting in full, so keep resolving the // rest of the selection; anything else dooms the request. const pending = isDocNotReadyError(error) From 396ef43f2776c957df43051ceedca233ba4ea1d6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 16:05:55 -0700 Subject: [PATCH 07/12] fix(files): name the entry that exceeds the per-document render cap CI also caught a raw setTimeout promise in the new test; use sleep(). A document that renders past its own allowance was reported with the aggregate message, telling the user to select fewer files when the selection was within the budget. That case now names the offending entry and states the per-document limit, so the aggregate message only appears when the selection really is too large. --- .../workspaces/[id]/files/download/route.test.ts | 8 ++++++-- .../app/api/workspaces/[id]/files/download/route.ts | 13 +++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts index 8ef7e7376c9..c604ce118d0 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -2,6 +2,7 @@ * @vitest-environment node */ import { createMockRequest } from '@sim/testing' +import { sleep } from '@sim/utils/helpers' import JSZip from 'jszip' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -130,7 +131,10 @@ describe('workspace files download route', () => { const response = await GET(requestFor('fileIds=f1'), context) expect(response.status).toBe(400) - expect((await response.json()).error).toContain('exceeds') + // Names the offending entry rather than blaming the whole selection. + const body = await response.json() + expect(body.error).toContain('huge.docx') + expect(body.error).not.toContain('Selected files total') }) it('lets an uploaded office file larger than the render headroom through', async () => { @@ -199,7 +203,7 @@ describe('workspace files download route', () => { mockFetchServableWorkspaceFileBuffer.mockImplementation(async (file: { name: string }) => { if (file.name === 'broken.txt') throw new Error('storage down') // Lands after the hard failure has already aborted the shared controller. - await new Promise((resolve) => setTimeout(resolve, 1)) + await sleep(1) throw new PayloadSizeLimitError({ label: 'servable file download', maxBytes: 1 }) }) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index 06bf254d93e..aeffc9a2620 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -126,6 +126,7 @@ export const GET = withRouteHandler( const controller = new AbortController() let renderedBytes = 0 let overLimit = false + let overLimitFileName: string | null = null const downloads = await mapWithConcurrency( filesToZip, @@ -153,6 +154,7 @@ export const GET = withRouteHandler( // downgrade an actionable 400 into an opaque 500. if (error instanceof PayloadSizeLimitError) { overLimit = true + overLimitFileName ??= file.name controller.abort() return { buffer: null, pendingName: null, error: null } } @@ -173,6 +175,17 @@ export const GET = withRouteHandler( // Size first: the request cannot succeed at any size-adjacent retry, and a // descriptive 400 beats an opaque 500 raised by whatever the abort cancelled. + if (overLimitFileName) { + // Naming the entry that blew its own allowance: an aggregate message here + // would tell the user to select fewer files when the selection was fine. + return NextResponse.json( + { + error: `"${overLimitFileName}" is too large to include in a zip. A single document may render up to ${formatFileSize(RENDERED_DOCUMENT_HEADROOM_BYTES)}; download it on its own instead.`, + }, + { status: 400 } + ) + } + if (overLimit || renderedBytes > MAX_ZIP_DOWNLOAD_BYTES) { return overLimitResponse(renderedBytes, ' once documents are rendered') } From 8a02ca665de63c1c6137b9a230bfc40e1c46d6a8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 16:20:59 -0700 Subject: [PATCH 08/12] fix(files): return a retryable 409 from the v1 download for compiling docs Resolving rendered bytes there meant a still-compiling artifact threw into the generic catch, so a caller that would previously have received (corrupt) source now got a 500 with nothing to indicate the request is worth retrying. Also drops the fileNames parameter added to docNotReadyResponse last round: all 44 call sites pass a single argument, and the one route that names pending files builds its 409 from docNotReadyMessage because its error envelope differs. --- apps/sim/app/api/v1/files/[fileId]/route.ts | 6 ++++++ apps/sim/lib/uploads/utils/servable-file-response.ts | 10 ++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/sim/app/api/v1/files/[fileId]/route.ts b/apps/sim/app/api/v1/files/[fileId]/route.ts index 8befb1c2462..40ec025932b 100644 --- a/apps/sim/app/api/v1/files/[fileId]/route.ts +++ b/apps/sim/app/api/v1/files/[fileId]/route.ts @@ -10,6 +10,7 @@ import { fetchServableWorkspaceFileBuffer, getWorkspaceFile, } from '@/lib/uploads/contexts/workspace' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/servable-file-response' import { performDeleteWorkspaceFileItems } from '@/lib/workspace-files/orchestration' import { checkRateLimit, @@ -93,6 +94,11 @@ export const GET = withRouteHandler(async (request: NextRequest, context: FileRo }, }) } catch (error) { + // A generated doc whose artifact is still compiling is retryable, not a fault: + // without this the caller sees a 500 and has no reason to try again. + if (isDocNotReadyError(error)) { + return NextResponse.json({ error: docNotReadyMessage() }, { status: 409 }) + } logger.error(`[${requestId}] Error downloading file:`, error) return NextResponse.json({ error: 'Failed to download file' }, { status: 500 }) } diff --git a/apps/sim/lib/uploads/utils/servable-file-response.ts b/apps/sim/lib/uploads/utils/servable-file-response.ts index c4c07cad922..63ffb0223f0 100644 --- a/apps/sim/lib/uploads/utils/servable-file-response.ts +++ b/apps/sim/lib/uploads/utils/servable-file-response.ts @@ -26,14 +26,12 @@ export function docNotReadyMessage(fileNames?: string[]): string { * by every tool route that downloads workspace files so the status, body shape, * and user-facing copy stay identical instead of being re-typed per route. * - * Batch callers pass `fileNames` so the message names the pending documents. + * Routes whose error envelope differs, or that resolved a batch and want the pending + * files named, build the 409 themselves from {@link docNotReadyMessage}. */ -export function docNotReadyResponse(error: unknown, fileNames?: string[]): NextResponse | null { +export function docNotReadyResponse(error: unknown): NextResponse | null { if (isDocNotReadyError(error)) { - return NextResponse.json( - { success: false, error: docNotReadyMessage(fileNames) }, - { status: 409 } - ) + return NextResponse.json({ success: false, error: docNotReadyMessage() }, { status: 409 }) } return null } From 339b648b886477dbfc91caf733d6f59aba2e998f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 16:26:36 -0700 Subject: [PATCH 09/12] fix(files): attribute a size rejection to the cap that actually bound it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every PayloadSizeLimitError was reported as the entry exceeding its per-document render allowance, quoting the 50 MiB headroom. That is wrong when the shared budget was the smaller cap, and wrong for extensions that carry no headroom at all — a video rejected because the budget ran out was told it exceeded a per-document render limit. The entry is now blamed only when its own allowance was the smaller of the two caps, and the message quotes the allowance that applied rather than the constant. The mapper also returns its outcome instead of mutating closure state, which is what the aggregate and per-entry branches now read. --- .../[id]/files/download/route.test.ts | 15 +++++ .../workspaces/[id]/files/download/route.ts | 64 ++++++++++++------- 2 files changed, 55 insertions(+), 24 deletions(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts index c604ce118d0..8b9208e00e1 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -137,6 +137,21 @@ describe('workspace files download route', () => { expect(body.error).not.toContain('Selected files total') }) + it('blames the shared budget, not the entry, when the entry had no smaller cap', async () => { + // .mp4 has no render headroom, so its cap is whatever is left of the budget. + mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'clip.mp4', 'folder-1')]) + mockFetchServableWorkspaceFileBuffer.mockRejectedValue( + new PayloadSizeLimitError({ label: 'servable file download', maxBytes: 1 }) + ) + + const response = await GET(requestFor('fileIds=f1'), context) + + expect(response.status).toBe(400) + const body = await response.json() + expect(body.error).toContain('Selected files total') + expect(body.error).not.toContain('clip.mp4') + }) + it('lets an uploaded office file larger than the render headroom through', async () => { const big = { ...workspaceFile('f1', 'deck.pptx', 'folder-1'), size: 80 * 1024 * 1024 } mockListWorkspaceFiles.mockResolvedValue([big]) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index aeffc9a2620..ec09af594c0 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -125,68 +125,84 @@ export const GET = withRouteHandler( // abort flag stops reads that have not started yet. const controller = new AbortController() let renderedBytes = 0 - let overLimit = false - let overLimitFileName: string | null = null + + interface DownloadOutcome { + buffer: Buffer | null + pendingName: string | null + /** Set only when an entry's own allowance bound it, never the shared budget. */ + overLimitEntry: { name: string; allowance: number } | null + overLimit: boolean + error: unknown + } + const skipped: DownloadOutcome = { + buffer: null, + pendingName: null, + overLimitEntry: null, + overLimit: false, + error: null, + } const downloads = await mapWithConcurrency( filesToZip, ZIP_MATERIALIZE_CONCURRENCY, - async (file) => { - if (controller.signal.aborted) return { buffer: null, pendingName: null, error: null } + async (file): Promise => { + if (controller.signal.aborted) return skipped const remaining = Math.max(0, MAX_ZIP_DOWNLOAD_BYTES - renderedBytes) + const allowance = isRenderableDocumentName(file.name) + ? Math.max(file.size, RENDERED_DOCUMENT_HEADROOM_BYTES) + : remaining try { - const allowance = isRenderableDocumentName(file.name) - ? Math.max(file.size, RENDERED_DOCUMENT_HEADROOM_BYTES) - : remaining const { buffer } = await fetchServableWorkspaceFileBuffer(file, { maxBytes: Math.min(remaining, allowance), signal: controller.signal, }) renderedBytes += buffer.length - if (renderedBytes > MAX_ZIP_DOWNLOAD_BYTES) { - overLimit = true - controller.abort() - } - return { buffer, pendingName: null, error: null } + const overLimit = renderedBytes > MAX_ZIP_DOWNLOAD_BYTES + if (overLimit) controller.abort() + return { ...skipped, buffer, overLimit } } catch (error) { // Recorded even when another worker already aborted: a size rejection // describes this file, so losing it to someone else's cancellation would // downgrade an actionable 400 into an opaque 500. if (error instanceof PayloadSizeLimitError) { - overLimit = true - overLimitFileName ??= file.name controller.abort() - return { buffer: null, pendingName: null, error: null } + // Attributed to this entry only when its own allowance was the smaller + // of the two caps; otherwise the shared budget is what ran out. + return { + ...skipped, + overLimit: true, + overLimitEntry: allowance < remaining ? { name: file.name, allowance } : null, + } } // Any other error from an already-aborted read is a consequence of the // cancellation, not a cause. Checked before this worker aborts anything so // the worker that actually failed still records its own error. - if (controller.signal.aborted) { - return { buffer: null, pendingName: null, error: null } - } + if (controller.signal.aborted) return skipped // A pending artifact is worth reporting in full, so keep resolving the // rest of the selection; anything else dooms the request. const pending = isDocNotReadyError(error) if (!pending) controller.abort() - return { buffer: null, pendingName: pending ? file.name : null, error } + return { ...skipped, pendingName: pending ? file.name : null, error } } } ) // Size first: the request cannot succeed at any size-adjacent retry, and a // descriptive 400 beats an opaque 500 raised by whatever the abort cancelled. - if (overLimitFileName) { - // Naming the entry that blew its own allowance: an aggregate message here - // would tell the user to select fewer files when the selection was fine. + const overLimitEntry = downloads.find((result) => result.overLimitEntry)?.overLimitEntry + if (overLimitEntry) { + // Naming the entry that blew its own allowance, and quoting the allowance that + // actually applied: an aggregate message here would tell the user to select + // fewer files when the selection was fine. return NextResponse.json( { - error: `"${overLimitFileName}" is too large to include in a zip. A single document may render up to ${formatFileSize(RENDERED_DOCUMENT_HEADROOM_BYTES)}; download it on its own instead.`, + error: `"${overLimitEntry.name}" is too large to include in a zip. Entries are capped at ${formatFileSize(overLimitEntry.allowance)}; download it on its own instead.`, }, { status: 400 } ) } - if (overLimit || renderedBytes > MAX_ZIP_DOWNLOAD_BYTES) { + if (downloads.some((result) => result.overLimit)) { return overLimitResponse(renderedBytes, ' once documents are rendered') } From 78f322fc0be7794f420cc283764e727bd3ed1e8e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 16:36:10 -0700 Subject: [PATCH 10/12] improvement(files): surface archive download errors in place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bulk and folder downloads navigated to the API route, so any rejection replaced the Files page with raw JSON — the user lost their view and got an unstyled error body. That was survivable when the only failures were "too many files" and "too large"; resolving rendered documents adds a 409 for a still-compiling artifact, which is reachable in normal use. Both now fetch the zip and save the blob, matching what single-file download already did, and show the server's message as a toast. The server writes that copy for the user — which document is compiling, which entry is too large — so it is worth showing rather than discarding. Single-file download surfaces its error too instead of only logging it. --- .../workspace/[workspaceId]/files/files.tsx | 22 +++++-- apps/sim/lib/uploads/client/download.ts | 61 +++++++++++++++---- 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index c6bcecde52d..2f85a03f24b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -27,7 +27,7 @@ import { usePostHog } from 'posthog-js/react' import { getDocumentIcon } from '@/components/icons/document-icons' import { useLimitUpgradeToast } from '@/lib/billing/client' import { captureEvent } from '@/lib/posthog/client' -import { triggerFileDownload } from '@/lib/uploads/client/download' +import { triggerArchiveDownload, triggerFileDownload } from '@/lib/uploads/client/download' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' import { @@ -951,6 +951,7 @@ export function Files() { }) } catch (err) { logger.error('Failed to download file:', err) + toast.error(toError(err).message) } }, [workspaceId] @@ -1071,7 +1072,7 @@ export function Files() { setShowDeleteConfirm(true) }, [selectedFileIds, selectedFolderIds, files, folders]) - const handleBulkDownload = useCallback(() => { + const handleBulkDownload = useCallback(async () => { const selectedFiles = files.filter((file) => selectedFileIds.includes(file.id)) if (selectedFiles.length === 1 && selectedFolderIds.length === 0) { handleDownload(selectedFiles[0]) @@ -1088,7 +1089,14 @@ export function Files() { is_bulk: true, file_count: selectedFileIds.length + selectedFolderIds.length, }) - window.location.href = `/api/workspaces/${workspaceId}/files/download?${query.toString()}` + try { + await triggerArchiveDownload( + `/api/workspaces/${workspaceId}/files/download?${query.toString()}` + ) + } catch (err) { + logger.error('Failed to download selection:', err) + toast.error(toError(err).message) + } }, [selectedFileIds, selectedFolderIds, files, handleDownload, workspaceId]) const fileDetailBreadcrumbs = useMemo(() => { @@ -1285,8 +1293,14 @@ export function Files() { return } if (item.kind === 'folder') { - window.location.href = `/api/workspaces/${workspaceId}/files/download?folderIds=${encodeURIComponent(item.folder.id)}` + const folderId = item.folder.id closeContextMenu() + triggerArchiveDownload( + `/api/workspaces/${workspaceId}/files/download?folderIds=${encodeURIComponent(folderId)}` + ).catch((err) => { + logger.error('Failed to download folder:', err) + toast.error(toError(err).message) + }) return } handleDownload(item.file) diff --git a/apps/sim/lib/uploads/client/download.ts b/apps/sim/lib/uploads/client/download.ts index 9cbdd88d263..ed69a2ffdb2 100644 --- a/apps/sim/lib/uploads/client/download.ts +++ b/apps/sim/lib/uploads/client/download.ts @@ -1,5 +1,35 @@ import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +/** Hand a fetched blob to the browser as a file save, then release the object URL. */ +function saveBlob(blob: Blob, fileName: string): void { + const objectUrl = URL.createObjectURL(blob) + const anchor = document.createElement('a') + anchor.href = objectUrl + anchor.download = fileName + document.body.appendChild(anchor) + anchor.click() + document.body.removeChild(anchor) + URL.revokeObjectURL(objectUrl) +} + +function fileNameFromDisposition(response: Response, fallback: string): string { + return response.headers.get('Content-Disposition')?.match(/filename="([^"]+)"/)?.[1] ?? fallback +} + +/** + * Read the server's error copy off a failed download so the caller can surface it. + * These routes answer with `{ error }` and the message is written for the user — + * which document is still compiling, which entry is too large. + */ +async function downloadErrorMessage(response: Response, fallback: string): Promise { + try { + const body = await response.json() + return typeof body?.error === 'string' && body.error ? body.error : fallback + } catch { + return fallback + } +} + export async function triggerFileDownload(record: WorkspaceFileRecord): Promise { const isMarkdown = record.type === 'text/markdown' || @@ -10,17 +40,26 @@ export async function triggerFileDownload(record: WorkspaceFileRecord): Promise< ? `/api/files/export/${encodeURIComponent(record.id)}` : `/api/files/serve/${encodeURIComponent(record.key)}?context=workspace&t=${Date.now()}` + // boundary-raw-fetch: binary download read as a blob, not a JSON contract response const response = await fetch(url, { cache: 'no-store' }) - if (!response.ok) throw new Error(`Failed to download file: ${response.statusText}`) + if (!response.ok) { + throw new Error(await downloadErrorMessage(response, `Failed to download "${record.name}"`)) + } - const blob = await response.blob() - const objectUrl = URL.createObjectURL(blob) - const a = document.createElement('a') - a.href = objectUrl - a.download = - response.headers.get('Content-Disposition')?.match(/filename="([^"]+)"/)?.[1] ?? record.name - document.body.appendChild(a) - a.click() - document.body.removeChild(a) - URL.revokeObjectURL(objectUrl) + saveBlob(await response.blob(), fileNameFromDisposition(response, record.name)) +} + +/** + * Download a multi-file selection as a zip. Fetched rather than navigated to, so a + * rejection — a document still compiling, an entry too large — surfaces as an error + * the caller can show in place instead of replacing the page with raw JSON. + */ +export async function triggerArchiveDownload(url: string): Promise { + // boundary-raw-fetch: binary zip download read as a blob, not a JSON contract response + const response = await fetch(url, { cache: 'no-store' }) + if (!response.ok) { + throw new Error(await downloadErrorMessage(response, 'Failed to download the selected files')) + } + + saveBlob(await response.blob(), fileNameFromDisposition(response, 'workspace-files.zip')) } From 003e75cb0452a6d05715f7a672c9edcbac8b990a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 16:43:14 -0700 Subject: [PATCH 11/12] fix(files): blame the entry when its cap ties the remaining budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A renderable document whose allowance exactly equalled the remaining budget was reported as the selection being too large. Its own ceiling was still what made it unshippable, and downloading it on its own is the way through, so the aggregate copy sent the user down a path that could not work. Attribution is now keyed off whether the entry has a cap of its own at all — only renderable documents do — and whether that cap was the binding one, ties included. --- .../workspaces/[id]/files/download/route.test.ts | 15 +++++++++++++++ .../api/workspaces/[id]/files/download/route.ts | 13 +++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts index 8b9208e00e1..db70c84ab09 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.test.ts @@ -137,6 +137,21 @@ describe('workspace files download route', () => { expect(body.error).not.toContain('Selected files total') }) + it('blames the entry when its render ceiling exactly equals the remaining budget', async () => { + // Declared at the full budget, so allowance === remaining and the caps tie. + const doc = { ...workspaceFile('f1', 'report.docx', 'folder-1'), size: 250 * 1024 * 1024 } + mockListWorkspaceFiles.mockResolvedValue([doc]) + mockFetchServableWorkspaceFileBuffer.mockRejectedValue( + new PayloadSizeLimitError({ label: 'servable file download', maxBytes: 1 }) + ) + + const response = await GET(requestFor('fileIds=f1'), context) + + expect(response.status).toBe(400) + // Downloading it on its own is still the way through, so name it. + expect((await response.json()).error).toContain('report.docx') + }) + it('blames the shared budget, not the entry, when the entry had no smaller cap', async () => { // .mp4 has no render headroom, so its cap is whatever is left of the budget. mockListWorkspaceFiles.mockResolvedValue([workspaceFile('f1', 'clip.mp4', 'folder-1')]) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index ec09af594c0..d7b01e84ec7 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -148,7 +148,10 @@ export const GET = withRouteHandler( async (file): Promise => { if (controller.signal.aborted) return skipped const remaining = Math.max(0, MAX_ZIP_DOWNLOAD_BYTES - renderedBytes) - const allowance = isRenderableDocumentName(file.name) + // Only renderable documents carry a cap of their own; everything else is + // bounded solely by what is left of the shared budget. + const renderable = isRenderableDocumentName(file.name) + const allowance = renderable ? Math.max(file.size, RENDERED_DOCUMENT_HEADROOM_BYTES) : remaining try { @@ -166,12 +169,14 @@ export const GET = withRouteHandler( // downgrade an actionable 400 into an opaque 500. if (error instanceof PayloadSizeLimitError) { controller.abort() - // Attributed to this entry only when its own allowance was the smaller - // of the two caps; otherwise the shared budget is what ran out. + // Attributed to this entry when its own cap was the binding one — ties + // included, since the entry's ceiling still made it unshippable and + // downloading it alone is the way through. Otherwise the budget ran out. return { ...skipped, overLimit: true, - overLimitEntry: allowance < remaining ? { name: file.name, allowance } : null, + overLimitEntry: + renderable && allowance <= remaining ? { name: file.name, allowance } : null, } } // Any other error from an already-aborted read is a consequence of the From 95942d29a4381f3df4c49c3f7cd79fa442f6ab59 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 27 Jul 2026 17:05:06 -0700 Subject: [PATCH 12/12] fix(files): read renderable documents one at a time to remove the memory regression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Files that serve exactly their declared size cannot push residency past the pre-download check, however many run at once. Only renderable documents can exceed what they declared, so they were the sole source of overshoot — and with five in flight the worst case was five headrooms above the budget, which is worse than the hard cap the raw-byte reads used to give. Those documents now read one at a time while everything else stays parallel, so the overshoot is a single entry. The per-file read is a named function the two passes share; ordering is preserved by writing results back by index. --- .../workspaces/[id]/files/download/route.ts | 132 ++++++++++-------- 1 file changed, 76 insertions(+), 56 deletions(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index d7b01e84ec7..b1c5b92cb11 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -9,6 +9,7 @@ import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { buildWorkspaceFileFolderPathMap, fetchServableWorkspaceFileBuffer, @@ -33,9 +34,9 @@ const MAX_ZIP_DOWNLOAD_BYTES = 250 * 1024 * 1024 */ const RENDERED_DOCUMENT_HEADROOM_BYTES = 50 * 1024 * 1024 /** - * Reads in flight cannot see each other's bytes until they land, so peak memory - * scales with this. Kept well below the shared default: the entries here are whole - * files held in memory at once, not rows. + * Fan-out for files that serve exactly their declared size. Their total is already + * bounded by the pre-download check, so concurrency cannot push residency past it. + * Renderable documents are read one at a time instead — see below. */ const ZIP_MATERIALIZE_CONCURRENCY = 5 @@ -116,13 +117,14 @@ export const GET = withRouteHandler( // Generated docs (docx/pptx/pdf/xlsx) store their generation source, not the // rendered binary, so bytes are resolved through the servable reader — a raw - // read ships source text under a `.docx` name. The rendered artifact can be far - // larger than the declared source size, so each read is capped at whatever is - // left of the budget, with office extensions additionally held to their declared - // size plus render headroom. Concurrent reads cannot see each other's bytes until - // they land, so that per-entry allowance and the concurrency limit are together - // what bound peak memory. Once the budget is blown or a read hard-fails, the - // abort flag stops reads that have not started yet. + // read ships source text under a `.docx` name. + // + // Only those documents can exceed the size they declared, so only they can push + // residency past the budget the pre-download check already enforced. They are + // therefore read one at a time, which keeps the overshoot to a single entry; + // everything else stays parallel because its bytes are exactly its declared size. + // Once the budget is blown or a read hard-fails, the abort flag stops reads that + // have not started yet. const controller = new AbortController() let renderedBytes = 0 @@ -142,55 +144,73 @@ export const GET = withRouteHandler( error: null, } - const downloads = await mapWithConcurrency( - filesToZip, - ZIP_MATERIALIZE_CONCURRENCY, - async (file): Promise => { - if (controller.signal.aborted) return skipped - const remaining = Math.max(0, MAX_ZIP_DOWNLOAD_BYTES - renderedBytes) - // Only renderable documents carry a cap of their own; everything else is - // bounded solely by what is left of the shared budget. - const renderable = isRenderableDocumentName(file.name) - const allowance = renderable - ? Math.max(file.size, RENDERED_DOCUMENT_HEADROOM_BYTES) - : remaining - try { - const { buffer } = await fetchServableWorkspaceFileBuffer(file, { - maxBytes: Math.min(remaining, allowance), - signal: controller.signal, - }) - renderedBytes += buffer.length - const overLimit = renderedBytes > MAX_ZIP_DOWNLOAD_BYTES - if (overLimit) controller.abort() - return { ...skipped, buffer, overLimit } - } catch (error) { - // Recorded even when another worker already aborted: a size rejection - // describes this file, so losing it to someone else's cancellation would - // downgrade an actionable 400 into an opaque 500. - if (error instanceof PayloadSizeLimitError) { - controller.abort() - // Attributed to this entry when its own cap was the binding one — ties - // included, since the entry's ceiling still made it unshippable and - // downloading it alone is the way through. Otherwise the budget ran out. - return { - ...skipped, - overLimit: true, - overLimitEntry: - renderable && allowance <= remaining ? { name: file.name, allowance } : null, - } + const readEntry = async ( + file: WorkspaceFileRecord, + renderable: boolean + ): Promise => { + if (controller.signal.aborted) return skipped + const remaining = Math.max(0, MAX_ZIP_DOWNLOAD_BYTES - renderedBytes) + // Only renderable documents carry a cap of their own; everything else is + // bounded solely by what is left of the shared budget. + const allowance = renderable + ? Math.max(file.size, RENDERED_DOCUMENT_HEADROOM_BYTES) + : remaining + try { + const { buffer } = await fetchServableWorkspaceFileBuffer(file, { + maxBytes: Math.min(remaining, allowance), + signal: controller.signal, + }) + renderedBytes += buffer.length + const overLimit = renderedBytes > MAX_ZIP_DOWNLOAD_BYTES + if (overLimit) controller.abort() + return { ...skipped, buffer, overLimit } + } catch (error) { + // Recorded even when another worker already aborted: a size rejection + // describes this file, so losing it to someone else's cancellation would + // downgrade an actionable 400 into an opaque 500. + if (error instanceof PayloadSizeLimitError) { + controller.abort() + // Attributed to this entry when its own cap was the binding one — ties + // included, since the entry's ceiling still made it unshippable and + // downloading it alone is the way through. Otherwise the budget ran out. + return { + ...skipped, + overLimit: true, + overLimitEntry: + renderable && allowance <= remaining ? { name: file.name, allowance } : null, } - // Any other error from an already-aborted read is a consequence of the - // cancellation, not a cause. Checked before this worker aborts anything so - // the worker that actually failed still records its own error. - if (controller.signal.aborted) return skipped - // A pending artifact is worth reporting in full, so keep resolving the - // rest of the selection; anything else dooms the request. - const pending = isDocNotReadyError(error) - if (!pending) controller.abort() - return { ...skipped, pendingName: pending ? file.name : null, error } } + // Any other error from an already-aborted read is a consequence of the + // cancellation, not a cause. Checked before this worker aborts anything so + // the worker that actually failed still records its own error. + if (controller.signal.aborted) return skipped + // A pending artifact is worth reporting in full, so keep resolving the + // rest of the selection; anything else dooms the request. + const pending = isDocNotReadyError(error) + if (!pending) controller.abort() + return { ...skipped, pendingName: pending ? file.name : null, error } } - ) + } + + const outcomes = new Array(filesToZip.length) + const indices = filesToZip.map((_, index) => index) + await Promise.all([ + mapWithConcurrency( + indices.filter((index) => isRenderableDocumentName(filesToZip[index].name)), + 1, + async (index) => { + outcomes[index] = await readEntry(filesToZip[index], true) + } + ), + mapWithConcurrency( + indices.filter((index) => !isRenderableDocumentName(filesToZip[index].name)), + ZIP_MATERIALIZE_CONCURRENCY, + async (index) => { + outcomes[index] = await readEntry(filesToZip[index], false) + } + ), + ]) + const downloads = outcomes // Size first: the request cannot succeed at any size-adjacent retry, and a // descriptive 400 beats an opaque 500 raised by whatever the abort cancelled.