diff --git a/apps/docs/openapi-v2-files-audit.json b/apps/docs/openapi-v2-files-audit.json index 956ffba38ed..66a916f5767 100644 --- a/apps/docs/openapi-v2-files-audit.json +++ b/apps/docs/openapi-v2-files-audit.json @@ -745,7 +745,7 @@ "get": { "operationId": "readFileText", "summary": "Read File Text", - "description": "Extract text without changing the file. Use Unzip File to unpack archives or Download File for original bytes. Unsupported types return `400`, compiling documents return `409`, and oversized files return `413`. `degraded: true` indicates incomplete or synthesized text, including some legacy `.doc` and `.ppt` results; `truncated: true` indicates a parser limit.\n\nOAuth scope: `api:read`.", + "description": "Extract text without changing the file. Use Unzip File to unpack archives or Download File for original bytes. Unsupported types return `400`, compiling documents return `409`, and oversized files return `413`. `degraded: true` indicates incomplete or synthesized text, such as the legacy `.pptx` fallback; `truncated: true` indicates a parser limit.\n\nOAuth scope: `api:read`.", "x-sim-operation": "files.read_content", "x-oauth-scope": "api:read", "tags": ["Files"], diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data.test.ts index ea6108923db..17f2cf993fd 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data.test.ts @@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import * as XLSX from 'xlsx' import { readXlsxPreviewData, + readXlsxWorkbook, XLSX_MAX_COLUMNS, XLSX_MAX_ROWS, } from '@/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data' @@ -26,9 +27,11 @@ describe('readXlsxPreviewData', () => { const result = readXlsxPreviewData(XLSX, sheet) const options = toJson.mock.calls[0][1] as { range: { s: { r: number }; e: { r: number } } + raw?: boolean } expect(options.range.e.r - options.range.s.r).toBe(XLSX_MAX_ROWS) + expect(options.raw).toBe(false) expect(result.headers).toEqual(['header-a', 'header-b']) expect(result.rows).toHaveLength(XLSX_MAX_ROWS) expect(result.rows.slice(0, 2)).toEqual([ @@ -71,4 +74,37 @@ describe('readXlsxPreviewData', () => { expect(result.rowTruncated).toBe(false) expect(result.columnTruncated).toBe(true) }) + + /** + * Built through the viewer's own read path rather than by hand-setting `z`, + * so the assertions cover the read options as well as the conversion. + */ + function typedWorkbook(): ArrayBuffer { + const sheet = XLSX.utils.aoa_to_sheet([['Issued', 'Rate', 'Card', 'Elapsed']]) + sheet.A2 = { t: 'd', v: new Date(Date.UTC(2026, 2, 4)), z: 'm/d/yyyy' } + sheet.B2 = { t: 'n', v: 0.2, z: '0%' } + sheet.C2 = { t: 'n', v: 4111111111111111 } + sheet.D2 = { t: 'n', v: 1.25, z: '[h]:mm' } + sheet['!ref'] = 'A1:D2' + const book = XLSX.utils.book_new() + XLSX.utils.book_append_sheet(book, sheet, 'Ledger') + const bytes = XLSX.write(book, { type: 'buffer', bookType: 'xlsx' }) as Buffer + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer + } + + it('shows display text rather than stored values', () => { + const workbook = readXlsxWorkbook(XLSX, typedWorkbook()) + + const result = readXlsxPreviewData(XLSX, workbook.Sheets.Ledger) + + expect(result.rows).toEqual([['2026-03-04', '20%', '4111111111111111', '30:00']]) + }) + + it('reads the workbook with the display-text options', () => { + const read = vi.fn(XLSX.read) + + readXlsxWorkbook({ read, utils: XLSX.utils }, typedWorkbook()) + + expect(read.mock.calls[0][1]).toMatchObject({ type: 'array', cellDates: true, cellNF: true }) + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data.ts index a661d7f3cf7..14e8f30841f 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data.ts @@ -1,10 +1,25 @@ -import type { WorkSheet } from 'xlsx' +import type { WorkBook, WorkSheet } from 'xlsx' +import { + normalizeSheetDisplayText, + SHEET_DISPLAY_READ_OPTIONS, +} from '@/lib/file-parsers/sheet-display-text' export const XLSX_MAX_ROWS = 1_000 export const XLSX_MAX_COLUMNS = 200 interface XlsxModule { - utils: Pick + read: typeof import('xlsx').read + utils: Pick +} + +/** + * Reads a workbook for preview with the options that make its cells carry + * display text: without `cellDates` a date arrives as a bare serial and + * without `cellNF` no cell has a format, so every rendered `w` would be + * overwritten as a General number. + */ +export function readXlsxWorkbook(XLSX: XlsxModule, data: ArrayBuffer): WorkBook { + return XLSX.read(new Uint8Array(data), { type: 'array', ...SHEET_DISPLAY_READ_OPTIONS }) } interface XlsxPreviewData { @@ -18,12 +33,21 @@ export function readXlsxPreviewData(XLSX: XlsxModule, sheet: WorkSheet): XlsxPre const declaredRange = XLSX.utils.decode_range(sheet['!ref'] || 'A1') const lastPreviewRow = Math.min(declaredRange.e.r, declaredRange.s.r + XLSX_MAX_ROWS) const lastPreviewColumn = Math.min(declaredRange.e.c, declaredRange.s.c + XLSX_MAX_COLUMNS - 1) + const previewRange = { + s: declaredRange.s, + e: { r: lastPreviewRow, c: lastPreviewColumn }, + } + + /** + * Shown as the text a user sees in Excel: `raw: false` emits each cell's + * formatted text, so a sheet read through {@link readXlsxWorkbook} shows a + * date as ISO text and `20%` rather than a serial and `0.2`. + */ + normalizeSheetDisplayText(sheet, previewRange, XLSX.utils) const previewRows = XLSX.utils.sheet_to_json(sheet, { header: 1, - range: { - s: declaredRange.s, - e: { r: lastPreviewRow, c: lastPreviewColumn }, - }, + raw: false, + range: previewRange, }) return { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview.tsx index 431b5f213bc..11e4316c92e 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview.tsx @@ -10,6 +10,7 @@ import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { useHorizontalWheelScroll } from '@/app/workspace/[workspaceId]/files/components/file-viewer/use-horizontal-wheel-scroll' import { readXlsxPreviewData, + readXlsxWorkbook, XLSX_MAX_COLUMNS, XLSX_MAX_ROWS, } from '@/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data' @@ -55,7 +56,7 @@ export const XlsxPreview = memo(function XlsxPreview({ setRenderError(null) await assertOoxmlPreviewWithinLimits(data) const XLSX = await import('xlsx') - const workbook = XLSX.read(new Uint8Array(data), { type: 'array' }) + const workbook = readXlsxWorkbook(XLSX, data) if (!cancelled) { workbookRef.current = workbook setSheetNames(workbook.SheetNames) diff --git a/apps/sim/connectors/azure-devops/azure-devops.ts b/apps/sim/connectors/azure-devops/azure-devops.ts index 54e9719ea7b..1277e9fff9c 100644 --- a/apps/sim/connectors/azure-devops/azure-devops.ts +++ b/apps/sim/connectors/azure-devops/azure-devops.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' +import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { azureDevopsConnectorMeta } from '@/connectors/azure-devops/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -1182,7 +1183,7 @@ async function getFileDocument( return null } - const content = buffer.toString('utf8') + const content = decodeTextBuffer(buffer).text if (!content.trim()) return null const title = path.split('/').filter(Boolean).pop() || path diff --git a/apps/sim/connectors/bitbucket/bitbucket.test.ts b/apps/sim/connectors/bitbucket/bitbucket.test.ts index e36b329a4f2..3c43cdb8850 100644 --- a/apps/sim/connectors/bitbucket/bitbucket.test.ts +++ b/apps/sim/connectors/bitbucket/bitbucket.test.ts @@ -774,7 +774,7 @@ describe('bitbucket getDocument', () => { expect(doc?.skippedReason).toMatch(/Binary/) }) - it('surfaces non-UTF-8 source as skipped instead of indexing replacement characters', async () => { + it('decodes non-UTF-8 source as Windows-1252 instead of skipping or indexing replacement characters', async () => { mockApi([ [ /\/src\/[a-f0-9]+\/latin1\.txt$/, @@ -784,8 +784,9 @@ describe('bitbucket getDocument', () => { const doc = await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'file:latin1.txt', {}) - expect(doc?.skippedReason).toMatch(/Non-UTF-8/) - expect(doc?.content).toBe('') + expect(doc?.skippedReason).toBeUndefined() + expect(doc?.content).toContain('café') + expect(doc?.content).not.toContain('\uFFFD') }) it('returns null for a file the ref no longer carries', async () => { diff --git a/apps/sim/connectors/bitbucket/bitbucket.ts b/apps/sim/connectors/bitbucket/bitbucket.ts index b0d5239ee9a..03b3d38d3f6 100644 --- a/apps/sim/connectors/bitbucket/bitbucket.ts +++ b/apps/sim/connectors/bitbucket/bitbucket.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' +import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { bitbucketConnectorMeta } from '@/connectors/bitbucket/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -81,7 +82,6 @@ const BINARY_SNIFF_BYTES = 8000 */ const MAX_TREE_DEPTH = 5 const BINARY_SKIP_REASON = 'Binary file was not indexed' -const NON_UTF8_SKIP_REASON = 'Non-UTF-8 file was not indexed' /** * Bitbucket answers a raw read of an LFS-managed file with a 301 to Atlassian's * media services platform. The connector deliberately surfaces the file as @@ -1240,13 +1240,7 @@ export const bitbucketConnector: ConnectorConfig = { return markSkipped(stub, BINARY_SKIP_REASON) } - let text: string - try { - text = new TextDecoder('utf-8', { fatal: true }).decode(buffer) - } catch { - logger.info('Skipping non-UTF-8 Bitbucket file', { path }) - return markSkipped(stub, NON_UTF8_SKIP_REASON) - } + const text = decodeTextBuffer(buffer).text const body = composeBody(stub.title, text) if (!body.trim()) return null diff --git a/apps/sim/connectors/box/box.ts b/apps/sim/connectors/box/box.ts index 3f36323905f..d5f453ea759 100644 --- a/apps/sim/connectors/box/box.ts +++ b/apps/sim/connectors/box/box.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { boxConnectorMeta } from '@/connectors/box/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -132,7 +133,6 @@ const REPRESENTATION_EXTENSIONS = new Set([ 'odt', 'otp', 'pdf', - 'ppt', 'pptx', 'rtf', 'vi', @@ -319,7 +319,7 @@ async function fetchPlainTextContent( extension: string ): Promise { const buffer = await downloadWithinLimit(`${BOX_API_BASE}/files/${fileId}/content`, accessToken) - const text = buffer.toString('utf8') + const { text } = decodeTextBuffer(buffer) return HTML_EXTENSIONS.has(extension) ? htmlToPlainText(text) : text } @@ -347,7 +347,7 @@ async function fetchExtractedText( urlTemplate.replace('{+asset_path}', ''), accessToken ) - return buffer.toString('utf8') + return decodeTextBuffer(buffer).text } if (state === 'error' || !infoUrl) return null if (attempt === REPRESENTATION_POLL_ATTEMPTS) break diff --git a/apps/sim/connectors/databricks/databricks.ts b/apps/sim/connectors/databricks/databricks.ts index eb127d8dac8..88f1adfdc80 100644 --- a/apps/sim/connectors/databricks/databricks.ts +++ b/apps/sim/connectors/databricks/databricks.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { truncate } from '@sim/utils/string' import { validateDatabricksWorkspaceHost } from '@/lib/core/security/input-validation' +import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DATABRICKS_CONTENT_TYPES, @@ -586,7 +587,7 @@ async function exportNotebook( return { skippedReason: sizeLimitSkipReason(CONNECTOR_MAX_FILE_BYTES) } } - return { content: decoded.toString('utf8') } + return { content: decodeTextBuffer(decoded).text } } /** diff --git a/apps/sim/connectors/dropbox/dropbox.ts b/apps/sim/connectors/dropbox/dropbox.ts index fbeb6c5d4a7..132e6deb4ea 100644 --- a/apps/sim/connectors/dropbox/dropbox.ts +++ b/apps/sim/connectors/dropbox/dropbox.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { dropboxConnectorMeta } from '@/connectors/dropbox/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -147,7 +148,7 @@ async function downloadFileContent( throw new ConnectorFileTooLargeError(MAX_FILE_SIZE) } - const text = buffer.toString('utf8') + const { text } = decodeTextBuffer(buffer) return isHtml ? htmlToPlainText(text) : text } diff --git a/apps/sim/connectors/github/github.ts b/apps/sim/connectors/github/github.ts index 7a2c71d0218..d1f71f951ae 100644 --- a/apps/sim/connectors/github/github.ts +++ b/apps/sim/connectors/github/github.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { z } from 'zod' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { type RetryOptions, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { parseGitHubRepository } from '@/lib/oauth/github-repository' import { githubConnectorMeta } from '@/connectors/github/meta' @@ -296,7 +297,7 @@ async function fetchBlobContent( throw new ConnectorFileTooLargeError(maxBytes) } if (isBinaryBuffer(buffer)) return null - return buffer.toString('utf8') + return decodeTextBuffer(buffer).text } /** Resolves links within one snapshot; Contents can truncate dereferenced targets at 1 MiB. */ diff --git a/apps/sim/connectors/gitlab/gitlab.ts b/apps/sim/connectors/gitlab/gitlab.ts index def74730772..95c794b2a36 100644 --- a/apps/sim/connectors/gitlab/gitlab.ts +++ b/apps/sim/connectors/gitlab/gitlab.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { isPlainRecord } from '@sim/utils/object' import type { SecureFetchResponse } from '@/lib/core/security/input-validation.server' +import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { gitlabConnectorMeta } from '@/connectors/gitlab/meta' @@ -435,7 +436,7 @@ function fileToDocument( return skipped(sizeLimitSkipReason(MAX_FILE_SIZE), buffer.byteLength) } - const content = buffer.toString('utf8') + const content = decodeTextBuffer(buffer).text const body = composeBody(title, content) if (!body.trim()) return null diff --git a/apps/sim/connectors/google-drive/google-drive.ts b/apps/sim/connectors/google-drive/google-drive.ts index cd551cd9066..a6100525a0d 100644 --- a/apps/sim/connectors/google-drive/google-drive.ts +++ b/apps/sim/connectors/google-drive/google-drive.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { isPlainRecord } from '@sim/utils/object' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { type DrivePermission, driveFileAcl, @@ -212,10 +213,10 @@ async function fetchFilePayload( }, } } - return { content: bytes.toString('utf8'), mimeType: 'text/plain' } + return { content: decodeTextBuffer(bytes).text, mimeType: 'text/plain' } } if (file.mimeType === 'text/html') { - const html = (await downloadFile(accessToken, file.id, resourceKey)).toString('utf8') + const html = decodeTextBuffer(await downloadFile(accessToken, file.id, resourceKey)).text return { content: htmlToPlainText(html), mimeType: 'text/plain' } } const raw = rawFileType(file) @@ -228,7 +229,7 @@ async function fetchFilePayload( } if (isSupportedTextFile(file.mimeType)) { return { - content: (await downloadFile(accessToken, file.id, resourceKey)).toString('utf8'), + content: decodeTextBuffer(await downloadFile(accessToken, file.id, resourceKey)).text, mimeType: 'text/plain', } } diff --git a/apps/sim/connectors/s3/s3.ts b/apps/sim/connectors/s3/s3.ts index 351a553ace5..5e07d171339 100644 --- a/apps/sim/connectors/s3/s3.ts +++ b/apps/sim/connectors/s3/s3.ts @@ -4,6 +4,7 @@ import { isLoopbackHostname } from '@sim/security/ssrf' import { getErrorMessage, toError } from '@sim/utils/errors' import { truncate } from '@sim/utils/string' import { isHosted } from '@/lib/core/config/env-flags' +import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { s3ConnectorMeta } from '@/connectors/s3/meta' @@ -658,7 +659,7 @@ export const s3Connector: ConnectorConfig = { sizeLimitSkipReason(MAX_FILE_SIZE) ) } - const raw = body.toString('utf-8') + const raw = decodeTextBuffer(body).text const content = MARKUP_EXTENSIONS.has(getExtension(key)) ? htmlToPlainText(raw) : raw if (!content.trim()) return null diff --git a/apps/sim/connectors/sftp/sftp.ts b/apps/sim/connectors/sftp/sftp.ts index 6d03efc677f..d255f127b48 100644 --- a/apps/sim/connectors/sftp/sftp.ts +++ b/apps/sim/connectors/sftp/sftp.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { type Attributes, type Client, type SFTPWrapper, utils as ssh2Utils } from 'ssh2' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { createSftpConnection, getFileType, @@ -779,7 +780,7 @@ export const sftpConnector: ConnectorConfig = { return markSkipped(stub, 'File appears to be binary and was not indexed') } - const raw = buffer.toString('utf-8') + const raw = decodeTextBuffer(buffer).text const content = HTML_EXTENSIONS.has(getExtension(remotePath)) ? htmlToPlainText(raw) : raw if (!content.trim()) return null diff --git a/apps/sim/connectors/utils.test.ts b/apps/sim/connectors/utils.test.ts index 5090157e222..e28549c7262 100644 --- a/apps/sim/connectors/utils.test.ts +++ b/apps/sim/connectors/utils.test.ts @@ -1498,19 +1498,15 @@ describe('htmlToPlainText entity decoding', () => { describe('isIndexableConnectorFile', () => { it('accepts the Office and PDF formats the knowledge base can parse', () => { - for (const name of [ - 'sop.pdf', - 'sop.doc', - 'sop.docx', - 'sheet.xls', - 'sheet.xlsx', - 'deck.ppt', - 'deck.pptx', - ]) { + for (const name of ['sop.pdf', 'sop.doc', 'sop.docx', 'sheet.xls', 'sheet.xlsx', 'deck.pptx']) { expect(isIndexableConnectorFile(name)).toBe(true) } }) + it('refuses legacy .ppt up front because no parser reads it', () => { + expect(isIndexableConnectorFile('deck.ppt')).toBe(false) + }) + it('still accepts the plain-text formats connectors already synced', () => { for (const name of ['a.txt', 'a.md', 'a.html', 'a.htm', 'a.csv', 'a.log', 'a.tsv', 'a.rst']) { expect(isIndexableConnectorFile(name)).toBe(true) @@ -1578,6 +1574,30 @@ describe('extractConnectorText', () => { it('leaves whitespace-only content alone for the caller to reject', () => { expect(extractConnectorText(Buffer.from(' '), 'blank.txt')).toBe(' ') }) + + it('decodes a Latin-1 file as Windows-1252 instead of indexing mojibake', () => { + expect(extractConnectorText(Buffer.from('Caf\xe9 \xa3 42', 'latin1'), 'notes.txt')).toBe( + 'Café £ 42' + ) + }) + + it('strips a UTF-8 BOM', () => { + expect( + extractConnectorText( + Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('a,b')]), + 'data.csv' + ) + ).toBe('a,b') + }) + + it('decodes UTF-16 with a BOM', () => { + expect( + extractConnectorText( + Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from('

Hällo

', 'utf16le')]), + 'page.html' + ) + ).toBe('Hällo') + }) }) describe('pipelineParsedMimeType', () => { diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts index 3a0e4d78de9..88aee7614f7 100644 --- a/apps/sim/connectors/utils.ts +++ b/apps/sim/connectors/utils.ts @@ -4,6 +4,7 @@ import { isPayloadSizeLimitError, readResponseToBufferWithLimit, } from '@/lib/core/utils/stream-limits' +import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { MAX_FILE_SIZE as KB_DOCUMENT_MAX_BYTES } from '@/lib/uploads/utils/validation' import type { ExternalDocument } from '@/connectors/types' @@ -427,7 +428,6 @@ export const PIPELINE_PARSED_MIME_TYPES: ReadonlyMap = new Map([ ['xlsm', 'application/vnd.ms-excel.sheet.macroEnabled.12'], ['xlsb', 'application/vnd.ms-excel.sheet.binary.macroEnabled.12'], ['xltx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.template'], - ['ppt', 'application/vnd.ms-powerpoint'], ['pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation'], ['pptm', 'application/vnd.ms-powerpoint.presentation.macroEnabled.12'], ['potx', 'application/vnd.openxmlformats-officedocument.presentationml.template'], @@ -497,16 +497,18 @@ export function pipelineParsedMimeType(fileName: string): string | undefined { * * Only for formats that are already text — anything the shared parsers handle is * delivered to them verbatim instead, via {@link pipelineParsedMimeType}. HTML is - * additionally reduced to plain text; everything else is a UTF-8 decode. + * additionally reduced to plain text; everything else is decoded with the shared + * BOM/UTF-8/Windows-1252 detection so a Latin-1 file never indexes as mojibake. */ export function extractConnectorText(buffer: Buffer, fileName: string): string { const extension = connectorFileExtension(fileName) + const { text } = decodeTextBuffer(buffer) if (extension === 'html' || extension === 'htm') { - return htmlToPlainText(buffer.toString('utf8')) + return htmlToPlainText(text) } - return buffer.toString('utf8') + return text } /** diff --git a/apps/sim/lib/api/contracts/v2/openapi/files-audit.ts b/apps/sim/lib/api/contracts/v2/openapi/files-audit.ts index ceb223ae4d2..3515fa38708 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/files-audit.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/files-audit.ts @@ -393,7 +393,7 @@ const declaredRoutes = [ operationId: 'readFileText', summary: 'Read File Text', description: - 'Extract text without changing the file. Use Unzip File to unpack archives or Download File for original bytes. Unsupported types return `400`, compiling documents return `409`, and oversized files return `413`. `degraded: true` indicates incomplete or synthesized text, including some legacy `.doc` and `.ppt` results; `truncated: true` indicates a parser limit.', + 'Extract text without changing the file. Use Unzip File to unpack archives or Download File for original bytes. Unsupported types return `400`, compiling documents return `409`, and oversized files return `413`. `degraded: true` indicates incomplete or synthesized text, such as the legacy `.pptx` fallback; `truncated: true` indicates a parser limit.', errors: [...RESOURCE_CONFLICT_ERRORS, 'PayloadTooLarge'], success: { description: 'The extracted text and its extraction-quality flags.' }, }), diff --git a/apps/sim/lib/copilot/vfs/file-reader.test.ts b/apps/sim/lib/copilot/vfs/file-reader.test.ts index 531240786b6..8d48df797bb 100644 --- a/apps/sim/lib/copilot/vfs/file-reader.test.ts +++ b/apps/sim/lib/copilot/vfs/file-reader.test.ts @@ -6,13 +6,17 @@ import { randomFillSync } from 'node:crypto' import { crc32 } from 'node:zlib' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { fetchWorkspaceFileBuffer } = vi.hoisted(() => ({ +const { fetchWorkspaceFileBuffer, mockParseBuffer } = vi.hoisted(() => ({ fetchWorkspaceFileBuffer: vi.fn(), + mockParseBuffer: vi.fn(), })) vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ fetchWorkspaceFileBuffer, })) +vi.mock('@/lib/file-parsers', () => ({ + parseBuffer: mockParseBuffer, +})) import { MAX_IMAGE_READ_BYTES, @@ -21,6 +25,7 @@ import { MAX_TEXT_READ_BYTES, readFileRecord, } from '@/lib/copilot/vfs/file-reader' +import { readPlaceholder } from '@/lib/copilot/vfs/read-placeholders' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { MAX_TRANSCODE_INPUT_BYTES } from '@/lib/uploads/server/heic' @@ -201,3 +206,46 @@ describe('readFileRecord', () => { SHARP_TEST_TIMEOUT_MS ) }) + +describe('readFileRecord parseable documents', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + function documentRecord(name: string, type: string, size: number) { + return { ...imageRecord(name, size, type), id: 'wf_doc' } + } + + it('returns the parsed text of a document', async () => { + fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('bytes')) + mockParseBuffer.mockResolvedValue({ + content: 'Quarterly review\nSecond line', + metadata: { extractionMethod: 'word-extractor' }, + }) + + const result = await readFileRecord(documentRecord('review.doc', 'application/msword', 5)) + + expect(result).toEqual({ content: 'Quarterly review\nSecond line', totalLines: 2 }) + }) + + /** + * A parser that could only scrape bytes flags the result `degraded`; that must + * reach the model as the could-not-parse placeholder, never as file content. + */ + it('reports degraded parser output as could-not-parse instead of handing it to the model', async () => { + fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('bytes')) + mockParseBuffer.mockResolvedValue({ + content: '[Content_Types].xml _rels/.rels theme/theme/themeManager.xml', + metadata: { degraded: true, warning: 'Basic text extraction used' }, + }) + + const result = await readFileRecord( + documentRecord('deck.pptx', 'application/vnd.ms-powerpoint', 5) + ) + + expect(result).toEqual( + readPlaceholder.couldNotParse('deck.pptx', 'application/vnd.ms-powerpoint', 5) + ) + expect(result?.content).not.toContain('[Content_Types].xml') + }) +}) diff --git a/apps/sim/lib/copilot/vfs/file-reader.ts b/apps/sim/lib/copilot/vfs/file-reader.ts index 75bb5c5bdf0..ae00204fa8f 100644 --- a/apps/sim/lib/copilot/vfs/file-reader.ts +++ b/apps/sim/lib/copilot/vfs/file-reader.ts @@ -104,7 +104,7 @@ const TEXT_TYPES = new Set([ 'application/javascript', ]) -const PARSEABLE_EXTENSIONS = new Set(['pdf', 'docx', 'doc', 'xlsx', 'xls', 'pptx', 'ppt']) +const PARSEABLE_EXTENSIONS = new Set(['pdf', 'docx', 'doc', 'xlsx', 'xls', 'pptx']) export function isReadableFileType(contentType: string): boolean { return TEXT_TYPES.has(contentType) || contentType.startsWith('text/') @@ -587,6 +587,10 @@ export async function readFileRecord( try { const { parseBuffer } = await import('@/lib/file-parsers') const result = await parseBuffer(fetched.buffer, ext) + if (result.metadata?.degraded === true) { + /** Scraped ZIP internals or placeholder prose, not the document's text. */ + throw new Error(result.metadata.warning ?? 'Parser returned degraded output') + } const content = result.content || '' const lines = content.split('\n').length span.setAttributes({ diff --git a/apps/sim/lib/file-parsers/csv-parser.ts b/apps/sim/lib/file-parsers/csv-parser.ts index 6d4c6c9ec23..bec222e1f95 100644 --- a/apps/sim/lib/file-parsers/csv-parser.ts +++ b/apps/sim/lib/file-parsers/csv-parser.ts @@ -1,10 +1,16 @@ -import { createReadStream, existsSync } from 'fs' +import { existsSync } from 'fs' +import { readFile } from 'fs/promises' import { Readable } from 'stream' import { createLogger } from '@sim/logger' import { type Options, parse } from 'csv-parse' import { FileParserError } from '@/lib/file-parsers/errors' import type { FileParseResult, FileParser } from '@/lib/file-parsers/types' -import { sanitizeTextForUTF8, truncationNotice } from '@/lib/file-parsers/utils' +import { + type DecodedText, + decodeTextBuffer, + sanitizeTextForUTF8, + truncationNotice, +} from '@/lib/file-parsers/utils' const logger = createLogger('CsvParser') @@ -12,10 +18,17 @@ const CONFIG = { MAX_PREVIEW_ROWS: 1000, // Only keep first 1000 rows for preview MAX_SAMPLE_ROWS: 100, // Sample for metadata MAX_ERRORS: 100, // Stop after 100 errors - STREAM_CHUNK_SIZE: 16384, // 16KB chunks for streaming } export class CsvParser implements FileParser { + /** + * Reads the whole file before parsing rather than streaming 16 KB chunks: + * encoding detection needs the complete byte sequence (a BOM-less UTF-16 or + * Windows-1252 file cannot be recognized per chunk, and a multi-byte UTF-8 + * sequence split across chunk boundaries would be misread). The upload size + * caps already bound the file, and `parseBuffer` — the production path — + * always held the full buffer. + */ async parseFile(filePath: string): Promise { if (!filePath) { throw new Error('No file path provided') @@ -25,11 +38,7 @@ export class CsvParser implements FileParser { throw new Error(`File not found: ${filePath}`) } - const stream = createReadStream(filePath, { - highWaterMark: CONFIG.STREAM_CHUNK_SIZE, - }) - - return this.parseStream(stream) + return this.parseBuffer(await readFile(filePath)) } async parseBuffer(buffer: Buffer): Promise { @@ -38,14 +47,18 @@ export class CsvParser implements FileParser { `Parsing CSV buffer, size: ${bufferSize} bytes (${(bufferSize / 1024 / 1024).toFixed(2)} MB)` ) + const decoded = decodeTextBuffer(buffer) const stream = new Readable({ read() {} }) - stream.push(buffer) + stream.push(decoded.text) stream.push(null) - return this.parseStream(stream) + return this.parseStream(stream, decoded) } - private parseStream(inputStream: NodeJS.ReadableStream): Promise { + private parseStream( + inputStream: NodeJS.ReadableStream, + decoded: DecodedText + ): Promise { return new Promise((resolve, reject) => { let rowCount = 0 let errorCount = 0 @@ -145,6 +158,8 @@ export class CsvParser implements FileParser { errors: errors.slice(0, 10), truncated: rowCount > CONFIG.MAX_PREVIEW_ROWS, sampledData: sampledRows, + encoding: decoded.encoding, + ...(decoded.warning ? { warning: decoded.warning } : {}), }, }) } diff --git a/apps/sim/lib/file-parsers/doc-parser.test.ts b/apps/sim/lib/file-parsers/doc-parser.test.ts index c7ed3cfe557..85939f37d04 100644 --- a/apps/sim/lib/file-parsers/doc-parser.test.ts +++ b/apps/sim/lib/file-parsers/doc-parser.test.ts @@ -3,11 +3,13 @@ */ import JSZip from 'jszip' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { FileParserError } from '@/lib/file-parsers/errors' import { ZipBombError } from '@/lib/file-parsers/ooxml-limits' -const { mockParseOfficeText, mockExtractRawText } = vi.hoisted(() => ({ +const { mockParseOfficeText, mockExtractRawText, mockWordExtract } = vi.hoisted(() => ({ mockParseOfficeText: vi.fn(), mockExtractRawText: vi.fn(), + mockWordExtract: vi.fn(), })) vi.mock('@/lib/file-parsers/officeparser-module', () => ({ @@ -17,11 +19,40 @@ vi.mock('mammoth', () => ({ default: { extractRawText: mockExtractRawText }, extractRawText: mockExtractRawText, })) +vi.mock('word-extractor', () => ({ + default: class WordExtractor { + extract(source: Buffer) { + return mockWordExtract(source) + } + }, +})) import { DocParser } from '@/lib/file-parsers/doc-parser' const CENTRAL_DIRECTORY_HEADER_SIGNATURE = 0x02014b50 +interface WordSections { + body?: string + headers?: string + footers?: string + footnotes?: string + endnotes?: string + textboxes?: string +} + +/** The accessor surface of word-extractor's `Document`, with empty sections by default. */ +function wordDocument(sections: WordSections) { + return { + getBody: () => sections.body ?? '', + getHeaders: () => sections.headers ?? '', + getFooters: () => sections.footers ?? '', + getFootnotes: () => sections.footnotes ?? '', + getEndnotes: () => sections.endnotes ?? '', + getAnnotations: () => '', + getTextboxes: () => sections.textboxes ?? '', + } +} + /** * Build a small OOXML-shaped archive whose central directory *declares* a huge * uncompressed size. The guard reads declared sizes without inflating anything, @@ -68,9 +99,11 @@ describe('DocParser.parseBuffer', () => { }) it('rejects a .doc that under-declares its uncompressed size', async () => { - // Declared sizes alone put this under every limit; officeparser and mammoth - // only notice the mismatch after inflating the entry in full, so the guard - // has to catch it before either library sees the buffer. + /** + * Declared sizes alone put this under every limit; officeparser and mammoth + * only notice the mismatch after inflating the entry in full, so the guard + * has to catch it before either library sees the buffer. + */ const zip = new JSZip() zip.file('word/document.xml', 'A'.repeat(4 * 1024 * 1024)) const honest = (await zip.generateAsync({ @@ -112,16 +145,114 @@ describe('DocParser.parseBuffer', () => { const result = await new DocParser().parseBuffer(buffer) expect(result.content).toBe('hello') - expect(result.metadata.extractionMethod).toBe('officeparser') + expect(result.metadata?.extractionMethod).toBe('officeparser') + expect(mockWordExtract).not.toHaveBeenCalled() + }) + + it('reports an OOXML .doc with no text as no_extractable_text rather than scraping it', async () => { + const zip = new JSZip() + zip.file('word/document.xml', '') + const buffer = await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }) + mockParseOfficeText.mockResolvedValue('') + mockExtractRawText.mockResolvedValue({ value: '', messages: [] }) + + const error = await new DocParser().parseBuffer(buffer).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(FileParserError) + expect(error).toMatchObject({ code: 'no_extractable_text' }) }) - it('no-ops the guard for a legacy OLE .doc and parses it', async () => { - mockParseOfficeText.mockResolvedValue('legacy doc text') + it('reads a legacy OLE .doc through word-extractor and joins its sections', async () => { + mockWordExtract.mockResolvedValue( + wordDocument({ + body: 'Body paragraph “quoted”', + headers: 'Running header', + footers: 'Page footer', + footnotes: '', + endnotes: 'An endnote', + textboxes: 'Pull quote in a text box', + }) + ) const result = await new DocParser().parseBuffer(buildLegacyOleDoc()) - expect(mockParseOfficeText).toHaveBeenCalledOnce() - expect(result.content).toBe('legacy doc text') + expect(mockWordExtract).toHaveBeenCalledOnce() + expect(mockParseOfficeText).not.toHaveBeenCalled() + expect(result.content).toBe( + 'Body paragraph “quoted”\n\nRunning header\n\nPage footer\n\nAn endnote\n\nPull quote in a text box' + ) + expect(result.metadata).toMatchObject({ + extractionMethod: 'word-extractor', + degraded: false, + characterCount: result.content.length, + }) + }) + + it('maps a Word 6/95 magic-number rejection to unsupported_type', async () => { + mockWordExtract.mockRejectedValue( + new Error('This does not seem to be a Word document: Invalid magic number: a5dc') + ) + + const error = await new DocParser() + .parseBuffer(buildLegacyOleDoc()) + .catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(FileParserError) + expect(error).toMatchObject({ code: 'unsupported_type' }) + expect((error as Error).message).toMatch(/Word 6\/95/) + }) + + it.each([ + new Error('Invalid Short Sector Allocation Table'), + new RangeError('Attempt to access memory outside buffer bounds'), + ])( + 'maps any other word-extractor failure to a stable invalid_format message with the cause retained: %s', + async (libraryError) => { + mockWordExtract.mockRejectedValue(libraryError) + + const error = await new DocParser() + .parseBuffer(buildLegacyOleDoc()) + .catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(FileParserError) + expect(error).toMatchObject({ + code: 'invalid_format', + message: 'This .doc file could not be read', + }) + expect((error as FileParserError).cause).toBe(libraryError) + } + ) + + it('reports a legacy .doc with no text as no_extractable_text', async () => { + mockWordExtract.mockResolvedValue(wordDocument({ body: ' \n' })) + + await expect(new DocParser().parseBuffer(buildLegacyOleDoc())).rejects.toMatchObject({ + code: 'no_extractable_text', + }) + }) + + it('returns a plain-text file misnamed .doc as its decoded text', async () => { + const result = await new DocParser().parseBuffer( + Buffer.from('Vendor list\nBloomberg\nCaf\xe9\n', 'latin1') + ) + + expect(result.content).toBe('Vendor list\nBloomberg\nCafé') + expect(result.metadata?.degraded).toBeFalsy() + expect(result.metadata?.encoding).toBe('windows-1252') + expect(mockWordExtract).not.toHaveBeenCalled() + }) + + it('rejects bytes that are neither OLE, ZIP nor text instead of scraping placeholder prose', async () => { + const png = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.from(Array.from({ length: 512 }, (_, index) => (index * 7919) % 256)), + ]) + + const error = await new DocParser().parseBuffer(png).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(FileParserError) + expect(error).toMatchObject({ code: 'invalid_format' }) + expect(mockWordExtract).not.toHaveBeenCalled() }) it('rejects an empty buffer', async () => { diff --git a/apps/sim/lib/file-parsers/doc-parser.ts b/apps/sim/lib/file-parsers/doc-parser.ts index 4c684acdf2a..0db1c78dfef 100644 --- a/apps/sim/lib/file-parsers/doc-parser.ts +++ b/apps/sim/lib/file-parsers/doc-parser.ts @@ -1,14 +1,42 @@ import { existsSync } from 'fs' import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' -import { FileParserError } from '@/lib/file-parsers/errors' +import { getErrorMessage } from '@sim/utils/errors' +import { FileParserError, toFileParserError } from '@/lib/file-parsers/errors' import { parseOfficeText } from '@/lib/file-parsers/officeparser-module' +import { sniffFileKind } from '@/lib/file-parsers/sniff' import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types' -import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' -import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard' +import { decodeTextBuffer, sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' +import { assertOoxmlArchiveWithinLimits, isZipShaped } from '@/lib/file-parsers/zip-guard' const logger = createLogger('DocParser') +/** word-extractor's rejection of a Word 6/95 (or non-Word) `FIB` identifier. */ +const WORD_6_95_MAGIC_PATTERN = /Invalid magic number/i + +interface LegacyDocSections { + body: string + headers: string + footers: string + footnotes: string + endnotes: string + textboxes: string +} + +function joinSections(sections: LegacyDocSections): string { + return [ + sections.body, + sections.headers, + sections.footers, + sections.footnotes, + sections.endnotes, + sections.textboxes, + ] + .map((section) => section.trim()) + .filter((section) => section.length > 0) + .join('\n\n') +} + export class DocParser implements FileParser { async parseFile(filePath: string, options: FileParseOptions = {}): Promise { if (!filePath) { @@ -24,10 +52,15 @@ export class DocParser implements FileParser { } /** - * A `.doc` upload is only routed here by extension — `officeparser` and - * `mammoth` both accept an OOXML/ZIP container regardless of its name, so the - * zip-bomb guard must run here exactly as it does in the docx/pptx/xlsx - * parsers. It no-ops for genuine legacy OLE `.doc` buffers. + * Routes on the container rather than the name: a genuine OLE2 `.doc` goes to + * word-extractor, a ZIP-shaped one is a misnamed OOXML package for + * officeparser/mammoth, and plain text is returned as-is. Anything else is a + * typed `invalid_format` — the former byte scrape returned ZIP part names or a + * placeholder sentence, which automated callers then indexed as prose. + * + * `officeparser` and `mammoth` both accept an OOXML/ZIP container regardless of + * its name, so the zip-bomb guard runs here exactly as it does in the + * docx/pptx/xlsx parsers. It no-ops for genuine legacy OLE `.doc` buffers. */ async parseBuffer(buffer: Buffer, options: FileParseOptions = {}): Promise { try { @@ -38,103 +71,169 @@ export class DocParser implements FileParser { assertOoxmlArchiveWithinLimits(buffer) - try { - const result = await parseOfficeText(buffer, options) - - if (result) { - const resultString = typeof result === 'string' ? result : String(result) - const content = sanitizeTextForUTF8(resultString.trim()) - - if (content.length > 0) { - return { - content, - metadata: { - characterCount: content.length, - extractionMethod: 'officeparser', - }, - } - } - } - } catch (officeError) { - options.signal?.throwIfAborted() - logger.warn('officeparser failed, trying mammoth:', officeError) + if (isZipShaped(buffer)) { + return await this.parseOoxmlContainer(buffer, options) } - try { - const mammoth = await import('mammoth') - const result = await mammoth.extractRawText({ buffer }) - options.signal?.throwIfAborted() - - if (result.value && result.value.trim().length > 0) { - const content = sanitizeTextForUTF8(result.value.trim()) - return { - content, - metadata: { - characterCount: content.length, - extractionMethod: 'mammoth', - messages: result.messages, - }, - } - } - } catch (mammothError) { - options.signal?.throwIfAborted() - logger.warn('mammoth failed:', mammothError) + const kind = sniffFileKind(buffer) + if (kind === 'ole2') { + return await this.parseLegacyDoc(buffer, options) + } + if (kind === 'text' || kind === 'html') { + return this.parsePlainText(buffer) } - options.signal?.throwIfAborted() - return this.fallbackExtraction(buffer) + throw new FileParserError( + 'invalid_format', + `File content does not match the .doc extension (detected ${kind}). Re-save it as DOCX and retry.` + ) } catch (error) { logger.error('DOC parsing error:', error) throw error } } - private fallbackExtraction(buffer: Buffer): FileParseResult { - const isBinaryDoc = buffer.length >= 2 && buffer[0] === 0xd0 && buffer[1] === 0xcf + /** A binary Word 97–2003 document, read through word-extractor's OLE2 reader. */ + private async parseLegacyDoc( + buffer: Buffer, + options: FileParseOptions + ): Promise { + const { default: WordExtractor } = await import('word-extractor') + options.signal?.throwIfAborted() - if (!isBinaryDoc) { - const textContent = buffer.toString('utf8').trim() + let sections: LegacyDocSections + try { + const document = await new WordExtractor().extract(buffer) + const raw = { filterUnicode: false } + sections = { + body: document.getBody(raw), + headers: document.getHeaders({ ...raw, includeFooters: false }), + footers: document.getFooters(raw), + footnotes: document.getFootnotes(raw), + endnotes: document.getEndnotes(raw), + /** + * `includeBody`/`includeHeadersAndFooters` select *which* text boxes are + * returned (those anchored in the body vs. in headers/footers), not + * whether body text is repeated — both default true, and both are wanted. + */ + textboxes: document.getTextboxes(raw), + } + } catch (error) { + options.signal?.throwIfAborted() + if (WORD_6_95_MAGIC_PATTERN.test(getErrorMessage(error))) { + throw new FileParserError( + 'unsupported_type', + 'This .doc file uses a Word 6/95 format that is not supported. Save it as .docx and retry.', + error + ) + } + /** word-extractor surfaces corrupt files as raw `RangeError`s; users get a stable message. */ + throw new FileParserError('invalid_format', 'This .doc file could not be read', error) + } + options.signal?.throwIfAborted() - if (textContent.length > 0) { - const printableChars = textContent.match(/[\x20-\x7E\n\r\t]/g)?.length || 0 - const isProbablyText = printableChars / textContent.length > 0.9 + const content = sanitizeTextForUTF8(joinSections(sections)) + if (content.length === 0) { + throw new FileParserError( + 'no_extractable_text', + 'No text could be extracted from this DOC file. Re-save it as DOCX to index it.' + ) + } - if (isProbablyText) { + return { + content, + metadata: { + characterCount: content.length, + extractionMethod: 'word-extractor', + degraded: false, + }, + } + } + + /** A `.docx` package saved under the wrong extension. */ + private async parseOoxmlContainer( + buffer: Buffer, + options: FileParseOptions + ): Promise { + let extracted = false + let lastError: unknown + + try { + const result = await parseOfficeText(buffer, options) + extracted = true + + if (result) { + const resultString = typeof result === 'string' ? result : String(result) + const content = sanitizeTextForUTF8(resultString.trim()) + + if (content.length > 0) { return { - content: sanitizeTextForUTF8(textContent), + content, metadata: { - extractionMethod: 'plaintext-fallback', - characterCount: textContent.length, - warning: 'File is not a valid DOC format, extracted as plain text', + characterCount: content.length, + extractionMethod: 'officeparser', }, } } } + } catch (officeError) { + options.signal?.throwIfAborted() + lastError = officeError + logger.warn('officeparser failed, trying mammoth:', officeError) } - const text = buffer.toString('utf8', 0, Math.min(buffer.length, 100000)) + try { + const mammoth = await import('mammoth') + const result = await mammoth.extractRawText({ buffer }) + options.signal?.throwIfAborted() + extracted = true + + if (result.value && result.value.trim().length > 0) { + const content = sanitizeTextForUTF8(result.value.trim()) + return { + content, + metadata: { + characterCount: content.length, + extractionMethod: 'mammoth', + messages: result.messages, + }, + } + } + } catch (mammothError) { + options.signal?.throwIfAborted() + lastError = mammothError + logger.warn('mammoth failed:', mammothError) + } - const readableText = text - .match(/[\x20-\x7E\s]{4,}/g) - ?.filter( - (chunk) => - chunk.trim().length > 10 && /[a-zA-Z]/.test(chunk) && !/^[\x00-\x1F]*$/.test(chunk) + options.signal?.throwIfAborted() + if (extracted) { + throw new FileParserError( + 'no_extractable_text', + 'No text could be extracted from this document. Re-save it as DOCX to index it.' ) - .join(' ') - .replace(/\s+/g, ' ') - .trim() + } + throw toFileParserError(lastError, 'invalid_format', 'Failed to parse DOC buffer') + } + + /** A real text file misnamed `.doc` is a genuine extraction, not a degraded one. */ + private parsePlainText(buffer: Buffer): FileParseResult { + const decoded = decodeTextBuffer(buffer) + const content = sanitizeTextForUTF8(decoded.text.trim()) - const content = readableText - ? sanitizeTextForUTF8(readableText) - : 'Unable to extract text from DOC file. Please convert to DOCX format for better results.' + if (content.length === 0) { + throw new FileParserError('no_extractable_text', 'The file contains no text') + } return { content, metadata: { - extractionMethod: 'fallback', - degraded: true, + extractionMethod: 'plaintext-fallback', characterCount: content.length, - warning: 'Basic text extraction used. For better results, convert to DOCX format.', + encoding: decoded.encoding, + warning: [ + 'File is not a valid DOC format, extracted as plain text', + ...(decoded.warning ? [decoded.warning] : []), + ].join('. '), }, } } diff --git a/apps/sim/lib/file-parsers/docx-parser.ts b/apps/sim/lib/file-parsers/docx-parser.ts index 7a6c6038849..35586d16b6b 100644 --- a/apps/sim/lib/file-parsers/docx-parser.ts +++ b/apps/sim/lib/file-parsers/docx-parser.ts @@ -6,23 +6,27 @@ import { isEncryptedOfficeParserError, toFileParserError, } from '@/lib/file-parsers/errors' +import { + assertHtmlStringWithinLimits, + htmlToStructuredText, + isHtmlComplexityError, +} from '@/lib/file-parsers/html-parser' import { parseOfficeText } from '@/lib/file-parsers/officeparser-module' +import { isEncryptedOoxmlContainer } from '@/lib/file-parsers/ooxml-encryption' import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types' import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard' const logger = createLogger('DocxParser') -interface MammothMessage { - type: 'warning' | 'error' - message: string -} - -interface MammothResult { - value: string - messages: MammothMessage[] -} - +/** + * Extracts DOCX text by rendering the document to HTML with mammoth and walking + * that HTML with the shared structured-text walker. mammoth's HTML keeps the + * heading levels, list nesting, table rows, and footnotes that its raw-text mode + * flattens to one paragraph per cell, so the output matches what the HTML parser + * produces for the same document. (mammoth's Markdown mode is deprecated and + * drops tables, so it is deliberately not used.) + */ export class DocxParser implements FileParser { async parseFile(filePath: string, options: FileParseOptions = {}): Promise { if (!filePath) { @@ -46,24 +50,29 @@ export class DocxParser implements FileParser { let parserReturnedEmpty = false try { - const result = await mammoth.extractRawText({ buffer }) + const htmlResult = await mammoth.convertToHtml({ buffer }) options.signal?.throwIfAborted() - if (result.value && result.value.trim().length > 0) { - let htmlResult: MammothResult = { value: '', messages: [] } - try { - htmlResult = await mammoth.convertToHtml({ buffer }) - } catch { - // HTML conversion is optional + const structured = this.structuredTextFromHtml(htmlResult.value) + if (structured) { + return { + content: sanitizeTextForUTF8(structured), + metadata: { + extractionMethod: 'mammoth-html', + messages: htmlResult.messages, + }, } - options.signal?.throwIfAborted() + } + const rawResult = await mammoth.extractRawText({ buffer }) + options.signal?.throwIfAborted() + + if (rawResult.value && rawResult.value.trim().length > 0) { return { - content: sanitizeTextForUTF8(result.value), + content: sanitizeTextForUTF8(rawResult.value), metadata: { extractionMethod: 'mammoth', - messages: [...result.messages, ...htmlResult.messages], - html: htmlResult.value, + messages: [...htmlResult.messages, ...rawResult.messages], }, } } @@ -98,6 +107,14 @@ export class DocxParser implements FileParser { extractionErrors.push(officeError) } + if (isEncryptedOoxmlContainer(buffer)) { + throw new FileParserError( + 'encrypted_file', + 'This document is encrypted or password-protected', + extractionErrors.length > 0 ? new AggregateError(extractionErrors) : undefined + ) + } + const isZipFile = buffer.length >= 2 && buffer[0] === 0x50 && buffer[1] === 0x4b if (!isZipFile) { const textContent = buffer.toString('utf8').trim() @@ -140,4 +157,23 @@ export class DocxParser implements FileParser { throw toFileParserError(error, 'invalid_format', 'Failed to parse DOCX buffer') } } + + /** + * Walks mammoth's HTML rendering under the HTML parser's size caps. A rendering + * too large to walk safely falls back to the raw-text path by returning empty, + * since mammoth has already materialised the document once at that point. + */ + private structuredTextFromHtml(html: string): string { + if (!html || html.trim().length === 0) return '' + try { + assertHtmlStringWithinLimits(html) + } catch (error) { + if (isHtmlComplexityError(error)) { + logger.warn('mammoth HTML exceeds walker limits, using raw text:', error.message) + return '' + } + throw error + } + return htmlToStructuredText(html).trim() + } } diff --git a/apps/sim/lib/file-parsers/errors.test.ts b/apps/sim/lib/file-parsers/errors.test.ts index 66d221b40ab..57bb66c17e2 100644 --- a/apps/sim/lib/file-parsers/errors.test.ts +++ b/apps/sim/lib/file-parsers/errors.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { FileParserError, + getFileParserErrorCode, isEncryptedOfficeParserError, toFileParserError, } from '@/lib/file-parsers/errors' @@ -56,4 +57,15 @@ describe('file parser errors', () => { ])('recognizes the SheetJS encrypted-workbook error: %s', (message) => { expect(isEncryptedOfficeParserError(new Error(message))).toBe(true) }) + + it('maps the archive guard classes onto parser codes without wrapping them', () => { + expect(getFileParserErrorCode(new ArchiveIntegrityError('Archive entries overlap'))).toBe( + 'invalid_format' + ) + expect(getFileParserErrorCode(new ZipBombError('Archive too large'))).toBe('complexity_limit') + expect(getFileParserErrorCode(new FileParserError('encrypted_file', 'locked'))).toBe( + 'encrypted_file' + ) + expect(getFileParserErrorCode(new Error('untyped'))).toBeUndefined() + }) }) diff --git a/apps/sim/lib/file-parsers/errors.ts b/apps/sim/lib/file-parsers/errors.ts index 0f3cbff5d27..aed6c95211d 100644 --- a/apps/sim/lib/file-parsers/errors.ts +++ b/apps/sim/lib/file-parsers/errors.ts @@ -31,6 +31,20 @@ export function isFileParserError(error: unknown): error is FileParserError { return error instanceof FileParserError } +/** + * The parser code an error maps to, including the archive guard's own classes + * (which are not `FileParserError` because `ooxml-limits` must stay browser-safe + * and dependency-free). Callers that branch on a code use this instead of + * `isFileParserError` so an archive rejection is never mistaken for an untyped, + * retryable failure. + */ +export function getFileParserErrorCode(error: unknown): FileParserErrorCode | undefined { + if (isFileParserError(error)) return error.code + if (error instanceof ArchiveIntegrityError) return 'invalid_format' + if (error instanceof ZipBombError) return 'complexity_limit' + return undefined +} + /** * Wraps an untyped parser-library exception without erasing a typed inner cause. * Archive safety and integrity failures remain typed so every caller can enforce diff --git a/apps/sim/lib/file-parsers/html-parser.test.ts b/apps/sim/lib/file-parsers/html-parser.test.ts index 17013dfd1aa..985ac35338f 100644 --- a/apps/sim/lib/file-parsers/html-parser.test.ts +++ b/apps/sim/lib/file-parsers/html-parser.test.ts @@ -9,6 +9,15 @@ import { HtmlComplexityError, HtmlParser } from '@/lib/file-parsers/html-parser' const parser = new HtmlParser() +describe('table cells with several paragraphs', () => { + it('separates block children inside a cell with a space', async () => { + const html = '

Заказчик

Исполняющий

ok
' + const result = await new HtmlParser().parseBuffer(Buffer.from(html)) + + expect(result.content).toContain('| Заказчик Исполняющий | ok |') + }) +}) + describe('HtmlParser', () => { it('reports empty input with the typed parser taxonomy', async () => { await expect(parser.parseBuffer(Buffer.alloc(0))).rejects.toMatchObject({ @@ -118,5 +127,83 @@ describe('HtmlParser', () => { expect(result.metadata?.listCount).toBe(1) expect(result.metadata?.tableCount).toBe(1) }) + + it('numbers ordered lists and keeps markers on nested items', async () => { + const buffer = Buffer.from( + `
  1. third
  2. fourth
    • nested
` + ) + + const result = await parser.parseBuffer(buffer) + + expect(result.content).toContain('3. third') + expect(result.content).toContain('4. fourth') + expect(result.content).toContain(' • nested') + expect(result.content).not.toContain('fourth nested') + }) + + it('renders a nested table inside its cell exactly once', async () => { + const buffer = Buffer.from( + `
Outer A

Intro

` + + `
Inner 1Inner 2
Inner 3
` + + `
Outer BPlain
` + ) + + const result = await parser.parseBuffer(buffer) + + expect(result.content).toContain('| Outer A | Intro Inner 1 / Inner 2 / Inner 3 |') + expect(result.content).toContain('| Outer B | Plain |') + for (const cell of ['Outer A', 'Inner 1', 'Inner 2', 'Inner 3', 'Outer B', 'Plain']) { + expect(result.content.split(cell)).toHaveLength(2) + } + expect(result.content.match(/\[Table\]/g)).toHaveLength(1) + expect(result.metadata?.tableCount).toBe(2) + }) + + it('keeps descriptive image alt text and drops file-name alt text', async () => { + const buffer = Buffer.from( + `Org chartpython-logo.gifImage 2

Body

` + ) + + const result = await parser.parseBuffer(buffer) + + expect(result.content).toContain('[Image: Org chart]') + expect(result.content).not.toContain('python-logo') + expect(result.content).not.toContain('Image 2') + }) + + it('separates block elements inside a list item', async () => { + const buffer = Buffer.from( + `
  • Versions

    Release Information

` + ) + + const result = await parser.parseBuffer(buffer) + + expect(result.content).toContain('• Versions Release Information') + }) + + it('drops endnote return links but keeps the endnote text', async () => { + const buffer = Buffer.from( + `

Body[1]

` + + `
  1. End text

` + ) + + const result = await parser.parseBuffer(buffer) + + expect(result.content).toContain('1. End text') + expect(result.content).not.toContain('↑') + }) + + it('drops footnote return links but keeps the footnote text', async () => { + const buffer = Buffer.from( + `

Body[1]

` + + `
  1. Note text

` + ) + + const result = await parser.parseBuffer(buffer) + + expect(result.content).toContain('Body[1]') + expect(result.content).toContain('1. Note text') + expect(result.content).not.toContain('↑') + }) }) }) diff --git a/apps/sim/lib/file-parsers/html-parser.ts b/apps/sim/lib/file-parsers/html-parser.ts index d7fd0e396d3..24bdf8cd47c 100644 --- a/apps/sim/lib/file-parsers/html-parser.ts +++ b/apps/sim/lib/file-parsers/html-parser.ts @@ -3,8 +3,9 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import * as cheerio from 'cheerio' import { FileParserError } from '@/lib/file-parsers/errors' +import { imageAltText } from '@/lib/file-parsers/office-text' import type { FileParseResult, FileParser } from '@/lib/file-parsers/types' -import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' +import { decodeTextBuffer, sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' const logger = createLogger('HtmlParser') @@ -78,6 +79,356 @@ function assertHtmlWithinLimits(buffer: Buffer): void { } } +/** + * The same caps for HTML that already exists as a string — markup another + * converter produced in memory (mammoth's DOCX rendering) — measured without + * copying it into a buffer. + */ +export function assertHtmlStringWithinLimits(html: string): void { + const byteLength = Buffer.byteLength(html, 'utf8') + if (byteLength > MAX_HTML_INPUT_BYTES) { + throw new HtmlComplexityError( + `HTML document is ${byteLength} bytes, above the maximum of ${MAX_HTML_INPUT_BYTES} bytes` + ) + } + + let count = 0 + let index = html.indexOf('<') + while (index !== -1) { + if (++count > MAX_HTML_MARKUP_TOKENS) { + throw new HtmlComplexityError( + `HTML document exceeds the maximum of ${MAX_HTML_MARKUP_TOKENS} markup tokens` + ) + } + index = html.indexOf('<', index + 1) + } +} + +const NON_CONTENT_SELECTOR = 'script, style, noscript, meta, link, iframe, object, embed, svg' + +/** Block elements inside a table cell; `.text()` would otherwise glue their words together. */ +const CELL_BLOCK_SELECTOR = 'p, div, li, br, h1, h2, h3, h4, h5, h6, tr' + +/** mammoth renders a footnote's or endnote's return link as ``. */ +const FOOTNOTE_BACKLINK_SELECTOR = 'a[href^="#footnote-ref"], a[href^="#endnote-ref"]' + +/** + * Strips the non-content markup and HTML comments from a loaded document so the + * structured walk sees only what a reader would. + */ +function stripNonContent($: cheerio.CheerioAPI): void { + $(NON_CONTENT_SELECTOR).remove() + $(FOOTNOTE_BACKLINK_SELECTOR).remove() + + $.root() + .contents() + .filter(function () { + return this.type === 'comment' + }) + .remove() +} + +/** + * Converts an HTML document into structured plain text: headings and paragraphs + * on their own lines, `•`/`1.` list markers with nesting indents, and tables as + * `[Table]` / `| a | b |` / `[/Table]` rows. Shared by {@link HtmlParser} and the + * DOCX parser, which routes mammoth's HTML rendering through the same walk so + * both formats produce the same shape. + */ +export function htmlToStructuredText(html: string): string { + const $ = cheerio.load(html) + stripNonContent($) + return extractStructuredText($) +} + +function extractStructuredText($: cheerio.CheerioAPI): string { + const contentParts: string[] = [] + + const rootElement = $('body').length > 0 ? $('body') : $.root() + + processElement($, rootElement, contentParts, 0) + + return contentParts.join('\n').trim() +} + +type AnyNode = ReturnType['contents']> extends cheerio.Cheerio + ? N + : never + +type ElementNode = Extract + +function isTagNode(node: AnyNode): node is ElementNode { + return node.type === 'tag' +} + +/** + * Recursively process elements to extract text with structure + */ +function processElement( + $: cheerio.CheerioAPI, + element: cheerio.Cheerio, + contentParts: string[], + depth: number +): void { + element.contents().each((_, node) => { + if (node.type === 'text') { + const text = $(node).text().trim() + if (text) { + contentParts.push(text) + } + return + } + + if (!isTagNode(node)) return + + const $node = $(node) + const tagName = node.tagName.toLowerCase() + + switch (tagName) { + case 'h1': + case 'h2': + case 'h3': + case 'h4': + case 'h5': + case 'h6': { + const headingText = $node.text().trim() + if (headingText) { + contentParts.push(`\n${headingText}\n`) + } + break + } + + case 'p': { + const paragraphText = $node.text().trim() + if (paragraphText) { + contentParts.push(`${paragraphText}\n`) + } + break + } + + case 'br': + contentParts.push('\n') + break + + case 'hr': + contentParts.push('\n---\n') + break + + case 'li': + processListItem($, $node, contentParts, depth, null) + break + + case 'ul': + case 'ol': + contentParts.push('\n') + processList($, $node, contentParts, depth + 1, tagName === 'ol') + contentParts.push('\n') + break + + case 'table': + processTable($, $node, contentParts) + break + + case 'blockquote': { + const quoteText = $node.text().trim() + if (quoteText) { + contentParts.push(`\n> ${quoteText}\n`) + } + break + } + + case 'pre': + case 'code': { + const codeText = $node.text().trim() + if (codeText) { + contentParts.push(`\n\`\`\`\n${codeText}\n\`\`\`\n`) + } + break + } + + case 'a': { + const linkText = $node.text().trim() + const href = $node.attr('href') + if (linkText) { + if (href?.startsWith('http')) { + contentParts.push(`${linkText} (${href})`) + } else { + contentParts.push(linkText) + } + } + break + } + + case 'img': { + const image = imageAltText($node.attr('alt')) + if (image) contentParts.push(image) + break + } + + default: + processElement($, $node, contentParts, depth) + } + }) +} + +/** + * Walks a list's children, numbering `
    ` items from its `start` attribute and + * bulleting `
      ` items. Non-item children are walked as ordinary content. + */ +function processList( + $: cheerio.CheerioAPI, + list: cheerio.Cheerio, + contentParts: string[], + depth: number, + ordered: boolean +): void { + const start = Number.parseInt(list.attr('start') ?? '1', 10) + let index = Number.isFinite(start) ? start : 1 + + list.children().each((_, child) => { + const $child = $(child) + if (isTagNode(child) && child.tagName.toLowerCase() === 'li') { + processListItem($, $child, contentParts, depth, ordered ? index++ : null) + } else { + processElement($, $child, contentParts, depth) + } + }) +} + +/** + * Emits a list item as one marked line built from its own inline text, then + * walks any nested lists so their items keep their own markers and indent. + */ +function processListItem( + $: cheerio.CheerioAPI, + item: cheerio.Cheerio, + contentParts: string[], + depth: number, + ordinal: number | null +): void { + const ownText: string[] = [] + const nestedLists: cheerio.Cheerio[] = [] + + item.contents().each((_, child) => { + if (isTagNode(child)) { + const childTag = child.tagName.toLowerCase() + if (childTag === 'ul' || childTag === 'ol') { + nestedLists.push($(child)) + return + } + } + const $child = $(child) + $child.find(CELL_BLOCK_SELECTOR).after(' ') + const text = $child.text().replace(/\s+/g, ' ').trim() + if (text) ownText.push(text) + }) + + const itemText = ownText.join(' ').trim() + if (itemText) { + const indent = ' '.repeat(Math.min(Math.max(depth - 1, 0), 3)) + const marker = ordinal === null ? '•' : `${ordinal}.` + contentParts.push(`${indent}${marker} ${itemText}`) + } + + for (const nested of nestedLists) { + const nestedTag = nested.prop('tagName')?.toLowerCase() + processList($, nested, contentParts, depth + 1, nestedTag === 'ol') + } +} + +/** A table's own rows: direct `` children and those under its section elements. */ +function directRows( + $: cheerio.CheerioAPI, + table: cheerio.Cheerio +): cheerio.Cheerio { + return table.children('thead, tbody, tfoot').children('tr').add(table.children('tr')) +} + +/** Nested tables whose nearest enclosing table is the cell's own. */ +function topLevelNestedTables( + $: cheerio.CheerioAPI, + cell: cheerio.Cheerio +): cheerio.Cheerio { + const own = cell.closest('table').get(0) + return cell.find('table').filter((_, nested) => $(nested).parents('table').get(0) === own) +} + +/** + * The cells of a table in reading order, each rendered with {@link cellText}, + * for a table nested inside another table's cell. + */ +function flattenedTableCells($: cheerio.CheerioAPI, table: cheerio.Cheerio): string[] { + const cells: string[] = [] + directRows($, table).each((_, row) => { + $(row) + .children('td, th') + .each((_, cell) => { + const text = cellText($, $(cell)) + if (text) cells.push(text) + }) + }) + return cells +} + +/** + * One cell's text on a single line. A text-only cell — the common case in a + * data table — is read directly. Block elements inside the cell get a space + * so adjacent paragraphs do not glue together — this mutates the live DOM, and + * runs before `extractHeadings`/`extractLinks`, so heading or link text inside a + * cell gains those spaces too. A nested table is rendered on a clone of the cell + * as its cells joined with ` / `, without `[Table]` markers or pipes, so the + * outer row stays one line and the inner text appears exactly once. + */ +function cellText($: cheerio.CheerioAPI, cell: cheerio.Cheerio): string { + if (cell.children().length === 0) { + return cell.text().replace(/\s+/g, ' ').trim() + } + + const nested = topLevelNestedTables($, cell) + if (nested.length === 0) { + cell.find(CELL_BLOCK_SELECTOR).after(' ') + return cell.text().replace(/\s+/g, ' ').trim() + } + + const clone = cell.clone() + topLevelNestedTables($, clone).each((_, table) => { + const $table = $(table) + $table.replaceWith(` ${flattenedTableCells($, $table).join(' / ')} `) + }) + clone.find(CELL_BLOCK_SELECTOR).after(' ') + return clone.text().replace(/\s+/g, ' ').trim() +} + +/** + * Renders a table as `[Table]`, one `| a | b |` line per row, `[/Table]`. Only + * the table's own rows and each row's own cells are visited, so a nested table + * contributes to its containing cell (see {@link cellText}) and is never + * emitted a second time as rows of its own. + */ +function processTable( + $: cheerio.CheerioAPI, + table: cheerio.Cheerio, + contentParts: string[] +): void { + contentParts.push('\n[Table]') + + directRows($, table).each((_, row) => { + const cells: string[] = [] + + $(row) + .children('td, th') + .each((_, cell) => { + cells.push(cellText($, $(cell))) + }) + + if (cells.length > 0) { + contentParts.push(`| ${cells.join(' | ')} |`) + } + }) + + contentParts.push('[/Table]\n') +} + export class HtmlParser implements FileParser { async parseFile(filePath: string): Promise { let buffer: Buffer @@ -107,23 +458,16 @@ export class HtmlParser implements FileParser { try { logger.info('Parsing HTML buffer, size:', buffer.length) - const htmlContent = buffer.toString('utf-8') + const decoded = decodeTextBuffer(buffer) + const htmlContent = decoded.text const $ = cheerio.load(htmlContent) - // Extract meta information before removing tags const title = $('title').text().trim() const metaDescription = $('meta[name="description"]').attr('content') || '' - $('script, style, noscript, meta, link, iframe, object, embed, svg').remove() - - $.root() - .contents() - .filter(function () { - return this.type === 'comment' - }) - .remove() + stripNonContent($) - const content = this.extractStructuredText($) + const content = extractStructuredText($) const sanitizedContent = sanitizeTextForUTF8(content) @@ -147,6 +491,8 @@ export class HtmlParser implements FileParser { links: links.slice(0, 50), hasImages: $('img').length > 0, imageCount: $('img').length, + encoding: decoded.encoding, + warning: decoded.warning, hasTable: $('table').length > 0, tableCount: $('table').length, hasList: $('ul, ol').length > 0, @@ -177,171 +523,6 @@ export class HtmlParser implements FileParser { } } - /** - * Extract structured text content preserving document hierarchy - */ - private extractStructuredText($: cheerio.CheerioAPI): string { - const contentParts: string[] = [] - - const rootElement = $('body').length > 0 ? $('body') : $.root() - - this.processElement($, rootElement, contentParts, 0) - - return contentParts.join('\n').trim() - } - - /** - * Recursively process elements to extract text with structure - */ - private processElement( - $: cheerio.CheerioAPI, - element: cheerio.Cheerio, - contentParts: string[], - depth: number - ): void { - element.contents().each((_, node) => { - if (node.type === 'text') { - const text = $(node).text().trim() - if (text) { - contentParts.push(text) - } - } else if (node.type === 'tag') { - const $node = $(node) - const tagName = node.tagName?.toLowerCase() - - switch (tagName) { - case 'h1': - case 'h2': - case 'h3': - case 'h4': - case 'h5': - case 'h6': { - const headingText = $node.text().trim() - if (headingText) { - contentParts.push(`\n${headingText}\n`) - } - break - } - - case 'p': { - const paragraphText = $node.text().trim() - if (paragraphText) { - contentParts.push(`${paragraphText}\n`) - } - break - } - - case 'br': - contentParts.push('\n') - break - - case 'hr': - contentParts.push('\n---\n') - break - - case 'li': { - const listItemText = $node.text().trim() - if (listItemText) { - const indent = ' '.repeat(Math.min(depth, 3)) - contentParts.push(`${indent}• ${listItemText}`) - } - break - } - - case 'ul': - case 'ol': - contentParts.push('\n') - this.processElement($, $node, contentParts, depth + 1) - contentParts.push('\n') - break - - case 'table': - this.processTable($, $node, contentParts) - break - - case 'blockquote': { - const quoteText = $node.text().trim() - if (quoteText) { - contentParts.push(`\n> ${quoteText}\n`) - } - break - } - - case 'pre': - case 'code': { - const codeText = $node.text().trim() - if (codeText) { - contentParts.push(`\n\`\`\`\n${codeText}\n\`\`\`\n`) - } - break - } - - case 'div': - case 'section': - case 'article': - case 'main': - case 'aside': - case 'nav': - case 'header': - case 'footer': - this.processElement($, $node, contentParts, depth) - break - - case 'a': { - const linkText = $node.text().trim() - const href = $node.attr('href') - if (linkText) { - if (href?.startsWith('http')) { - contentParts.push(`${linkText} (${href})`) - } else { - contentParts.push(linkText) - } - } - break - } - - case 'img': { - const alt = $node.attr('alt') - if (alt) { - contentParts.push(`[Image: ${alt}]`) - } - break - } - - default: - this.processElement($, $node, contentParts, depth) - } - } - }) - } - - /** - * Process table elements to extract structured data - */ - private processTable( - $: cheerio.CheerioAPI, - table: cheerio.Cheerio, - contentParts: string[] - ): void { - contentParts.push('\n[Table]') - - table.find('tr').each((_, row) => { - const $row = $(row) - const cells: string[] = [] - - $row.find('td, th').each((_, cell) => { - const cellText = $(cell).text().trim() - cells.push(cellText || '') - }) - - if (cells.length > 0) { - contentParts.push(`| ${cells.join(' | ')} |`) - } - }) - - contentParts.push('[/Table]\n') - } - /** * Extract heading structure for metadata */ diff --git a/apps/sim/lib/file-parsers/index.test.ts b/apps/sim/lib/file-parsers/index.test.ts index 5ed13efce2b..48254a577f6 100644 --- a/apps/sim/lib/file-parsers/index.test.ts +++ b/apps/sim/lib/file-parsers/index.test.ts @@ -59,7 +59,6 @@ vi.mock('@/lib/file-parsers/index', () => { txt: { parseFile: mockTxtParseFile }, md: { parseFile: mockMdParseFile }, pptx: { parseFile: mockPptxParseFile }, - ppt: { parseFile: mockPptxParseFile }, html: { parseFile: mockHtmlParseFile }, htm: { parseFile: mockHtmlParseFile }, } @@ -232,22 +231,6 @@ describe('File Parsers', () => { expect(result).toEqual(expectedResult) }) - it('should parse PPT files successfully', async () => { - const expectedResult = { - content: 'Parsed PPTX content', - metadata: { - slideCount: 5, - extractionMethod: 'officeparser', - }, - } - - mockPptxParseFile.mockResolvedValueOnce(expectedResult) - - const result = await parseFile('/test/files/presentation.ppt') - - expect(result).toEqual(expectedResult) - }) - it('should parse HTML files successfully', async () => { const expectedResult = { content: 'Parsed HTML content', @@ -304,13 +287,13 @@ describe('File Parsers', () => { expect(isSupportedFileType('txt')).toBe(true) expect(isSupportedFileType('md')).toBe(true) expect(isSupportedFileType('pptx')).toBe(true) - expect(isSupportedFileType('ppt')).toBe(true) expect(isSupportedFileType('html')).toBe(true) expect(isSupportedFileType('htm')).toBe(true) }) it('should return false for unsupported file types', () => { expect(isSupportedFileType('png')).toBe(false) + expect(isSupportedFileType('ppt')).toBe(false) expect(isSupportedFileType('unknown')).toBe(false) }) diff --git a/apps/sim/lib/file-parsers/index.ts b/apps/sim/lib/file-parsers/index.ts index fa5f36888a1..7c4cc43a512 100644 --- a/apps/sim/lib/file-parsers/index.ts +++ b/apps/sim/lib/file-parsers/index.ts @@ -13,9 +13,11 @@ import { parseJSONLBuffer, } from '@/lib/file-parsers/json-parser' import { MdParser } from '@/lib/file-parsers/md-parser' +import { ArchiveIntegrityError } from '@/lib/file-parsers/ooxml-limits' import { OpenDocumentParser } from '@/lib/file-parsers/opendocument-parser' import { PdfParser } from '@/lib/file-parsers/pdf-parser' import { PptxParser } from '@/lib/file-parsers/pptx-parser' +import { reconcileParserRoute, sniffFileKind } from '@/lib/file-parsers/sniff' import { TxtParser } from '@/lib/file-parsers/txt-parser' import type { FileParseOptions, @@ -38,9 +40,9 @@ const logger = createLogger('FileParser') * - `xlsm`/`xlsb`/`xltx`/`xls`/`ods` are all read natively by SheetJS. `ods` is * treated as a spreadsheet rather than routed to {@link OpenDocumentParser} so * its output keeps per-sheet structure instead of one flat text run. - * - `pptm`/`potx` are the PresentationML package `pptx` uses. `ppt` is the legacy - * OLE binary that no bundled library reads; it is mapped here so it degrades - * through the parser's own reporting rather than looking simply unsupported. + * - `pptm`/`potx` are the PresentationML package `pptx` uses. Legacy OLE `ppt` + * is deliberately absent: no bundled library reads it, and registering it only + * produced scraped placeholder prose, so uploads refuse it up front instead. * * Every parser module is imported statically and every dependency is a regular * (non-optional) one, so a broken install fails loudly at import. This previously @@ -70,7 +72,6 @@ const PARSERS = new Map([ ['xltx', new XlsxParser()], ['ods', new XlsxParser()], ['pptx', new PptxParser()], - ['ppt', new PptxParser()], ['pptm', new PptxParser()], ['potx', new PptxParser()], ['odt', new OpenDocumentParser()], @@ -121,6 +122,11 @@ export async function parseFile( } } +function joinWarnings(...warnings: Array): string | undefined { + const present = warnings.filter((warning): warning is string => Boolean(warning)) + return present.length > 0 ? present.join('. ') : undefined +} + /** * Parse a buffer based on file extension * @param buffer Buffer containing the file data @@ -131,7 +137,12 @@ export async function parseFile( * The zip-bomb guard runs here for every extension, not just the OOXML ones: * the extension is an attacker-controlled routing hint, and the guard no-ops * for buffers that are not ZIP archives. Individual parsers still call it so a - * direct `parser.parseBuffer` caller is covered too. + * direct `parser.parseBuffer` caller is covered too. Its integrity rejection is + * surfaced as a typed `invalid_format` so callers never retry a corrupt archive. + * + * After the guard, the bytes are sniffed and reconciled with the extension (see + * {@link reconcileParserRoute}); a re-routed parse records `detectedType` and a + * warning in its metadata. */ export async function parseBuffer( buffer: Buffer, @@ -147,26 +158,50 @@ export async function parseBuffer( throw new Error('No file extension provided') } - assertOoxmlArchiveWithinLimits(buffer) + try { + assertOoxmlArchiveWithinLimits(buffer) + } catch (error) { + if (error instanceof ArchiveIntegrityError) { + throw new FileParserError('invalid_format', error.message, error) + } + throw error + } const normalizedExtension = extension.toLowerCase() - const parser = PARSERS.get(normalizedExtension) - - if (!parser) { + if (!PARSERS.has(normalizedExtension)) { throw new FileParserError( 'unsupported_type', `Unsupported file type: ${normalizedExtension}. Supported types are: ${SUPPORTED_EXTENSIONS_TEXT}` ) } - if (!parser.parseBuffer) { + const kind = sniffFileKind(buffer, normalizedExtension) + const route = reconcileParserRoute(normalizedExtension, kind) + const parser = PARSERS.get(route.extension) + + if (!parser?.parseBuffer) { throw new FileParserError( 'unsupported_type', - `Parser for ${normalizedExtension} does not support buffer parsing` + `Parser for ${route.extension} does not support buffer parsing` ) } - return await parser.parseBuffer(buffer, options) + const result = await parser.parseBuffer(buffer, options) + if (!route.detectedType) return result + + logger.warn('Parsed buffer under a re-routed parser', { + extension: normalizedExtension, + detectedType: route.detectedType, + route: route.extension, + }) + return { + ...result, + metadata: { + ...result.metadata, + detectedType: route.detectedType, + warning: joinWarnings(route.warning, result.metadata?.warning), + }, + } } catch (error) { logger.error('Buffer parsing error:', error) throw error diff --git a/apps/sim/lib/file-parsers/json-parser.test.ts b/apps/sim/lib/file-parsers/json-parser.test.ts index fcc106706fb..cb3ec555997 100644 --- a/apps/sim/lib/file-parsers/json-parser.test.ts +++ b/apps/sim/lib/file-parsers/json-parser.test.ts @@ -42,4 +42,50 @@ describe('JSON parser complexity limits', () => { expect(JSON.parse(result.content)).toEqual({ items: [1, 2], name: 'test' }) expect(result.metadata).toMatchObject({ isArray: false, keys: ['items', 'name'], depth: 2 }) }) + + it('parses a BOM-prefixed JSON file and reports its encoding', async () => { + const result = await parseJSONBuffer( + Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('{"name":"Café"}')]) + ) + + expect(JSON.parse(result.content)).toEqual({ name: 'Café' }) + expect(result.metadata?.encoding).toBe('utf-8') + expect(result.metadata?.warning).toBeUndefined() + }) + + it('decodes a Windows-1252 JSON file instead of rejecting or mangling it', async () => { + const result = await parseJSONBuffer(Buffer.from('{"city":"Z\xfcrich"}', 'latin1')) + + expect(JSON.parse(result.content)).toEqual({ city: 'Zürich' }) + expect(result.metadata?.encoding).toBe('windows-1252') + expect(result.metadata?.warning).toMatch(/Windows-1252/) + }) + + it('parses JSON with comments and trailing commas leniently with a warning', async () => { + const jsonc = + '{\n // strict later\n "compilerOptions": { "strict": true, /* todo */ "target": "esnext", },\n "url": "http://example.com/a//b",\n}\n' + const result = await parseJSONBuffer(Buffer.from(jsonc)) + const parsed = JSON.parse(result.content) as { + compilerOptions: { target: string } + url: string + } + + expect(parsed.compilerOptions.target).toBe('esnext') + expect(parsed.url).toBe('http://example.com/a//b') + expect(result.metadata?.warning).toContain('comments') + }) + + it('still rejects JSON that is invalid even after comment stripping', async () => { + await expect(parseJSONBuffer(Buffer.from('{ "a": [1, 2 }'))).rejects.toMatchObject({ + code: 'invalid_format', + }) + }) + + it('parses BOM-prefixed JSON Lines', async () => { + const result = await parseJSONLBuffer( + Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('{"a":1}\n{"a":2}')]) + ) + + expect(JSON.parse(result.content)).toEqual([{ a: 1 }, { a: 2 }]) + }) }) diff --git a/apps/sim/lib/file-parsers/json-parser.ts b/apps/sim/lib/file-parsers/json-parser.ts index cd47fa9c5bd..562e4cbcd43 100644 --- a/apps/sim/lib/file-parsers/json-parser.ts +++ b/apps/sim/lib/file-parsers/json-parser.ts @@ -1,6 +1,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { FileParserError } from '@/lib/file-parsers/errors' import type { FileParseResult } from '@/lib/file-parsers/types' +import { type DecodedText, decodeTextBuffer } from '@/lib/file-parsers/utils' const MAX_JSON_DEPTH = 500 const MAX_JSON_NODES = 1_000_000 @@ -141,7 +142,14 @@ function assertJsonValueWithinLimits( return maxDepth } -function buildJsonResult(jsonData: unknown): FileParseResult { +function encodingMetadata(decoded: DecodedText): Record { + return { + encoding: decoded.encoding, + ...(decoded.warning ? { warning: decoded.warning } : {}), + } +} + +function buildJsonResult(jsonData: unknown, decoded: DecodedText): FileParseResult { const budget = { nodes: 0, serializedUnits: 0 } const depth = assertJsonValueWithinLimits(jsonData, budget) const formattedContent = JSON.stringify(jsonData, null, 2) @@ -156,18 +164,66 @@ function buildJsonResult(jsonData: unknown): FileParseResult { keys: isRecord ? Object.keys(jsonData as Record) : [], itemCount: isArray ? jsonData.length : undefined, depth, + ...encodingMetadata(decoded), }, } } -function parseJsonContent(content: string): FileParseResult { +/** + * Decodes before `JSON.parse`: a UTF-8 BOM is not JSON whitespace, so the raw + * `toString('utf-8')` read used to reject every BOM-prefixed file from Windows + * editors, and a Windows-1252 file silently lost its accented characters. + */ +const JSONC_WARNING = 'File is JSON with comments or trailing commas; parsed leniently' + +/** + * Removes `//` and `/* *\/` comments and trailing commas outside string + * literals, so a `tsconfig.json`, `.vscode` settings file or `devcontainer.json` + * — JSON with comments, which editors accept — parses like plain JSON. Runs + * only after strict parsing has failed, so valid JSON never goes through it. + */ +export function stripJsonComments(text: string): string { + let out = '' + let index = 0 + while (index < text.length) { + const char = text[index] + if (char === '"') { + let end = index + 1 + while (end < text.length && text[end] !== '"') { + if (text[end] === '\\') end++ + end++ + } + out += text.slice(index, end + 1) + index = end + 1 + continue + } + if (char === '/' && text[index + 1] === '/') { + const end = text.indexOf('\n', index) + index = end === -1 ? text.length : end + continue + } + if (char === '/' && text[index + 1] === '*') { + const end = text.indexOf('*/', index + 2) + index = end === -1 ? text.length : end + 2 + continue + } + out += char + index++ + } + return out.replace(/,(\s*[}\]])/g, '$1') +} + +function parseJsonContent(buffer: Uint8Array): FileParseResult { + const decoded = decodeTextBuffer(buffer) try { - return buildJsonResult(JSON.parse(content)) + return buildJsonResult(JSON.parse(decoded.text), decoded) } catch (error) { if (error instanceof FileParserError) throw error if (!(error instanceof SyntaxError)) { throw new FileParserError('runtime_failure', 'JSON processing failed unexpectedly', error) } + const lenient = parseJsonWithComments(decoded) + if (lenient) return lenient throw new FileParserError( 'invalid_format', `Invalid JSON: ${getErrorMessage(error, 'Unknown error')}`, @@ -176,15 +232,27 @@ function parseJsonContent(content: string): FileParseResult { } } +function parseJsonWithComments(decoded: DecodedText): FileParseResult | undefined { + let value: unknown + try { + value = JSON.parse(stripJsonComments(decoded.text)) + } catch { + return undefined + } + const result = buildJsonResult(value, decoded) + const warning = [decoded.warning, JSONC_WARNING].filter(Boolean).join('; ') + return { ...result, metadata: { ...result.metadata, warning } } +} + /** Parse a JSON file. */ export async function parseJSON(filePath: string): Promise { const fs = await import('fs/promises') - return parseJsonContent(await fs.readFile(filePath, 'utf-8')) + return parseJsonContent(await fs.readFile(filePath)) } /** Parse JSON from a buffer. */ export async function parseJSONBuffer(buffer: Buffer): Promise { - return parseJsonContent(buffer.toString('utf-8')) + return parseJsonContent(buffer) } function* iterateJsonLines(content: string): Generator<{ line: string; lineNumber: number }> { @@ -201,12 +269,13 @@ function* iterateJsonLines(content: string): Generator<{ line: string; lineNumbe } } -function parseJsonLinesContent(content: string): FileParseResult { +function parseJsonLinesContent(buffer: Uint8Array): FileParseResult { + const decoded = decodeTextBuffer(buffer) const items: unknown[] = [] const budget = { nodes: 0, serializedUnits: 0 } let depth = assertJsonValueWithinLimits([], budget) - for (const { line, lineNumber } of iterateJsonLines(content)) { + for (const { line, lineNumber } of iterateJsonLines(decoded.text)) { let item: unknown try { item = JSON.parse(line) @@ -229,6 +298,7 @@ function parseJsonLinesContent(content: string): FileParseResult { keys: [], itemCount: items.length, depth, + ...encodingMetadata(decoded), }, } } @@ -236,10 +306,10 @@ function parseJsonLinesContent(content: string): FileParseResult { /** Parse a JSON Lines file. */ export async function parseJSONL(filePath: string): Promise { const fs = await import('fs/promises') - return parseJsonLinesContent(await fs.readFile(filePath, 'utf-8')) + return parseJsonLinesContent(await fs.readFile(filePath)) } /** Parse JSON Lines from a buffer. */ export async function parseJSONLBuffer(buffer: Buffer): Promise { - return parseJsonLinesContent(buffer.toString('utf-8')) + return parseJsonLinesContent(buffer) } diff --git a/apps/sim/lib/file-parsers/md-parser.ts b/apps/sim/lib/file-parsers/md-parser.ts index a97e9450dfe..1ba52e16b7d 100644 --- a/apps/sim/lib/file-parsers/md-parser.ts +++ b/apps/sim/lib/file-parsers/md-parser.ts @@ -1,7 +1,7 @@ import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' import type { FileParseResult, FileParser } from '@/lib/file-parsers/types' -import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' +import { decodeTextBuffer, sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' const logger = createLogger('MdParser') @@ -25,14 +25,16 @@ export class MdParser implements FileParser { try { logger.info('Parsing buffer, size:', buffer.length) - const result = buffer.toString('utf-8') - const content = sanitizeTextForUTF8(result) + const decoded = decodeTextBuffer(buffer) + const content = sanitizeTextForUTF8(decoded.text) return { content, metadata: { characterCount: content.length, tokenCount: Math.floor(content.length / 4), + encoding: decoded.encoding, + ...(decoded.warning ? { warning: decoded.warning } : {}), }, } } catch (error) { diff --git a/apps/sim/lib/file-parsers/odf-text.test.ts b/apps/sim/lib/file-parsers/odf-text.test.ts new file mode 100644 index 00000000000..519c162b4bd --- /dev/null +++ b/apps/sim/lib/file-parsers/odf-text.test.ts @@ -0,0 +1,201 @@ +/** + * @vitest-environment node + */ +import JSZip, { type JSZipObject } from 'jszip' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { FileParserError } from '@/lib/file-parsers/errors' +import { extractOpenDocumentText } from '@/lib/file-parsers/odf-text' +import { MAX_OFFICE_TEXT_BYTES, MAX_OFFICE_XML_PART_BYTES } from '@/lib/file-parsers/office-text' + +const NS = + 'xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:presentation="urn:oasis:names:tc:opendocument:xmlns:presentation:1.0" xmlns:dc="http://purl.org/dc/elements/1.1/"' + +async function buildOdf(bodyXml: string, extraParts: Record = {}): Promise { + const zip = new JSZip() + zip.file('mimetype', 'application/vnd.oasis.opendocument.text', { compression: 'STORE' }) + zip.file( + 'content.xml', + `${bodyXml}` + ) + for (const [path, xml] of Object.entries(extraParts)) zip.file(path, xml) + return zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }) as Promise +} + +const text = (body: string) => buildOdf(`${body}`) + +describe('extractOpenDocumentText', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('drops annotations so the surrounding sentence stays intact', async () => { + const buffer = await text( + `Aaa MMFirst comment.comment ccc.` + ) + + expect(await extractOpenDocumentText(buffer)).toBe('Aaa comment ccc.') + }) + + it('treats tracked deletions as accepted and keeps insertions', async () => { + const buffer = await text( + `Mdeleted words` + + `Kept sentence.` + ) + + expect(await extractOpenDocumentText(buffer)).toBe('Kept sentence.') + }) + + it('renders headings, paragraphs, and nested lists', async () => { + const buffer = await text( + `PurposeBody line.` + + `oneone-atwo` + ) + + expect(await extractOpenDocumentText(buffer)).toBe( + 'Purpose\n\nBody line.\n\n• one\n • one-a\n• two' + ) + }) + + it('keeps header rows and expands repeated columns up to the cap', async () => { + const cell = (value: string, repeat?: number) => + `${value}` + const buffer = await text( + `${cell('Role')}${cell('Contact')}` + + `${cell('Owner')}${cell('x', 2)}` + + `${cell('', 1024)}` + ) + + expect(await extractOpenDocumentText(buffer)).toBe( + '[Table]\n| Role | Contact |\n| Owner | x | x |\n[/Table]' + ) + }) + + it('expands whitespace elements and appends footnotes after the paragraph', async () => { + const buffer = await text( + `ABCD1snoskaNext.` + ) + + expect(await extractOpenDocumentText(buffer)).toBe('A B\tC\nD[1]\n[1] snoska\n\nNext.') + }) + + it('emits presentation notes only when they have a body, skipping page chrome', async () => { + const buffer = await buildOdf( + `Slide title` + + `1` + + `Say hello` + + `Second slide` + ) + + expect(await extractOpenDocumentText(buffer)).toBe( + 'Slide title\n\n[Notes]\nSay hello\n\nSecond slide' + ) + }) + + it('flattens a nested table into its cell without markers', async () => { + const cell = (inner: string) => `${inner}` + const p = (t: string) => `${t}` + const inner = `${cell(p('In 1'))}${cell(p('In 2'))}${cell(p('In 3'))}` + const buffer = await text( + `${cell(p('Out A'))}${cell(p('Intro') + inner)}` + ) + + const result = await extractOpenDocumentText(buffer) + + expect(result).toBe('[Table]\n| Out A | Intro In 1 / In 2 / In 3 |\n[/Table]') + }) + + it('caps a text:s run instead of allocating what text:c asks for', async () => { + const buffer = await text(`AB`) + + expect(await extractOpenDocumentText(buffer)).toBe(`A${' '.repeat(100)}B`) + }) + + it('handles a long whitespace run inside the budget in linear time', async () => { + const spaces = ''.repeat(2_000) + const buffer = await text(`x${spaces}yz`) + + const started = performance.now() + const result = await extractOpenDocumentText(buffer) + + expect(result).toBe(`x${' '.repeat(200_000)}y\nz`) + expect(performance.now() - started).toBeLessThan(5_000) + }, 60_000) + + it('rejects a document whose expanded text exceeds the ceiling', async () => { + const spaces = ''.repeat(Math.ceil(MAX_OFFICE_TEXT_BYTES / 100) + 1) + const buffer = await text(`x${spaces}y`) + + await expect(extractOpenDocumentText(buffer)).rejects.toMatchObject({ + code: 'complexity_limit', + }) + }) + + it('stops walking at the ceiling before inflating later parts', async () => { + const spaces = ''.repeat(Math.ceil(MAX_OFFICE_TEXT_BYTES / 100) + 1) + const buffer = await buildOdf(`x${spaces}y`, { + 'Object 1/content.xml': `Embedded`, + }) + const zip = await JSZip.loadAsync(buffer) + const embedded = zip.file('Object 1/content.xml') as JSZipObject + const inflate = vi.spyOn(embedded, 'async') + vi.spyOn(JSZip, 'loadAsync').mockResolvedValueOnce(zip) + + await expect(extractOpenDocumentText(buffer)).rejects.toMatchObject({ + code: 'complexity_limit', + }) + expect(inflate).not.toHaveBeenCalled() + }) + + it('rejects an archive without content.xml as invalid_format', async () => { + const zip = new JSZip() + zip.file('mimetype', 'application/vnd.oasis.opendocument.text', { compression: 'STORE' }) + zip.file('junk.txt', 'not a document') + const buffer = (await zip.generateAsync({ type: 'nodebuffer' })) as Buffer + + await expect(extractOpenDocumentText(buffer)).rejects.toMatchObject({ + code: 'invalid_format', + }) + }) + + it('returns an empty string for a present but textless body', async () => { + expect(await extractOpenDocumentText(await text(''))).toBe('') + }) + + it('emits an image frame as its alternative text', async () => { + const buffer = await text( + `Org chart` + ) + + expect(await extractOpenDocumentText(buffer)).toBe('[Image: Org chart]') + }) + + it('drops a file-name image title', async () => { + const buffer = await text( + `python-icon.jpegafter` + ) + + expect(await extractOpenDocumentText(buffer)).toBe('after') + }) + + it('rejects a content part above the per-part size cap before parsing it', async () => { + const buffer = await text('Small') + const zip = await JSZip.loadAsync(buffer) + const entry = zip.file('content.xml') as JSZipObject & { + _data: { uncompressedSize: number } + } + entry._data.uncompressedSize = MAX_OFFICE_XML_PART_BYTES + 1 + vi.spyOn(JSZip, 'loadAsync').mockResolvedValueOnce(zip) + + await expect(extractOpenDocumentText(buffer)).rejects.toMatchObject({ + code: 'complexity_limit', + }) + }) + + it('includes embedded object content parts after the main document', async () => { + const buffer = await buildOdf(`Main`, { + 'Object 1/content.xml': `Embedded`, + }) + + expect(await extractOpenDocumentText(buffer)).toBe('Main\n\nEmbedded') + }) +}) diff --git a/apps/sim/lib/file-parsers/odf-text.ts b/apps/sim/lib/file-parsers/odf-text.ts new file mode 100644 index 00000000000..d78d8a111b9 --- /dev/null +++ b/apps/sim/lib/file-parsers/odf-text.ts @@ -0,0 +1,368 @@ +import JSZip from 'jszip' +import { FileParserError } from '@/lib/file-parsers/errors' +import { + assertTextWithinLimit, + chargeTextBudget, + childElements, + collapseWhitespace, + findFirst, + formatTableRow, + imageAltText, + isXmlElement, + joinBlocks, + NOTES_MARKER, + parseXml, + readXmlPart, + TABLE_CLOSE, + TABLE_OPEN, + type TextBudget, + trimLineEnds, + type XmlElement, +} from '@/lib/file-parsers/office-text' +import type { FileParseOptions } from '@/lib/file-parsers/types' + +/** + * Structured text extraction for OpenDocument text and presentation packages + * (`.odt`, `.odp`) that walks `content.xml` in document order. Headings and + * paragraphs become lines, lists get `•` markers with nesting indents, tables + * are rendered row by row, footnotes are appended after their paragraph, and + * reviewer annotations plus tracked deletions are dropped the way pandoc, + * odfpy, and LibreOffice's own text export drop them. + * + * Only `content.xml` and embedded `Object N/content.xml` parts are inflated. + */ + +const CONTENT_PART = 'content.xml' +const EMBEDDED_CONTENT_PART = /^Object (\d+)\/content\.xml$/ + +/** Subtrees whose text is review metadata rather than document content. */ +const SKIPPED_SUBTREES = new Set([ + 'office:annotation', + 'office:annotation-end', + 'text:tracked-changes', + 'office:change-info', + 'text:sequence-decls', + 'text:variable-decls', + 'text:user-field-decls', + 'office:forms', +]) + +/** Presentation frames that render layout chrome rather than slide content. */ +const SKIPPED_PRESENTATION_CLASSES = new Set(['header', 'footer', 'date-time', 'page-number']) + +/** Bounds `table:number-columns-repeated`, which spreadsheets inflate to 1024. */ +const MAX_REPEATED_COLUMNS = 32 + +/** Bounds one `text:s` run; `text:c` is attacker-controlled and would otherwise size an allocation. */ +const MAX_SPACE_RUN = 100 + +const MAX_LIST_INDENT = 3 + +interface WalkState { + blocks: string[] + /** Footnote bodies gathered while rendering the current paragraph. */ + pendingNotes: string[] + /** Inside a table cell, nested tables flatten to text rather than emitting markers. */ + inCell: boolean + /** Shared across the document so cells and note bodies count toward one ceiling. */ + budget: TextBudget +} + +function newState(budget: TextBudget, inCell = false): WalkState { + return { blocks: [], pendingNotes: [], inCell, budget } +} + +/** Records a piece of emitted text against the document ceiling before it is kept. */ +function emit(state: WalkState, pieces: string[], text: string): void { + chargeTextBudget(state.budget, text.length) + pieces.push(text) +} + +function isSkipped(element: XmlElement): boolean { + if (SKIPPED_SUBTREES.has(element.name)) return true + if (element.name === 'draw:frame') { + const presentationClass = element.attribs['presentation:class'] + return presentationClass !== undefined && SKIPPED_PRESENTATION_CLASSES.has(presentationClass) + } + return false +} + +/** + * Inline text of a paragraph-like element, expanding ODF whitespace elements and + * collecting footnote bodies into `state.pendingNotes`. + */ +function inlineText(element: XmlElement, state: WalkState): string { + const pieces: string[] = [] + for (const child of element.children) { + if (child.type === 'text') { + emit(state, pieces, child.data) + continue + } + if (!isXmlElement(child) || isSkipped(child)) continue + + switch (child.name) { + case 'text:s': { + const count = Number.parseInt(child.attribs['text:c'] ?? '1', 10) + const bounded = Number.isFinite(count) && count > 0 ? Math.min(count, MAX_SPACE_RUN) : 1 + emit(state, pieces, ' '.repeat(bounded)) + break + } + case 'text:tab': + emit(state, pieces, '\t') + break + case 'text:line-break': + emit(state, pieces, '\n') + break + case 'draw:frame': { + const image = imageFrameText(child, state) + pieces.push(image === null ? inlineText(child, state) : image) + break + } + case 'text:note': { + const citation = findFirst(child, 'text:note-citation') + const body = findFirst(child, 'text:note-body') + const label = citation ? collapseWhitespace(inlineText(citation, state)) : '' + const bodyText = body ? collapseWhitespace(blockText(body, state.budget)) : '' + if (label) pieces.push(`[${label}]`) + if (bodyText) state.pendingNotes.push(label ? `[${label}] ${bodyText}` : bodyText) + break + } + default: + pieces.push(inlineText(child, state)) + } + } + return pieces.join('') +} + +/** Renders a container's block children to a single string, for cells and note bodies. */ +function blockText(container: XmlElement, budget: TextBudget, inCell = false): string { + const state = newState(budget, inCell) + walkChildren(container, state, 0) + return [...state.blocks, ...state.pendingNotes].join('\n') +} + +function flushNotes(state: WalkState): void { + if (state.pendingNotes.length === 0) return + state.blocks.push(...state.pendingNotes) + state.pendingNotes = [] +} + +function emitParagraph(element: XmlElement, state: WalkState, heading: boolean): void { + const text = trimLineEnds(inlineText(element, state)).trim() + if (text) { + state.blocks.push(heading ? `\n${text}\n` : text) + } + flushNotes(state) + if (text) state.blocks.push('') +} + +function emitList(list: XmlElement, state: WalkState, depth: number): void { + const indent = ' '.repeat(Math.min(depth, MAX_LIST_INDENT)) + for (const item of childElements(list)) { + if (item.name !== 'text:list-item' && item.name !== 'text:list-header') continue + let markerPending = item.name === 'text:list-item' + for (const child of childElements(item)) { + if (isSkipped(child)) continue + if (child.name === 'text:list') { + emitList(child, state, depth + 1) + continue + } + if (child.name === 'text:p' || child.name === 'text:h') { + const text = collapseWhitespace(inlineText(child, state)) + if (text) { + state.blocks.push(markerPending ? `${indent}• ${text}` : `${indent} ${text}`) + markerPending = false + } + flushNotes(state) + continue + } + walkElement(child, state, depth + 1) + } + } +} + +function cellText(cell: XmlElement, budget: TextBudget): string { + return collapseWhitespace(blockText(cell, budget, true)) +} + +function repeatCount(element: XmlElement, attribute: string, cap: number): number { + const raw = element.attribs[attribute] + if (raw === undefined) return 1 + const parsed = Number.parseInt(raw, 10) + if (!Number.isFinite(parsed) || parsed < 1) return 1 + return Math.min(parsed, cap) +} + +function tableRows(container: XmlElement, rows: string[], state: WalkState): void { + for (const child of childElements(container)) { + if (isSkipped(child)) continue + switch (child.name) { + case 'table:table-row': { + const cells: string[] = [] + for (const cell of childElements(child)) { + if (cell.name !== 'table:table-cell' && cell.name !== 'table:covered-table-cell') continue + const text = cellText(cell, state.budget) + const repeats = repeatCount(cell, 'table:number-columns-repeated', MAX_REPEATED_COLUMNS) + for (let i = 0; i < repeats; i++) cells.push(text) + } + if (cells.some((cell) => cell.length > 0)) rows.push(formatTableRow(cells)) + break + } + case 'table:table-header-rows': + case 'table:table-rows': + case 'table:table-row-group': + tableRows(child, rows, state) + break + default: + break + } + } +} + +/** Every non-empty cell of a table in reading order, for a table nested inside a cell. */ +function flattenedCells(container: XmlElement, cells: string[], state: WalkState): void { + for (const child of childElements(container)) { + if (isSkipped(child)) continue + if (child.name === 'table:table-row') { + for (const cell of childElements(child)) { + if (cell.name !== 'table:table-cell' && cell.name !== 'table:covered-table-cell') continue + const text = cellText(cell, state.budget) + if (text) cells.push(text) + } + } else if ( + child.name === 'table:table-header-rows' || + child.name === 'table:table-rows' || + child.name === 'table:table-row-group' + ) { + flattenedCells(child, cells, state) + } + } +} + +function emitTable(table: XmlElement, state: WalkState): void { + if (state.inCell) { + const cells: string[] = [] + flattenedCells(table, cells, state) + if (cells.length > 0) state.blocks.push(cells.join(' / ')) + return + } + const rows: string[] = [] + tableRows(table, rows, state) + if (rows.length > 0) { + state.blocks.push('', TABLE_OPEN, ...rows, TABLE_CLOSE, '') + } +} + +/** + * A frame holding an image contributes its alternative text, as `` + * does in HTML. Returns `null` for a frame that is not an image (a text box). + */ +function imageFrameText(frame: XmlElement, state: WalkState): string | null { + const children = childElements(frame) + if (!children.some((child) => child.name === 'draw:image')) return null + const alt = children.find((child) => child.name === 'svg:title' || child.name === 'svg:desc') + return alt ? (imageAltText(inlineText(alt, state)) ?? '') : '' +} + +function emitNotes(notes: XmlElement, state: WalkState): void { + const body = collapseWhitespace(blockText(notes, state.budget)) + if (body) state.blocks.push(NOTES_MARKER, body) +} + +function walkElement(element: XmlElement, state: WalkState, depth: number): void { + if (isSkipped(element)) return + + switch (element.name) { + case 'text:h': + emitParagraph(element, state, true) + break + case 'text:p': + emitParagraph(element, state, false) + break + case 'text:list': + emitList(element, state, depth) + if (depth === 0) state.blocks.push('') + break + case 'draw:frame': { + const image = imageFrameText(element, state) + if (image === null) { + walkChildren(element, state, depth) + } else if (image) { + state.blocks.push(image) + } + break + } + case 'table:table': + emitTable(element, state) + break + case 'presentation:notes': + emitNotes(element, state) + break + case 'draw:page': + walkChildren(element, state, depth) + state.blocks.push('') + break + default: + walkChildren(element, state, depth) + } +} + +function walkChildren(container: XmlElement, state: WalkState, depth: number): void { + for (const child of childElements(container)) { + walkElement(child, state, depth) + } +} + +function contentBlocks(contentXml: string, budget: TextBudget): string[] { + const document = parseXml(contentXml) + const body = findFirst(document, 'office:body') + if (!body) return [] + const state = newState(budget) + walkChildren(body, state, 0) + flushNotes(state) + return state.blocks +} + +function embeddedContentParts(zip: JSZip): string[] { + const parts: Array<{ index: number; path: string }> = [] + for (const path of Object.keys(zip.files)) { + const match = EMBEDDED_CONTENT_PART.exec(path) + if (match) parts.push({ index: Number(match[1]), path }) + } + return parts.sort((a, b) => a.index - b.index).map((part) => part.path) +} + +/** + * Extracts structured text from an OpenDocument text or presentation package. + * The caller must already have applied the archive size guard; each XML part is + * additionally bounded by {@link readXmlPart} and the assembled text by + * {@link assertTextWithinLimit}. An archive without `content.xml` + * is not an OpenDocument file at all and is rejected as `invalid_format`; a + * present but textless body yields an empty string for the caller to classify. + */ +export async function extractOpenDocumentText( + buffer: Buffer, + options: FileParseOptions = {} +): Promise { + const zip = await JSZip.loadAsync(buffer) + options.signal?.throwIfAborted() + + if (!zip.file(CONTENT_PART)) { + throw new FileParserError( + 'invalid_format', + 'The archive has no content.xml, so it is not an OpenDocument file' + ) + } + + const budget: TextBudget = { used: 0 } + const sections: string[] = [] + for (const path of [CONTENT_PART, ...embeddedContentParts(zip)]) { + const xml = await readXmlPart(zip, path) + options.signal?.throwIfAborted() + if (xml === null) continue + const blocks = contentBlocks(xml, budget) + if (blocks.length > 0) sections.push(joinBlocks(blocks), '') + } + + return assertTextWithinLimit(joinBlocks(sections)) +} diff --git a/apps/sim/lib/file-parsers/office-text.ts b/apps/sim/lib/file-parsers/office-text.ts new file mode 100644 index 00000000000..bd78699dbfc --- /dev/null +++ b/apps/sim/lib/file-parsers/office-text.ts @@ -0,0 +1,175 @@ +import { DomUtils, parseDocument } from 'htmlparser2' +import type JSZip from 'jszip' +import type { JSZipObject } from 'jszip' +import { FileParserError } from '@/lib/file-parsers/errors' + +/** + * Shared XML primitives for the OOXML and OpenDocument structured-text walkers. + * Both formats are ZIP archives of namespaced XML parts; htmlparser2 in XML mode + * keeps the `prefix:local` tag names verbatim, so the walkers match on them + * directly without a namespace-aware parser. + */ + +export type XmlDocument = ReturnType +export type XmlNode = XmlDocument['children'][number] +export type XmlElement = Extract + +/** Opens a structured table block in walker output. */ +export const TABLE_OPEN = '[Table]' + +/** Closes a structured table block in walker output. */ +export const TABLE_CLOSE = '[/Table]' + +/** Introduces presenter notes that follow a slide's body text. */ +export const NOTES_MARKER = '[Notes]' + +/** + * Bounds a single XML part before it is parsed. htmlparser2 retains roughly + * 25 bytes of DOM per byte of markup, so the archive guard's 64 MB entry cap + * alone would let one slide or `content.xml` part cost over a gigabyte. + */ +export const MAX_OFFICE_XML_PART_BYTES = 16 * 1024 * 1024 + +function declaredUncompressedSize(entry: JSZipObject): number | undefined { + const data = (entry as JSZipObject & { _data?: { uncompressedSize?: number } })._data + const size = data?.uncompressedSize + return typeof size === 'number' && Number.isFinite(size) ? size : undefined +} + +/** + * Hard ceiling on the text a walker assembles from one document. The part cap + * bounds the markup, but ODF whitespace and repeat attributes can expand a + * small part many times over, so the output is bounded on its own. + */ +export const MAX_OFFICE_TEXT_BYTES = MAX_OFFICE_XML_PART_BYTES + +/** Running total of emitted text, shared by every walk state of one document. */ +export interface TextBudget { + used: number +} + +export function chargeTextBudget(budget: TextBudget, length: number): void { + budget.used += length + if (budget.used > MAX_OFFICE_TEXT_BYTES) { + throw new FileParserError( + 'complexity_limit', + `Document text exceeds the maximum of ${MAX_OFFICE_TEXT_BYTES} bytes` + ) + } +} + +/** The assembled output must fit the same ceiling once joined. */ +export function assertTextWithinLimit(text: string): string { + const bytes = Buffer.byteLength(text, 'utf8') + if (bytes > MAX_OFFICE_TEXT_BYTES) { + throw new FileParserError( + 'complexity_limit', + `Document text is ${bytes} bytes, above the maximum of ${MAX_OFFICE_TEXT_BYTES} bytes` + ) + } + return text +} + +function xmlPartTooLarge(path: string, bytes: number): FileParserError { + return new FileParserError( + 'complexity_limit', + `Document part ${path} is ${bytes} bytes, above the maximum of ${MAX_OFFICE_XML_PART_BYTES} bytes` + ) +} + +/** + * Inflates one XML part as a string, or returns `null` when the archive has no + * such entry. Rejects a part above {@link MAX_OFFICE_XML_PART_BYTES} on its + * declared size before inflating, and on its real size afterwards in case the + * declaration lied. + */ +export async function readXmlPart(zip: JSZip, path: string): Promise { + const entry = zip.file(path) + if (!entry) return null + + const declared = declaredUncompressedSize(entry) + if (declared !== undefined && declared > MAX_OFFICE_XML_PART_BYTES) { + throw xmlPartTooLarge(path, declared) + } + + const xml = await entry.async('string') + const actual = Buffer.byteLength(xml, 'utf8') + if (actual > MAX_OFFICE_XML_PART_BYTES) throw xmlPartTooLarge(path, actual) + return xml +} + +export function parseXml(xml: string): XmlDocument { + return parseDocument(xml, { xmlMode: true }) +} + +export function isXmlElement(node: XmlNode): node is XmlElement { + return node.type === 'tag' +} + +/** Direct element children in document order. */ +export function childElements(node: XmlDocument | XmlElement): XmlElement[] { + return node.children.filter(isXmlElement) +} + +/** First descendant (or the node itself) with the given tag name, in document order. */ +export function findFirst(node: XmlDocument | XmlElement, tagName: string): XmlElement | null { + const found = DomUtils.findOne((element) => element.name === tagName, node.children, true) + return found ?? null +} + +/** Every descendant with the given tag name, in document order. */ +export function findAll(node: XmlDocument | XmlElement, tagName: string): XmlElement[] { + return DomUtils.findAll((element) => element.name === tagName, node.children) +} + +/** A bare file name (`python-logo.gif`) rather than a description. */ +const FILENAME_LIKE = /^[^\s]+\.[a-z0-9]{2,4}$/i + +/** Auto-generated captions that name the object, not its content (`Picture 3`). */ +const AUTO_CAPTION = /^(?:picture|image|graphic|photo|figure|chart|diagram|screenshot)\s*\d*$/i + +/** + * Renders an image's alternative text as `[Image: …]`, or `null` when the text + * is a file name or an auto-generated caption, which would only add noise. + */ +export function imageAltText(raw: string | undefined): string | null { + const text = raw ? collapseWhitespace(raw) : '' + if (!text || FILENAME_LIKE.test(text) || AUTO_CAPTION.test(text)) return null + return `[Image: ${text}]` +} + +/** + * Strips trailing spaces and tabs from every line in linear time. The obvious + * `/[ \t]+\n/` is quadratic on a long whitespace run — each position scans + * the run, fails on the newline, and backtracks — which a document inside the + * text budget can still trigger. + */ +export function trimLineEnds(text: string): string { + return text.includes('\n') + ? text + .split('\n') + .map((line) => line.trimEnd()) + .join('\n') + : text +} + +/** Collapses internal whitespace so a cell or list item occupies a single line. */ +export function collapseWhitespace(text: string): string { + return text.replace(/\s+/g, ' ').trim() +} + +/** Renders one table row in the `| a | b |` shape the HTML walker produces. */ +export function formatTableRow(cells: string[]): string { + return `| ${cells.map(collapseWhitespace).join(' | ')} |` +} + +/** + * Joins emitted blocks with single newlines and squeezes runs of blank lines to + * one, so walker output reads like the HTML walker's. + */ +export function joinBlocks(blocks: string[]): string { + return blocks + .join('\n') + .replace(/\n{3,}/g, '\n\n') + .trim() +} diff --git a/apps/sim/lib/file-parsers/ooxml-encryption.test.ts b/apps/sim/lib/file-parsers/ooxml-encryption.test.ts new file mode 100644 index 00000000000..e7cea4850e6 --- /dev/null +++ b/apps/sim/lib/file-parsers/ooxml-encryption.test.ts @@ -0,0 +1,89 @@ +/** + * @vitest-environment node + * + * The fixtures are real password-protected packages (`msoffcrypto-tool -e -p x`) + * around one-part OOXML documents, gzipped because an OLE container is mostly + * zero-padded sectors. + */ +import { gunzipSync } from 'zlib' +import { describe, expect, it } from 'vitest' +import { parseBuffer } from '@/lib/file-parsers' +import { DocxParser } from '@/lib/file-parsers/docx-parser' +import type { FileParserError } from '@/lib/file-parsers/errors' +import { isEncryptedOoxmlContainer } from '@/lib/file-parsers/ooxml-encryption' +import { PptxParser } from '@/lib/file-parsers/pptx-parser' +import { sniffFileKind } from '@/lib/file-parsers/sniff' + +const ENCRYPTED_DOCX_GZ_BASE64 = + 'H4sIAGMbomoC/+1X63LaRhRe3xLbvSVpm6Zummj01xMLMObiARKu5maDhbEx/2RpkRTQxbpEiE6foi/QPELfov/aF+hM+7vP0LhHQtjgS4qcdCbtcDSfVrur3W/POd+u4Ndf7v/++qe1P9AlS6EF9OZsBd0Za5sDzI8q94Z1B2/Ozs5GzWcz+0/ZXwAnfwuQu0Uvl07O7wKWvfqqV87s/2ej/bvglTfl/yPAx4BPAJ8CPhseAeg+4AHgc8AXgC8BDwFfAR4BvgasAb4BPAZ8C3gCeDrT1AdhNFLgMhCB8kiGUkM28mMP0dL5mb96wzs/2g/+/OHFb3Ojcgl54gJzWFmXVQV2jDhURwy0dOHOQ/2fjIQv0rg/0/CvQFt/Ydh3B+WAyQA0YAUOM0a6D/8fwRdwxL04pf/jfYfApwGjCFmQkX+7h+bn/PrvtFW9vqve70KpTs2/BvxOKJe8b/80/M5vCPVG/hLEoQPRmM4eoznX/7tuLv3Hv+Fq3ok9f0mLw4xcXd+khW8Rf0cnL7y+A+BjgEd3fdaQ5N//c/0t/wv+X13fZf/nzv1fuQX/HdjvGnBJwOL37EHut8d//B2t/oyuO3+GPvvJwBPgn/d+B/s5f14voQ/CEoBd8Jt1VaADOhCHDZR1I+EoX4TSOaM2rj0pR/8B5m7Jv+yNFcbmIKb4LoTew94d5w/dci+0vPHOefodKsAVR2m0CWUA7s/QForAXAV4CsNT0G3LwMpy0OP0pqE9CNEOQksARWEdznvfoz0feZl2rTflavEd9LMIl3OWJZ73pR7xCmu6qMhJMrgRIAksswonynySbB4UnsVIQjcYmWN6ioyTpI118nlqNQEvabZqwCgCZpD1JCkYhrpNUTorYInRNySR1RRd6RgbrCJRSqcjspgKBQIR6mIoORy7rfoY3cV2fjiBolEqo+uWonGjidjbTsRizRChjzEwmVolwBLQn2MMhtCZntEQB+B7MEISJz2F7V5U4aWMaID3oS2oCYwuDPsiYdKd5dxYURWwlu7xiiYagpQk0/kG6bVmBUaU3YCPnnYVDmcz2eGMY4MaxfRWMES6azpkeiYw9evHUSG9X9zP6cKu3js17biC08kkSVCeIxx4UZINzMMkNuFFH3NFiWEr2E6SxWgsf1DPqSzuW6H+oFnmKdEIxwKDer1ktdgdmdcq3BFHxy1VKUU4I9Irx5l4tlqy6L7J0KHc3qaYwWyb1UIvsW2Ke9qOUNl3VjDB5a23aR0Lxa1ov2oclYPF9awmMdncAOe79dO+1amp5iFt4oJcpLSc0I20u9FCPNe0ZKrNS3y0dsjHalUr3WkJUUHTapkc38Lilm2N+zueWD11noaJdsLUxHcWXWoixQl1+9xfiCuhq6KcVUzZAKUEHCN9SGlSPJdsQmbvV1lv5R1TXeVVfJ895JVqNdI41Te5cuV0L25N5PwQa7CjsFYEopKsmhCHk82yWR1kep0GHexTgRa9b9NBR6tvpb12Rm8dgz62jmjVzq9LMTrSs/mDWn2Qjm1WqMpuiellysVolRustyOiKOUbZkBqxfXjTLDHtw/SdBi3mu2XSss6CbHtaqbF8SdH+YPs8aR2IZcj6Z6adSUv7uGddT5Xa/ZZaT1c4HE/TDeOG5VOM23Uwqp8agYK7IUcXWlMCMiTKXVJp4mxgzG1imY2s5nN7Hr7GzWns/kAGAAA' + +const ENCRYPTED_PPTX_GZ_BASE64 = + 'H4sIAGMbomoC/+1X6XLaRhxffCS2ezk90tRNE436kYnFfXgMCWc5jY1ijPkmJCHJQgeSDIhOn6JPkEfoW/Rb+wKdaT/3GRr3LyEw+EiRk86kHX6an1a7q93f/o9dwa+/PPj91U87f6ArSKJV9PpiE92bafMAVyaV7XHd4uuLi4tJ88US/yn8BbTitwqxW3NiacX8PnDDqW855RL/P0z276pT3hb/D4AfAj8Cfgz8ZHwEoAfAT4GfAT8HfgF8CPwS+Aj4FXAH+DXwMfAb4BPg02VOvReoIwUuA2Eoh2QoNWQiN3iI1qdn/tYt72yb3/7544vfPJNyHTnJBbBUaVtVBXUWMegQUdAiwp2D+j8Bhy/SrD2L6G9CW3N13HcPZUHJAJKwAkuZRboL+x/BF3Civbag/bN9DdDTQFGAKMjIPbbRiset/VZbxem7bn0VSnVh/R3Qt1y57nz7F9G3fkOot+oXwQ8d8MZieIw8tv337Vi69z9p57zle+5KLo4jcn198wjdwf9Wnrxw+l6CHgU6um2zhiT39k/zb+NfsP/6+q7a75nav3kH/Xuw3zXQkkDF7dmD7G+Pe/9bufozuun8GdvsJgJPQH/F+R3s5vx5tY7eC+wDq2A3bWeBDuyAH3ZRxvaElfkClNYZtXvjSTn5D+C5o/6GM5afmQNb4LsQeAd7d1Y/cMe90HTGW+fp9ygPVxylUBBKH9yfoTCKwFx5eArBk99uS8PKstBj9aag3Q/e9kOLD0VhHdZ7P6ADF3FZdK23xWrtLfJnDS7rLNt/PpS6WJ/VdEGRE7h/14djrEwrjCBzCfz4Zf5ZDMd0g5IZqqvIbAI3WR1/ntzah5c0UzVgFAYzyHoC5w1D3SMIneZZidJ3JYHWFF3pGLu0IhFKpyPQLBHw+SLE5VB8PHZPdTFaZM3ceAJFI1RK1weKxkwmou86Ec1qhgB9lMHiyS0MsA/9WcqgMJ3qGqQwAtv9ERxrdxVavKzCS2nBAOsDYajxlM6P+yIh3J5lClpQeVZLdTlFEwxeSuCpHIk7rRmeEmTb4ZOnqsKwmXRmPOPMILKQCvsDuL2mBtU9ByWjUeo3Cp1ooXQSPTgQK4ejQqZ+lEjgGOEYwoAVRdlgOZjExBzvs0xBougyayZwbzx1RjPUSY3zBYM9PlApZWOh7qE3HY360t5cLO7jQsHsca0qNc9CEZMvyrUo02kRXKffaqfj1GnmSM61BrWIVulUvyNz/kM9yA6sFcxpOeuNnxLkcEie5Ml2RSI4pVxmcrKXPxO9ZpiRWnwuJvfystDS2iO5e3zaaxV5PdALV7zhiJQelmIcdWiGQhSXP2YDjJFXqrU6FeNm7Z0NrJ6chmGuHTvXhLdOuuRciPfVvam94FdMVwU5o5zLBmSKzwLuIpXmk+cK5tLs3WbWG3Vnsm7YIVizTFVSPjJrjkLZaLvX0eZj3mA12FGsVgChoqyegx/C2rEharnqSZnyM1xcUgVCVKzYvVH2xhmddbRK/WGkeMCHRyLfF1PRFKlJDNlUKKGYZcXwmdlrZKrNYIb3tc979WqgXsvmvWJTHXTVYbd9FB4OhFCdSRdSLGmWut6mqJn1wbwdEEtHrJqmg8qo1OWH3pxeSkfNdinOHx0V6+1+q1hnBlwtTmVTfrUcyl2mo50acwnkpClxJU/3Zw7G5BZaYokllrgZfwOgB9C8ABgAAA==' + +const ENCRYPTED_XLSX_GZ_BASE64 = + 'H4sIAGMbomoC/+1X23LaRhhefIrtnpIe0tRNE41umRgJCWw8QILBnAzEgG0OVxWSEDLoYEkc5E6fok+Q6171LXrXvkBn2us+Q+P+EgIDtlPkpDNph0/zabW72v32P+wKfv3lwe+vftr6A80gipbR68sNtDbR5gEujSr3h3WLry8vL0fNlwv8p/AX0IrfMsRuxYmlFfN7wHWnvumUC/z/MNq/y055W/w/AH4I/Aj4MfCT4RGAHgA/BX4G/Bz4BfAh8EvgI+BXwC3g18DHwG+AT4BPFzn1XqCEFLgMhKEDJEOpIRO5wUO0Oj7zN29559sf6T9/ePGbZ1SuIie5AJYqa6uqoM4jDh0hBlracBeg/k/A4Ys0ac88+hvQ1loe9q2hBCgZwDKswFLmke7C/kfwBRxpr8xp/2TfKehpoChCFGTkHvfRkset/VZbzum7bn0eSnVu/S3Qt1y56nz759G3fkOot+pnwA9N8MZ8eIw8tv337Fi693/ZznnL98JMLg4jcn1906Dv4H8rT144fcegx4CObtusIcm9/eP8W/8X7L++vln7PWP7N+6gvwb7XQMtCVTcnj3I/va497+Vqz+jm86foc1uIvAE9Jec38Fuzp9Xq+i9QBiYB7tZOwt0YBP8sI3itieszBehtM6o7RtPytF/AM8d9dedsa2JObA5vgv+d7B3J/X9d9wLVWe8dZ5+h5JwhVAMUVAScH+GAigIcyXhiYYn0m7bh5UloMfqjUE7Cd4moYVAO7AO673vUcFFXOZd622xWnmL/FmByzrLws8HUgfr8ZouKnIEJ7cJHONlVuFEWYjgJ8fJZ7s4phuMzDEdReYjuMnr+PPoZhhe0kzVgFEYzCDrEbxlGOqez6ezLV5i9G1JZDVFV5rGNqtIPqXZFFne5yeIoO9qKD4cu6e6GN3mzYPhBIrmUxld7ysaN5qIvetELK8ZIvQxBo9HNzFAGPoTjMFgOtMxyuIF2E4GcazRUdj2VRVe2hcNsN4fgFqL0VvDviCN27OMwYpqi9diHUHRRKMlRfDYQRl3WuMtRpRth4+e8grHx/fjwxknBpXTsQDpx+01nTKdLih1iXqZERLJE38zxWZ9sTQZSGdjkQiO+RxDOLAiIxu8AJOYmON9nktLDHvImxF8IB0mzmXqqKt0S4F+vjI4alF580xkBIol6WRaEQoFqqYM9IZWkrK1smBktYNeRS3opXQzWaOzfvqEDcgDLlZTy0HVVEK5RNJewZSWs15/lUjlKFOudWOH+XSlKbFBszrIky9DNZo8ZXcprtI2qVyPyqd8fJIpqWklM9gNlU6Pq+f16lmoGSQ5PUWalNCj+tn9Klelgrwwae9kYPXoOAxT7VhXE9866aJTIQ6re2N7wa+YropyXOnKBmQKYQF3kUrTyTODqTR7t5n1Rt2JrIuTOiOfFQ+93X4iY3rVVrt77i9OxfyU12BH8VoahDKy2gU/BAe9UL5c7JB1gZbriXLD8BOVPox6o+yNMzrrOJa8TIM76hTLR8FkQz3TG72LC4LY0RS2QPaEtlyrZHr1XIYOFnckb7+XrZdyXqbXqpybHC0JHbNYuTgv6EVCziuZY0n0Vnf7M3ZALB0xOtXoGUFvl31Z3U0N2HIgbtRZPRWva2d8P1Vp1NtmWuIpuqPQV+lop8ZUAjlp6pvJ0/DEwRjdRAsssMACN+NvkOlqlgAYAAA=' + +const decode = (gzBase64: string): Buffer => gunzipSync(Buffer.from(gzBase64, 'base64')) +const encryptedDocx = decode(ENCRYPTED_DOCX_GZ_BASE64) +const encryptedPptx = decode(ENCRYPTED_PPTX_GZ_BASE64) +const encryptedXlsx = decode(ENCRYPTED_XLSX_GZ_BASE64) + +const OLE2_MAGIC = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]) + +describe('isEncryptedOoxmlContainer', () => { + it('recognizes encrypted Word, PowerPoint, and Excel packages', () => { + expect(isEncryptedOoxmlContainer(encryptedDocx)).toBe(true) + expect(isEncryptedOoxmlContainer(encryptedPptx)).toBe(true) + expect(isEncryptedOoxmlContainer(encryptedXlsx)).toBe(true) + }) + + it('does not flag a plain OLE container or a non-OLE buffer', () => { + expect(isEncryptedOoxmlContainer(Buffer.concat([OLE2_MAGIC, Buffer.alloc(4096)]))).toBe(false) + expect( + isEncryptedOoxmlContainer( + Buffer.concat([Buffer.from('PK'), Buffer.from('EncryptedPackage', 'utf16le')]) + ) + ).toBe(false) + }) + + it('requires both streams in the directory', () => { + const onlyPackage = Buffer.concat([ + OLE2_MAGIC, + Buffer.alloc(512), + Buffer.from('EncryptedPackage', 'utf16le'), + ]) + expect(isEncryptedOoxmlContainer(onlyPackage)).toBe(false) + }) +}) + +describe('encrypted package routing', () => { + it('sniffs an encrypted package as its own kind', () => { + expect(sniffFileKind(encryptedPptx)).toBe('encrypted-ooxml') + expect(sniffFileKind(Buffer.concat([OLE2_MAGIC, Buffer.alloc(4096)]))).toBe('ole2') + }) + + it.each([ + ['docx', encryptedDocx], + ['pptx', encryptedPptx], + ['xlsx', encryptedXlsx], + ['doc', encryptedDocx], + ])('rejects an encrypted package labelled .%s as encrypted_file', async (extension, buffer) => { + await expect(parseBuffer(buffer, extension)).rejects.toMatchObject({ + code: 'encrypted_file', + }) + }) + + it('classifies an encrypted deck before the legacy .ppt rejection', async () => { + await expect( + new PptxParser().parseBuffer(encryptedPptx) + ).rejects.toMatchObject({ code: 'encrypted_file' }) + }) + + it('classifies an encrypted document before the plaintext fallback', async () => { + const error = await new DocxParser() + .parseBuffer(encryptedDocx) + .catch((caught: unknown) => caught) + + expect(error).toMatchObject({ code: 'encrypted_file' }) + }) +}) diff --git a/apps/sim/lib/file-parsers/ooxml-encryption.ts b/apps/sim/lib/file-parsers/ooxml-encryption.ts new file mode 100644 index 00000000000..b1e212bfa44 --- /dev/null +++ b/apps/sim/lib/file-parsers/ooxml-encryption.ts @@ -0,0 +1,30 @@ +/** + * Detects an encrypted OOXML document. Word, Excel, and PowerPoint wrap a + * password-protected `.docx`/`.xlsx`/`.pptx` in an OLE2 compound file whose + * directory holds the `EncryptionInfo` and `EncryptedPackage` streams, so the + * bytes look like a legacy binary while the ZIP package inside is unreadable. + * The ZIP parsers and officeparser both fail on it with generic messages, which + * is why the sniff has to recognize it up front. + */ + +const OLE2_SIGNATURE = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]) + +/** The directory sits in the first sectors; 64 KiB covers every real file. */ +const DIRECTORY_SCAN_BYTES = 64 * 1024 + +const ENCRYPTED_PACKAGE_STREAM = Buffer.from('EncryptedPackage', 'utf16le') +const ENCRYPTION_INFO_STREAM = Buffer.from('EncryptionInfo', 'utf16le') + +export function isOle2Container(buffer: Buffer): boolean { + return buffer.length >= OLE2_SIGNATURE.length && buffer.subarray(0, 8).equals(OLE2_SIGNATURE) +} + +/** + * Whether the buffer is an OLE2 container carrying an encrypted OOXML package. + * Bounded to the leading {@link DIRECTORY_SCAN_BYTES}; never inflates anything. + */ +export function isEncryptedOoxmlContainer(buffer: Buffer): boolean { + if (!isOle2Container(buffer)) return false + const head = buffer.subarray(0, DIRECTORY_SCAN_BYTES) + return head.includes(ENCRYPTED_PACKAGE_STREAM) && head.includes(ENCRYPTION_INFO_STREAM) +} diff --git a/apps/sim/lib/file-parsers/ooxml-presentation.test.ts b/apps/sim/lib/file-parsers/ooxml-presentation.test.ts new file mode 100644 index 00000000000..cde9e3c3e6d --- /dev/null +++ b/apps/sim/lib/file-parsers/ooxml-presentation.test.ts @@ -0,0 +1,376 @@ +/** + * @vitest-environment node + */ +import JSZip, { type JSZipObject } from 'jszip' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { FileParserError } from '@/lib/file-parsers/errors' +import { MAX_OFFICE_XML_PART_BYTES } from '@/lib/file-parsers/office-text' +import { extractPresentationText } from '@/lib/file-parsers/ooxml-presentation' + +const NS = + 'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"' + +function shape(text: string, placeholderType?: string): string { + const ph = + placeholderType === undefined ? '' : `` + return `${ph}${text}` +} + +function slideXml(spTree: string): string { + return `${spTree}` +} + +function notesXml(spTree: string): string { + return `${spTree}` +} + +interface DeckSlide { + index: number + spTree: string + notesSpTree?: string + /** Extra `` elements for the slide's own rels part. */ + extraRels?: string +} + +const RELS_NS = 'xmlns="http://schemas.openxmlformats.org/package/2006/relationships"' +const REL_TYPE = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships' + +/** Lists the given slide part names in `p:sldIdLst` order, resolved through the presentation rels. */ +function presentationParts(zip: JSZip, order: number[], absoluteTargets = false): void { + const ids = order.map((n, i) => ``).join('') + zip.file( + 'ppt/presentation.xml', + `${ids}` + ) + const rels = order + .map( + (n) => + `` + ) + .join('') + zip.file( + 'ppt/_rels/presentation.xml.rels', + `${rels}` + ) +} + +async function buildDeck( + slides: DeckSlide[], + order?: number[], + absoluteTargets = false +): Promise { + const zip = new JSZip() + zip.file('[Content_Types].xml', '') + zip.file('ppt/media/image1.png', Buffer.from([0x89, 0x50, 0x4e, 0x47])) + if (order) presentationParts(zip, order, absoluteTargets) + for (const slide of slides) { + zip.file(`ppt/slides/slide${slide.index}.xml`, slideXml(slide.spTree)) + const rels: string[] = [] + if (slide.notesSpTree !== undefined) { + rels.push( + `` + ) + zip.file(`ppt/notesSlides/notesSlide${slide.index}.xml`, notesXml(slide.notesSpTree)) + } + if (slide.extraRels) rels.push(slide.extraRels) + if (rels.length > 0) { + zip.file( + `ppt/slides/_rels/slide${slide.index}.xml.rels`, + `${rels.join('')}` + ) + } + } + return zip.generateAsync({ type: 'nodebuffer' }) as Promise +} + +const DIAGRAM_FRAME = `` + +const CHART_FRAME = `` + +function diagramDataXml(points: string[]): string { + const pts = points + .map( + (text, i) => + `${text}` + ) + .join('') + return `${pts}` +} + +const CHART_XML = `Revenue by quarterSheet1!$B$1Sales1st Qtr2nd Qtr10Costs1st Qtr2nd QtrQuarter` + +describe('extractPresentationText', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('emits titles and body paragraphs while skipping layout placeholders', async () => { + const buffer = await buildDeck([ + { + index: 1, + spTree: + shape('Deck Title', 'ctrTitle') + + shape('First point', 'body') + + shape('7', 'sldNum') + + shape('2026-01-01', 'dt') + + shape('Confidential', 'ftr') + + shape('testdoc', 'hdr'), + }, + ]) + + const text = await extractPresentationText(buffer) + + expect(text).toBe('Deck Title\n\nFirst point') + }) + + it('renders a graphic-frame table as rows', async () => { + const cell = (value: string) => + `${value}` + const spTree = + shape('Roles', 'title') + + `${cell('Role')}${cell('Contact')}${cell('Owner')}${cell('ops@example.com')}` + + const text = await extractPresentationText(await buildDeck([{ index: 1, spTree }])) + + expect(text).toContain('[Table]\n| Role | Contact |\n| Owner | ops@example.com |\n[/Table]') + }) + + it('takes only the body placeholder from a notes page', async () => { + const buffer = await buildDeck([ + { + index: 1, + spTree: shape('Slide body', 'body'), + notesSpTree: + shape('testdoc', 'hdr') + + shape('Speaker reminder', 'body') + + shape('1', 'sldNum') + + ``, + }, + ]) + + const text = await extractPresentationText(buffer) + + expect(text).toBe('Slide body\n[Notes]\nSpeaker reminder') + }) + + it('omits the notes marker when the notes body is empty', async () => { + const buffer = await buildDeck([ + { index: 1, spTree: shape('Only slide', 'body'), notesSpTree: shape('3', 'sldNum') }, + ]) + + expect(await extractPresentationText(buffer)).toBe('Only slide') + }) + + it('orders slide10 after slide9 and separates slides with a blank line', async () => { + const buffer = await buildDeck([ + { index: 10, spTree: shape('Tenth') }, + { index: 9, spTree: shape('Ninth') }, + { index: 2, spTree: shape('Second') }, + ]) + + expect(await extractPresentationText(buffer)).toBe('Second\n\nNinth\n\nTenth') + }) + + it('recurses into group shapes in document order', async () => { + const spTree = `${shape('Grouped one')}${shape('Nested two')}${shape('After group')}` + + expect(await extractPresentationText(await buildDeck([{ index: 1, spTree }]))).toBe( + 'Grouped one\nNested two\nAfter group' + ) + }) + + it('joins runs within a paragraph and turns line breaks into newlines', async () => { + const spTree = `Hello worldagain` + + expect(await extractPresentationText(await buildDeck([{ index: 1, spTree }]))).toBe( + 'Hello world\nagain' + ) + }) + + it('walks the fallback branch of an AlternateContent wrapper, else its first choice', async () => { + const spTree = + `${shape('Choice text')}${shape('Fallback text')}` + + `${shape('Only choice')}` + + expect(await extractPresentationText(await buildDeck([{ index: 1, spTree }]))).toBe( + 'Fallback text\nOnly choice' + ) + }) + + it('skips slide-number fields outside their placeholder but keeps date fields', async () => { + const spTree = `Page 3696/29/2021kept` + + expect(await extractPresentationText(await buildDeck([{ index: 1, spTree }]))).toBe( + 'Page 6/29/2021kept' + ) + }) + + it('still drops a date field inside a dt placeholder', async () => { + const spTree = `6/29/2021${shape('Body')}` + + expect(await extractPresentationText(await buildDeck([{ index: 1, spTree }]))).toBe('Body') + }) + + it('emits a picture as its alternative text unless it is a file name or auto caption', async () => { + const pic = (descr: string) => + `` + const spTree = pic('Org chart') + pic('python-logo.gif') + pic('Picture 2') + shape('Caption') + + expect(await extractPresentationText(await buildDeck([{ index: 1, spTree }]))).toBe( + '[Image: Org chart]\nCaption' + ) + }) + + it('ignores a notes relationship that escapes ppt/notesSlides', async () => { + const zip = new JSZip() + zip.file('ppt/slides/slide1.xml', slideXml(shape('Body'))) + zip.file( + 'ppt/slides/_rels/slide1.xml.rels', + `` + ) + zip.file('docProps/app.xml', notesXml(shape('Leaked', 'body'))) + const buffer = (await zip.generateAsync({ type: 'nodebuffer' })) as Buffer + + expect(await extractPresentationText(buffer)).toBe('Body') + }) + + it('rejects a slide part above the per-part size cap before parsing it', async () => { + const buffer = await buildDeck([{ index: 1, spTree: shape('Small') }]) + const zip = await JSZip.loadAsync(buffer) + const entry = zip.file('ppt/slides/slide1.xml') as JSZipObject & { + _data: { uncompressedSize: number } + } + entry._data.uncompressedSize = MAX_OFFICE_XML_PART_BYTES + 1 + vi.spyOn(JSZip, 'loadAsync').mockResolvedValueOnce(zip) + + await expect(extractPresentationText(buffer)).rejects.toMatchObject({ + code: 'complexity_limit', + }) + }) + + it('follows the presentation sldIdLst order rather than part numbering', async () => { + const buffer = await buildDeck( + [ + { index: 1, spTree: shape('One') }, + { index: 2, spTree: shape('Two') }, + { index: 3, spTree: shape('Three') }, + ], + [3, 1, 2] + ) + + expect(await extractPresentationText(buffer)).toBe('Three\n\nOne\n\nTwo') + }) + + it('resolves package-absolute relationship targets against the package root', async () => { + const buffer = await buildDeck( + [ + { index: 1, spTree: shape('One') }, + { index: 2, spTree: shape('Two') }, + { + index: 3, + spTree: shape('Three'), + extraRels: ``, + }, + ], + [3, 1, 2], + true + ) + const zip = await JSZip.loadAsync(buffer) + zip.file('ppt/notesSlides/notesSlide3.xml', notesXml(shape('Absolute note', 'body'))) + const withNotes = (await zip.generateAsync({ type: 'nodebuffer' })) as Buffer + + expect(await extractPresentationText(withNotes)).toBe( + 'Three\n[Notes]\nAbsolute note\n\nOne\n\nTwo' + ) + }) + + it('rejects a package-absolute target outside ppt/', async () => { + const buffer = await buildDeck([ + { + index: 1, + spTree: shape('Body'), + extraRels: ``, + }, + ]) + const zip = await JSZip.loadAsync(buffer) + zip.file('docProps/app.xml', notesXml(shape('Leaked', 'body'))) + const withDecoy = (await zip.generateAsync({ type: 'nodebuffer' })) as Buffer + + expect(await extractPresentationText(withDecoy)).toBe('Body') + }) + + it('skips slide ids whose target is missing and falls back when none resolve', async () => { + const withMissing = await buildDeck( + [ + { index: 1, spTree: shape('One') }, + { index: 2, spTree: shape('Two') }, + ], + [2, 9, 1] + ) + expect(await extractPresentationText(withMissing)).toBe('Two\n\nOne') + + const noneResolve = await buildDeck([{ index: 1, spTree: shape('Only') }], [7]) + expect(await extractPresentationText(noneResolve)).toBe('Only') + }) + + it('reads SmartArt text from the diagram data part in document order', async () => { + const zip = new JSZip() + zip.file('ppt/slides/slide1.xml', slideXml(shape('Process', 'title') + DIAGRAM_FRAME)) + zip.file( + 'ppt/slides/_rels/slide1.xml.rels', + `` + ) + zip.file('ppt/diagrams/data1.xml', diagramDataXml(['Plan', 'Build', 'Ship'])) + zip.file('ppt/diagrams/layout1.xml', '') + const buffer = (await zip.generateAsync({ type: 'nodebuffer' })) as Buffer + + expect(await extractPresentationText(buffer)).toBe('Process\n\nPlan\nBuild\nShip') + }) + + it('summarizes a chart as title, axis titles, series, and categories', async () => { + const zip = new JSZip() + zip.file('ppt/slides/slide1.xml', slideXml(CHART_FRAME)) + zip.file( + 'ppt/slides/_rels/slide1.xml.rels', + `` + ) + zip.file('ppt/charts/chart1.xml', CHART_XML) + const buffer = (await zip.generateAsync({ type: 'nodebuffer' })) as Buffer + + expect(await extractPresentationText(buffer)).toBe( + '[Chart]\nRevenue by quarter\nQuarter\nSales\nCosts\n1st Qtr\n2nd Qtr\n[/Chart]' + ) + }) + + it('ignores a diagram target that escapes ppt/', async () => { + const zip = new JSZip() + zip.file('ppt/slides/slide1.xml', slideXml(shape('Body') + DIAGRAM_FRAME)) + zip.file( + 'ppt/slides/_rels/slide1.xml.rels', + `` + ) + zip.file('docProps/data1.xml', diagramDataXml(['Leaked'])) + const buffer = (await zip.generateAsync({ type: 'nodebuffer' })) as Buffer + + expect(await extractPresentationText(buffer)).toBe('Body') + }) + + it('includes text carried by a connector shape', async () => { + const spTree = `Yes${shape('After')}` + + expect(await extractPresentationText(await buildDeck([{ index: 1, spTree }]))).toBe( + 'Yes\nAfter' + ) + }) + + it('rejects when the signal is already aborted', async () => { + const controller = new AbortController() + controller.abort() + + await expect( + extractPresentationText(await buildDeck([{ index: 1, spTree: shape('x') }]), { + signal: controller.signal, + }) + ).rejects.toThrow() + }) +}) diff --git a/apps/sim/lib/file-parsers/ooxml-presentation.ts b/apps/sim/lib/file-parsers/ooxml-presentation.ts new file mode 100644 index 00000000000..9e82ea2ad79 --- /dev/null +++ b/apps/sim/lib/file-parsers/ooxml-presentation.ts @@ -0,0 +1,462 @@ +import JSZip from 'jszip' +import { + assertTextWithinLimit, + childElements, + findAll, + findFirst, + formatTableRow, + imageAltText, + isXmlElement, + joinBlocks, + NOTES_MARKER, + parseXml, + readXmlPart, + TABLE_CLOSE, + TABLE_OPEN, + trimLineEnds, + type XmlElement, +} from '@/lib/file-parsers/office-text' +import type { FileParseOptions } from '@/lib/file-parsers/types' + +/** + * Structured text extraction for PresentationML (`.pptx`/`.pptm`/`.potx`) that + * walks the slide XML directly instead of flattening every `` in the + * package. Slides are visited in the order the deck displays them; each shape + * tree is read in document order, recursing into group shapes; placeholder + * shapes that only carry layout boilerplate (slide number, date, header, + * footer) are skipped; tables are rendered row by row; SmartArt and chart text + * are read from their own parts; and presenter notes contribute only their + * body placeholder, which is how python-pptx, MarkItDown, and Docling read them. + * + * Only the matched XML parts are inflated — media entries are never touched. + */ + +const SLIDE_PART = /^ppt\/slides\/slide(\d+)\.xml$/ +const SLIDE_PART_ANY = /^ppt\/slides\/[^/]+\.xml$/ +const PRESENTATION_PART = 'ppt/presentation.xml' +const PRESENTATION_RELS_PART = 'ppt/_rels/presentation.xml.rels' +const NOTES_RELATIONSHIP_SUFFIX = '/notesSlide' +const DIAGRAM_DATA_RELATIONSHIP_SUFFIX = '/diagramData' +const NOTES_PART_PREFIX = 'ppt/notesSlides/' +const PACKAGE_PREFIX = 'ppt/' +const DIAGRAM_GRAPHIC_URI_SUFFIX = '/diagram' +const CHART_GRAPHIC_URI_SUFFIX = '/chart' + +/** Opens and closes the modest chart summary (title, axis titles, series, categories). */ +const CHART_OPEN = '[Chart]' +const CHART_CLOSE = '[/Chart]' + +type Relationships = Map + +interface SlideContext { + zip: JSZip + /** The slide part's own relationships, for diagram, chart, and notes targets. */ + rels: Relationships + /** Directory the slide's relationship targets resolve against. */ + baseDir: string +} + +/** + * The slide-number field's cached text is the layout's, not the author's. Date + * fields keep their text: outside a `dt` placeholder (already skipped) a deck's + * dates are content, and python-pptx keeps them too. + */ +const SLIDE_NUMBER_FIELD_TYPE = 'slidenum' + +/** Layout-chrome placeholders whose text is a field, not slide content. */ +const SKIPPED_PLACEHOLDER_TYPES = new Set(['sldNum', 'dt', 'ftr', 'hdr']) + +const TITLE_PLACEHOLDER_TYPES = new Set(['title', 'ctrTitle']) + +function placeholderType(shape: XmlElement): string | null { + const nonVisual = childElements(shape).find((child) => child.name === 'p:nvSpPr') + if (!nonVisual) return null + const placeholder = findFirst(nonVisual, 'p:ph') + if (!placeholder) return null + return placeholder.attribs.type ?? 'body' +} + +/** + * Concatenates a DrawingML paragraph's runs, turning `` into a newline + * and skipping slide-number fields wherever they appear. + */ +function paragraphText(paragraph: XmlElement): string { + const pieces: string[] = [] + const visit = (element: XmlElement): void => { + if (element.name === 'a:br') { + pieces.push('\n') + return + } + if (element.name === 'a:fld' && element.attribs.type === SLIDE_NUMBER_FIELD_TYPE) { + return + } + if (element.name === 'a:t') { + for (const child of element.children) { + if (child.type === 'text') pieces.push(child.data) + } + return + } + for (const child of element.children) { + if (isXmlElement(child)) visit(child) + } + } + visit(paragraph) + return trimLineEnds(pieces.join('')).trim() +} + +/** One line per `` in a text body, skipping empty paragraphs. */ +function textBodyLines(container: XmlElement): string[] { + const lines: string[] = [] + for (const paragraph of findAll(container, 'a:p')) { + const text = paragraphText(paragraph) + if (text) lines.push(text) + } + return lines +} + +function shapeBlocks(shape: XmlElement): string[] { + const type = placeholderType(shape) + if (type && SKIPPED_PLACEHOLDER_TYPES.has(type)) return [] + + const textBody = childElements(shape).find((child) => child.name === 'p:txBody') + if (!textBody) return [] + + const lines = textBodyLines(textBody) + if (lines.length === 0) return [] + + if (type && TITLE_PLACEHOLDER_TYPES.has(type)) { + return [lines.join(' '), ''] + } + return [lines.join('\n')] +} + +function tableBlocks(table: XmlElement): string[] { + const rows: string[] = [] + for (const row of findAll(table, 'a:tr')) { + const cells = childElements(row) + .filter((cell) => cell.name === 'a:tc') + .map((cell) => textBodyLines(cell).join(' ')) + if (cells.some((cell) => cell.length > 0)) rows.push(formatTableRow(cells)) + } + return rows.length > 0 ? [TABLE_OPEN, ...rows, TABLE_CLOSE] : [] +} + +/** Every `` run under a node, for chart titles and axis titles. */ +function runText(node: XmlElement): string { + return findAll(node, 'a:t') + .map((run) => run.children.map((child) => (child.type === 'text' ? child.data : '')).join('')) + .join('') + .replace(/\s+/g, ' ') + .trim() +} + +/** Cached cell values (`c:pt/c:v`) of a chart reference, in index order. */ +function cachedValues(node: XmlElement): string[] { + const values: string[] = [] + for (const point of findAll(node, 'c:pt')) { + const value = findFirst(point, 'c:v') + const text = value ? runText(value) || textContent(value) : '' + if (text) values.push(text) + } + return values +} + +function textContent(node: XmlElement): string { + return node.children + .map((child) => (child.type === 'text' ? child.data : '')) + .join('') + .trim() +} + +/** + * SmartArt keeps its text in the diagram data part: one `dgm:pt` per node, each + * with its own text body, in document order. + */ +function diagramLines(dataXml: string): string[] { + const lines: string[] = [] + for (const point of findAll(parseXml(dataXml), 'dgm:pt')) { + const text = textBodyLines(point).join(' ').trim() + if (text) lines.push(text) + } + return lines +} + +/** + * A modest chart summary: the chart title, axis titles, one line per series + * name, and one line per category (taken from the first series that has any). + */ +function chartLines(chartXml: string): string[] { + const chart = findFirst(parseXml(chartXml), 'c:chart') + if (!chart) return [] + + const lines: string[] = [] + const title = childElements(chart).find((child) => child.name === 'c:title') + const titleText = title ? runText(title) : '' + if (titleText) lines.push(titleText) + + for (const axis of findAll(chart, 'c:catAx').concat(findAll(chart, 'c:valAx'))) { + const axisTitle = childElements(axis).find((child) => child.name === 'c:title') + const axisText = axisTitle ? runText(axisTitle) : '' + if (axisText) lines.push(axisText) + } + + let categories: string[] = [] + for (const series of findAll(chart, 'c:ser')) { + const name = childElements(series).find((child) => child.name === 'c:tx') + const nameText = name ? cachedValues(name).join(' ') || runText(name) : '' + if (nameText) lines.push(nameText) + if (categories.length === 0) { + const category = childElements(series).find((child) => child.name === 'c:cat') + if (category) categories = cachedValues(category) + } + } + lines.push(...categories) + + return lines.length > 0 ? [CHART_OPEN, ...lines, CHART_CLOSE] : [] +} + +/** + * Resolves a relationship target — relative to a directory, or package-absolute + * when it starts with `/` — and clamps it inside `ppt/`, so a crafted `.rels` + * cannot point the walker at an arbitrary entry. + */ +function resolvePackagePath(baseDir: string, target: string): string | null { + const absolute = target.startsWith('/') + const segments = absolute ? [] : baseDir.split('/').filter(Boolean) + for (const part of (absolute ? target.slice(1) : target).split('/')) { + if (part === '..') { + if (segments.length === 0) return null + segments.pop() + } else if (part && part !== '.') { + segments.push(part) + } + } + const path = segments.join('/') + return path.startsWith(PACKAGE_PREFIX) && path.endsWith('.xml') ? path : null +} + +function parseRelationships(relsXml: string | null): Relationships { + const rels: Relationships = new Map() + if (relsXml === null) return rels + for (const relationship of findAll(parseXml(relsXml), 'Relationship')) { + const { Id: id, Type: type, Target: target } = relationship.attribs + if (id && target) rels.set(id, { type: type ?? '', target }) + } + return rels +} + +function relationshipTarget(context: SlideContext, id: string | undefined): string | null { + const relationship = id ? context.rels.get(id) : undefined + return relationship ? resolvePackagePath(context.baseDir, relationship.target) : null +} + +function relationshipTargetByType(context: SlideContext, typeSuffix: string): string | null { + for (const relationship of context.rels.values()) { + if (relationship.type.endsWith(typeSuffix)) { + return resolvePackagePath(context.baseDir, relationship.target) + } + } + return null +} + +/** The `r:dm` data-model relationship, falling back to the slide's only diagram-data part. */ +function diagramDataPath(context: SlideContext, graphicData: XmlElement): string | null { + const relIds = findFirst(graphicData, 'dgm:relIds') + return ( + relationshipTarget(context, relIds?.attribs['r:dm']) ?? + relationshipTargetByType(context, DIAGRAM_DATA_RELATIONSHIP_SUFFIX) + ) +} + +/** Tables inline; SmartArt and charts live in their own parts, reached through the slide rels. */ +async function graphicFrameBlocks(context: SlideContext, frame: XmlElement): Promise { + const table = findFirst(frame, 'a:tbl') + if (table) return tableBlocks(table) + + const graphicData = findFirst(frame, 'a:graphicData') + if (!graphicData) return [] + const uri = graphicData.attribs.uri ?? '' + + if (uri.endsWith(DIAGRAM_GRAPHIC_URI_SUFFIX)) { + const path = diagramDataPath(context, graphicData) + const xml = path ? await readXmlPart(context.zip, path) : null + return xml ? diagramLines(xml) : [] + } + + if (uri.endsWith(CHART_GRAPHIC_URI_SUFFIX)) { + const chart = findFirst(graphicData, 'c:chart') + const path = relationshipTarget(context, chart?.attribs['r:id']) + const xml = path ? await readXmlPart(context.zip, path) : null + return xml ? chartLines(xml) : [] + } + + return [] +} + +/** A connector can carry a text body; it reads like any other shape's text. */ +function connectorBlocks(connector: XmlElement): string[] { + const textBody = childElements(connector).find((child) => child.name === 'p:txBody') + if (!textBody) return [] + const lines = textBodyLines(textBody) + return lines.length > 0 ? [lines.join('\n')] : [] +} + +/** A picture contributes its alternative text, as the HTML walker does for ``. */ +function pictureBlocks(picture: XmlElement): string[] { + const nonVisual = childElements(picture).find((child) => child.name === 'p:nvPicPr') + const properties = nonVisual ? findFirst(nonVisual, 'p:cNvPr') : null + const image = imageAltText(properties?.attribs.descr) + return image ? [image] : [] +} + +/** + * Markup-compatibility wrapper: the `mc:Fallback` branch is what every + * consumer renders, so it is preferred; otherwise the first `mc:Choice`. + */ +function alternateContentBranch(element: XmlElement): XmlElement | null { + const children = childElements(element) + return ( + children.find((child) => child.name === 'mc:Fallback') ?? + children.find((child) => child.name === 'mc:Choice') ?? + null + ) +} + +/** Walks a shape tree (or group) in document order. */ +async function shapeTreeBlocks(context: SlideContext, tree: XmlElement): Promise { + const blocks: string[] = [] + for (const child of childElements(tree)) { + switch (child.name) { + case 'p:sp': + blocks.push(...shapeBlocks(child)) + break + case 'p:cxnSp': + blocks.push(...connectorBlocks(child)) + break + case 'p:grpSp': + blocks.push(...(await shapeTreeBlocks(context, child))) + break + case 'p:graphicFrame': + blocks.push(...(await graphicFrameBlocks(context, child))) + break + case 'p:pic': + blocks.push(...pictureBlocks(child)) + break + case 'mc:AlternateContent': { + const branch = alternateContentBranch(child) + if (branch) blocks.push(...(await shapeTreeBlocks(context, branch))) + break + } + default: + break + } + } + return blocks +} + +async function slideBodyBlocks(context: SlideContext, slideXml: string): Promise { + const document = parseXml(slideXml) + const tree = findFirst(document, 'p:spTree') + return tree ? shapeTreeBlocks(context, tree) : [] +} + +/** Only the `body` placeholder of a notes page carries the presenter's notes. */ +function notesBodyLines(notesXml: string): string[] { + const document = parseXml(notesXml) + const tree = findFirst(document, 'p:spTree') + if (!tree) return [] + + const lines: string[] = [] + for (const shape of findAll(tree, 'p:sp')) { + if (placeholderType(shape) !== 'body') continue + const textBody = childElements(shape).find((child) => child.name === 'p:txBody') + if (textBody) lines.push(...textBodyLines(textBody)) + } + return lines +} + +/** The slide's notes part, accepted only under `ppt/notesSlides/`. */ +function notesPartPath(context: SlideContext): string | null { + const path = relationshipTargetByType(context, NOTES_RELATIONSHIP_SUFFIX) + return path?.startsWith(NOTES_PART_PREFIX) ? path : null +} + +/** Physical part order — the fallback when the presentation part cannot say. */ +function slidePartsByNumber(zip: JSZip): string[] { + const slides: Array<{ index: number; path: string }> = [] + for (const path of Object.keys(zip.files)) { + const match = SLIDE_PART.exec(path) + if (match) slides.push({ index: Number(match[1]), path }) + } + return slides.sort((a, b) => a.index - b.index).map((slide) => slide.path) +} + +/** + * The order the deck displays: `p:sldIdLst` in `ppt/presentation.xml`, each id + * resolved through the presentation's relationships. Reordering slides in + * PowerPoint changes this list, not the part names. Ids whose target is missing + * are skipped; when nothing resolves, the physical order is used instead. + */ +async function slidePartsInOrder(zip: JSZip): Promise { + const presentationXml = await readXmlPart(zip, PRESENTATION_PART) + const relsXml = await readXmlPart(zip, PRESENTATION_RELS_PART) + if (presentationXml === null || relsXml === null) return slidePartsByNumber(zip) + + const rels = parseRelationships(relsXml) + const ordered: string[] = [] + const seen = new Set() + for (const slideId of findAll(parseXml(presentationXml), 'p:sldId')) { + const relationship = rels.get(slideId.attribs['r:id'] ?? '') + const path = relationship ? resolvePackagePath(PACKAGE_PREFIX, relationship.target) : null + if (!path || !SLIDE_PART_ANY.test(path) || seen.has(path) || !zip.file(path)) continue + seen.add(path) + ordered.push(path) + } + return ordered.length > 0 ? ordered : slidePartsByNumber(zip) +} + +function slideRelsPath(slidePath: string): string { + const slash = slidePath.lastIndexOf('/') + return `${slidePath.slice(0, slash)}/_rels/${slidePath.slice(slash + 1)}.rels` +} + +/** + * Extracts structured text from a PresentationML package. Slides are separated + * by a blank line; presenter notes follow their slide under a `[Notes]` marker. + * The caller must already have applied the archive size guard; each XML part is + * additionally bounded by {@link readXmlPart}. + */ +export async function extractPresentationText( + buffer: Buffer, + options: FileParseOptions = {} +): Promise { + const zip = await JSZip.loadAsync(buffer) + options.signal?.throwIfAborted() + + const slideBlocks: string[] = [] + for (const slidePath of await slidePartsInOrder(zip)) { + const slideXml = await readXmlPart(zip, slidePath) + options.signal?.throwIfAborted() + if (slideXml === null) continue + + const context: SlideContext = { + zip, + rels: parseRelationships(await readXmlPart(zip, slideRelsPath(slidePath))), + baseDir: slidePath.slice(0, slidePath.lastIndexOf('/')), + } + const blocks = await slideBodyBlocks(context, slideXml) + options.signal?.throwIfAborted() + + const notesPath = notesPartPath(context) + const notesXml = notesPath ? await readXmlPart(zip, notesPath) : null + options.signal?.throwIfAborted() + if (notesXml) { + const notes = notesBodyLines(notesXml) + if (notes.length > 0) blocks.push(NOTES_MARKER, ...notes) + } + + if (blocks.length > 0) slideBlocks.push(joinBlocks(blocks), '') + } + + return assertTextWithinLimit(joinBlocks(slideBlocks)) +} diff --git a/apps/sim/lib/file-parsers/opendocument-parser.ts b/apps/sim/lib/file-parsers/opendocument-parser.ts index 1a33e8a72d7..9d7852f2b1c 100644 --- a/apps/sim/lib/file-parsers/opendocument-parser.ts +++ b/apps/sim/lib/file-parsers/opendocument-parser.ts @@ -1,7 +1,13 @@ import { existsSync } from 'fs' import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' -import { FileParserError, isEncryptedOfficeParserError } from '@/lib/file-parsers/errors' +import { getErrorMessage } from '@sim/utils/errors' +import { + FileParserError, + isEncryptedOfficeParserError, + isFileParserError, +} from '@/lib/file-parsers/errors' +import { extractOpenDocumentText } from '@/lib/file-parsers/odf-text' import { parseOfficeText } from '@/lib/file-parsers/officeparser-module' import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types' import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' @@ -14,11 +20,13 @@ const logger = createLogger('OpenDocumentParser') * the formats LibreOffice, OpenOffice, and Google Docs exports produce, which * turn up in document libraries alongside their Microsoft equivalents. * - * `officeparser` handles the OpenDocument container natively. Unlike the legacy - * `.doc`/`.ppt` parsers this deliberately has **no** best-effort fallback: an - * OpenDocument file is a ZIP whose text lives in `content.xml`, so a failure here - * means the archive is unreadable or has no text, and scraping the raw bytes would - * only produce XML markup. Throwing lets the caller record a real failure. + * The primary path walks `content.xml` directly so tables keep their rows, + * lists keep their markers, and reviewer annotations and tracked deletions are + * dropped instead of being spliced into the body. `officeparser` remains the + * fallback for an archive the walker cannot read, and is what classifies + * encrypted packages. Unlike the legacy `.doc`/`.ppt` parsers this deliberately + * has **no** best-effort byte scrape: a failure means the archive is unreadable + * or has no text, and throwing lets the caller record a real failure. * * Spreadsheets (`.ods`) go to `XlsxParser` instead, which SheetJS reads natively * and renders with per-sheet structure rather than one flat text run. @@ -45,25 +53,38 @@ export class OpenDocumentParser implements FileParser { /** * The container is a ZIP, so the decompression-bomb guard applies exactly as - * it does for OOXML — and it must run before officeparser inflates anything. + * it does for OOXML — and it must run before anything inflates an entry. */ assertOoxmlArchiveWithinLimits(buffer) - let extracted: string + let extracted = '' + let extractionMethod = 'odf-walker' try { - const result = await parseOfficeText(buffer, options) - extracted = typeof result === 'string' ? result : '' - } catch (error) { + extracted = await extractOpenDocumentText(buffer, options) + } catch (walkerError) { options.signal?.throwIfAborted() - logger.error('OpenDocument parsing failed', { error: (error as Error).message }) - if (isEncryptedOfficeParserError(error)) { - throw new FileParserError( - 'encrypted_file', - 'This OpenDocument file is encrypted or password-protected', - error - ) + if (isFileParserError(walkerError) && walkerError.code === 'complexity_limit') { + throw walkerError + } + logger.warn('OpenDocument walker failed, trying officeparser', { + error: getErrorMessage(walkerError), + }) + extractionMethod = 'officeparser' + try { + const result = await parseOfficeText(buffer, options) + extracted = typeof result === 'string' ? result : '' + } catch (error) { + options.signal?.throwIfAborted() + logger.error('OpenDocument parsing failed', { error: getErrorMessage(error) }) + if (isEncryptedOfficeParserError(error)) { + throw new FileParserError( + 'encrypted_file', + 'This OpenDocument file is encrypted or password-protected', + error + ) + } + throw new FileParserError('invalid_format', 'Failed to parse OpenDocument file', error) } - throw new FileParserError('invalid_format', 'Failed to parse OpenDocument file', error) } const content = sanitizeTextForUTF8(extracted.trim()) @@ -78,7 +99,7 @@ export class OpenDocumentParser implements FileParser { content, metadata: { characterCount: content.length, - extractionMethod: 'officeparser', + extractionMethod, }, } } diff --git a/apps/sim/lib/file-parsers/parser-formats.test.ts b/apps/sim/lib/file-parsers/parser-formats.test.ts index 0864cbc736d..cfb6906aa1f 100644 --- a/apps/sim/lib/file-parsers/parser-formats.test.ts +++ b/apps/sim/lib/file-parsers/parser-formats.test.ts @@ -2,11 +2,11 @@ * @vitest-environment node * * Pins the `degraded` metadata contract to the parsers' real behaviour, using - * genuine OOXML archives rather than mocks. `DocParser` and `PptxParser` never - * throw by design — on a legacy OLE binary or a deck with no text they return a - * placeholder sentence or scraped ZIP internals. Automated callers rely on - * `degraded` to tell that apart from a real extraction, so if a parser stops - * setting the flag these tests are what catches it. + * genuine OOXML archives rather than mocks. `DocParser` never throws by design — + * on a legacy OLE binary it returns a placeholder sentence or scraped bytes, and + * automated callers rely on `degraded` to tell that apart from a real + * extraction. `PptxParser` instead rejects with a typed error for a legacy + * binary or a text-free deck, so nothing scraped ever reaches the index. */ import JSZip from 'jszip' import { describe, expect, it } from 'vitest' @@ -129,36 +129,49 @@ describe('PptxParser degraded reporting', () => { const result = await new PptxParser().parseBuffer(buffer) expect(result.content).toContain('Quarterly Market Data Review') + expect(result.metadata?.extractionMethod).toBe('ooxml-walker') expect(result.metadata?.degraded).toBeFalsy() }) /** - * A deck of images has no text for officeparser to return, and the fallback - * then scrapes the archive — the observed output begins `[Content_Types].xml`. - * Indexing that would put ZIP internals into the vector store. + * A deck of images has no slide text. The old byte-scrape fallback returned + * the archive's own file names (`[Content_Types].xml`) as content; a typed + * rejection keeps ZIP internals out of the vector store. */ - it('flags a deck with no extractable text as degraded', async () => { + it('reports a deck with no extractable text as a typed failure', async () => { const buffer = await buildPptx('') - const result = await new PptxParser().parseBuffer(buffer) + const error = await new PptxParser().parseBuffer(buffer).catch((caught: unknown) => caught) - expect(result.metadata?.degraded).toBe(true) + expect(error).toBeInstanceOf(FileParserError) + expect(error).toMatchObject({ code: 'no_extractable_text' }) }) - it('flags a legacy OLE .ppt binary as degraded', async () => { - const result = await new PptxParser().parseBuffer(buildLegacyOleBinary()) + /** No pure-JS extractor reads PowerPoint 97 binaries, so the parser says so. */ + it('rejects a legacy OLE .ppt binary as unsupported', async () => { + const error = await new PptxParser() + .parseBuffer(buildLegacyOleBinary()) + .catch((caught: unknown) => caught) - expect(result.metadata?.degraded).toBe(true) - expect(result.content).toContain('Unable to extract text') + expect(error).toBeInstanceOf(FileParserError) + expect(error).toMatchObject({ code: 'unsupported_type' }) + expect((error as FileParserError).message).toContain('.pptx') }) }) describe('DocParser degraded reporting', () => { - it('flags a legacy OLE .doc binary as degraded', async () => { - const result = await new DocParser().parseBuffer(buildLegacyOleBinary()) + /** + * An OLE2 header with no valid compound-file structure behind it used to fall + * through to the byte scrape and come back as degraded placeholder prose. It is + * now a typed rejection, so nothing downstream can index the placeholder. + */ + it('rejects an OLE .doc binary that word-extractor cannot read as invalid_format', async () => { + const error = await new DocParser() + .parseBuffer(buildLegacyOleBinary()) + .catch((caught: unknown) => caught) - expect(result.metadata?.degraded).toBe(true) - expect(result.content).toContain('Unable to extract text') + expect(error).toBeInstanceOf(FileParserError) + expect(error).toMatchObject({ code: 'invalid_format' }) }) /** @@ -182,6 +195,8 @@ describe('DocxParser', () => { const result = await new DocxParser().parseBuffer(buffer) expect(result.content).toContain('Market Data SOP body text') + expect(result.metadata?.extractionMethod).toBe('mammoth-html') + expect(result.metadata?.html).toBeUndefined() expect(result.metadata?.degraded).toBeFalsy() }) @@ -274,6 +289,7 @@ describe('OpenDocumentParser', () => { const result = await new OpenDocumentParser().parseBuffer(buffer) expect(result.content).toContain('OpenDocument paragraph') + expect(result.metadata?.extractionMethod).toBe('odf-walker') expect(result.metadata?.degraded).toBeFalsy() }) diff --git a/apps/sim/lib/file-parsers/pdf-furniture.test.ts b/apps/sim/lib/file-parsers/pdf-furniture.test.ts new file mode 100644 index 00000000000..604b4681e42 --- /dev/null +++ b/apps/sim/lib/file-parsers/pdf-furniture.test.ts @@ -0,0 +1,299 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + furnitureThreshold, + isPageNumber, + normalizeFurnitureText, + type PdfPageLines, + suppressFurniture, +} from '@/lib/file-parsers/pdf-furniture' +import type { PdfLine } from '@/lib/file-parsers/pdf-lines' + +const PAGE_HEIGHT = 792 +const PITCH = 12 + +function line(text: string, y: number, height = 11): PdfLine { + return { text, y, height } +} + +/** Body lines at a regular pitch running down from `top`. */ +function body(count: number, top: number, label = 'Body'): PdfLine[] { + return Array.from({ length: count }, (_, i) => line(`${label} line ${i + 1}`, top - i * PITCH)) +} + +/** A page with a top header, a bottom footer, and a body separated from both by a margin gap. */ +function page(index: number, options: { header?: string; footer?: string } = {}): PdfPageLines { + const lines: PdfLine[] = [] + if (options.header) lines.push(line(options.header, 729)) + lines.push(...body(40, 660, `Page ${index}`)) + if (options.footer) lines.push(line(options.footer, 55)) + return { lines, pageHeight: PAGE_HEIGHT } +} + +function texts(pages: PdfLine[][]): string[][] { + return pages.map((lines) => lines.map((entry) => entry.text)) +} + +describe('suppressFurniture', () => { + it('drops a header repeated on enough pages but keeps its first occurrence', () => { + const pages = [1, 2, 3, 4].map((i) => page(i, { header: 'ACME Corp — Internal Use Only' })) + + const result = texts(suppressFurniture(pages)) + + expect(result[0]).toContain('ACME Corp — Internal Use Only') + for (const remaining of result.slice(1)) { + expect(remaining).not.toContain('ACME Corp — Internal Use Only') + expect(remaining).toHaveLength(40) + } + }) + + it('never applies the frequency rule to a single page', () => { + const result = texts(suppressFurniture([page(1, { header: 'Draft' })])) + + expect(result[0]).toContain('Draft') + }) + + it('treats two matching pages as furniture when the document has exactly two pages', () => { + const pages = [1, 2].map((i) => + page(i, { footer: `Confidential draft, do not distribute — Page ${i} of 2` }) + ) + + const result = texts(suppressFurniture(pages)) + + expect(result[0]).toContain('Confidential draft, do not distribute — Page 1 of 2') + expect(result[1]).not.toContain('Confidential draft, do not distribute — Page 2 of 2') + }) + + it('requires the fraction threshold on longer documents without a streak', () => { + const pages = Array.from({ length: 10 }, (_, i) => + page(i, { footer: i % 4 === 0 ? 'Sporadic note' : undefined }) + ) + + const result = texts(suppressFurniture(pages)) + + expect(result.flat().filter((text) => text === 'Sporadic note')).toHaveLength(3) + }) + + it('detects a footer by a three-page streak even when its title changes per chapter', () => { + const pages = Array.from({ length: 12 }, (_, i) => { + const chapter = i < 6 ? 'Chapter 1 Filing Information' : 'Chapter 2 Filing Status' + const pageNumber = i + 6 + const footer = + i % 2 === 0 + ? `${pageNumber} ${chapter} Publication 17 (2025)` + : `Publication 17 (2025) ${chapter} ${pageNumber}` + return page(i, { footer }) + }) + + const result = texts(suppressFurniture(pages)) + const footers = result.flat().filter((text) => text.includes('Publication 17')) + + expect(footers).toEqual([ + '6 Chapter 1 Filing Information Publication 17 (2025)', + '12 Chapter 2 Filing Status Publication 17 (2025)', + ]) + }) + + it('drops page numbers in the bands regardless of repetition', () => { + const pages: PdfPageLines[] = [ + { lines: [line('Page 1 of 3', 55), ...body(3, 600)], pageHeight: PAGE_HEIGHT }, + { lines: [line('2', 55), ...body(3, 600), line('ii', 729)], pageHeight: PAGE_HEIGHT }, + { lines: [line('- 3 -', 55), ...body(3, 600), line('925', 60)], pageHeight: PAGE_HEIGHT }, + ] + + const result = texts(suppressFurniture(pages)) + + expect(result[0]).toEqual(['Body line 1', 'Body line 2', 'Body line 3']) + expect(result[1]).toEqual(['Body line 1', 'Body line 2', 'Body line 3']) + expect(result[2]).toEqual(['Body line 1', 'Body line 2', 'Body line 3', '925']) + }) + + it('drops a folio outside the band only when a margin gap separates it from the text', () => { + const pages = [1, 2].map((i) => ({ + lines: [...body(20, 700), line(`${i}`, 100)], + pageHeight: PAGE_HEIGHT, + })) + const tableCells = [1, 2].map(() => ({ + lines: [...body(50, 700), line('1', 700 - 50 * PITCH)], + pageHeight: PAGE_HEIGHT, + })) + + for (const kept of texts(suppressFurniture(pages))) expect(kept).toHaveLength(20) + for (const kept of texts(suppressFurniture(tableCells))) expect(kept).toContain('1') + }) + + it('ignores band text longer than the furniture cap, such as a repeated table header', () => { + const header = `SKU Product Unit price Lead time Notes ${'Column '.repeat(14)}`.trim() + expect(header.length).toBeGreaterThan(120) + const pages = [1, 2, 3, 4].map((i) => page(i, { header })) + + const result = texts(suppressFurniture(pages)) + + for (const kept of result) expect(kept).toContain(header) + }) + + it('merges same-baseline fragments into one key before matching', () => { + const pages = [1, 2, 3, 4].map((i) => ({ + lines: [ + ...body(3, 600), + line(`${i}`, 31.3), + line('Chapter 1', 31.3), + line('Publication 17 (2025)', 32.5), + ], + pageHeight: PAGE_HEIGHT, + })) + + const result = texts(suppressFurniture(pages)) + + expect(result[0].slice(3)).toEqual(['1', 'Chapter 1', 'Publication 17 (2025)']) + expect(result[3]).toHaveLength(3) + }) + + it('keeps a table header that repeats at the top of every page while dropping the running header above it', () => { + const pages = Array.from({ length: 5 }, () => { + const rows = Array.from({ length: 50 }, (_, i) => + line(`${1000 + i} ${2000 + i} ${3000 + i}`, 740 - (i + 1) * 8, 7.5) + ) + return { + lines: [ + line('2025 Tax Table — Continued', 772, 10), + line('Single Married filing jointly Head of household', 740, 7.5), + ...rows, + line('Need more information? Visit IRS.gov.', 31, 10), + ], + pageHeight: PAGE_HEIGHT, + } + }) + + const result = texts(suppressFurniture(pages)) + + for (const kept of result) { + expect(kept).toContain('Single Married filing jointly Head of household') + } + expect(result.flat().filter((text) => text === '2025 Tax Table — Continued')).toHaveLength(1) + expect( + result.flat().filter((text) => text === 'Need more information? Visit IRS.gov.') + ).toHaveLength(1) + }) + + it('keeps a footnote that sits directly under the body while dropping the footer below it', () => { + const pages = Array.from({ length: 4 }, () => ({ + lines: [ + ...body(52, 700), + line( + '* This column must also be used by a qualifying surviving spouse.', + 700 - 52 * PITCH, + 8 + ), + line('Visit IRS.gov.', 30, 10), + ], + pageHeight: PAGE_HEIGHT, + })) + + const result = texts(suppressFurniture(pages)) + + for (const kept of result) { + expect(kept).toContain('* This column must also be used by a qualifying surviving spouse.') + } + expect(result.flat().filter((text) => text === 'Visit IRS.gov.')).toHaveLength(1) + }) + + it('keeps a band line whose text also occurs in body positions', () => { + const pages = [1, 2, 3, 4].map((i) => ({ + lines: [ + line('Quarter Revenue Margin', 760), + ...body(40, 660), + ...(i === 1 ? [line('Quarter Revenue Margin', 400)] : []), + ], + pageHeight: PAGE_HEIGHT, + })) + + for (const kept of texts(suppressFurniture(pages))) { + expect(kept).toContain('Quarter Revenue Margin') + } + }) + + it('never drops a band line that would leave a hyphenated word orphaned', () => { + const pages = [1, 2, 3, 4].map(() => ({ + lines: [line('Married filing sepa-', 760), ...body(40, 660)], + pageHeight: PAGE_HEIGHT, + })) + + for (const kept of texts(suppressFurniture(pages))) { + expect(kept).toContain('Married filing sepa-') + } + }) + + it('leaves body text alone even when it repeats', () => { + const pages = [1, 2, 3, 4].map(() => ({ + lines: [line('Repeated body sentence.', 400)], + pageHeight: PAGE_HEIGHT, + })) + + for (const kept of texts(suppressFurniture(pages))) { + expect(kept).toEqual(['Repeated body sentence.']) + } + }) + + it('skips pages without a page height or line geometry', () => { + const pages: PdfPageLines[] = [1, 2, 3].map(() => ({ + lines: [{ text: 'Header', height: 0 }, line('Header', 729)], + })) + + for (const kept of texts(suppressFurniture(pages))) expect(kept).toEqual(['Header', 'Header']) + }) +}) + +describe('normalizeFurnitureText', () => { + it('keys mirrored facing-page footers identically', () => { + expect(normalizeFurnitureText('6 Chapter 1 Filing Information Publication 17 (2025)')).toBe( + normalizeFurnitureText('Publication 17 (2025) Chapter 1 Filing Information 17') + ) + }) + + it('collapses case, digits, and edge punctuation', () => { + expect(normalizeFurnitureText(' Confidential DRAFT — Page 12 of 40. ')).toBe( + '# # confidential draft of page' + ) + }) +}) + +describe('isPageNumber', () => { + it('matches the page-number shapes', () => { + expect(isPageNumber('Page 3', 10)).toBe(true) + expect(isPageNumber('page 3 of 10', 10)).toBe(true) + expect(isPageNumber('3 / 10', 10)).toBe(true) + expect(isPageNumber('7', 10)).toBe(true) + expect(isPageNumber('xiv', 20)).toBe(true) + expect(isPageNumber('CD', 500)).toBe(true) + expect(isPageNumber('— 12 —', 20)).toBe(true) + }) + + it('rejects bare numbers beyond the page count and ordinary text', () => { + expect(isPageNumber('925', 142)).toBe(false) + expect(isPageNumber('2120', 142)).toBe(false) + expect(isPageNumber('Chapter 1', 10)).toBe(false) + expect(isPageNumber('civilian', 10)).toBe(false) + }) + + it('rejects words that merely look like roman numerals', () => { + for (const word of ['mix', 'mild', 'civil', 'vivid', 'mimic', 'dim']) { + expect(isPageNumber(word, 1000)).toBe(false) + } + expect(isPageNumber('CD', 10)).toBe(false) + expect(isPageNumber('xiv', 10)).toBe(false) + expect(isPageNumber('ii', 1)).toBe(false) + }) +}) + +describe('furnitureThreshold', () => { + it('scales with the page count', () => { + expect(furnitureThreshold(1)).toBeUndefined() + expect(furnitureThreshold(2)).toBe(2) + expect(furnitureThreshold(3)).toBe(3) + expect(furnitureThreshold(10)).toBe(5) + expect(furnitureThreshold(142)).toBe(71) + }) +}) diff --git a/apps/sim/lib/file-parsers/pdf-furniture.ts b/apps/sim/lib/file-parsers/pdf-furniture.ts new file mode 100644 index 00000000000..ff65d0b827b --- /dev/null +++ b/apps/sim/lib/file-parsers/pdf-furniture.ts @@ -0,0 +1,393 @@ +/** + * Detects running headers, footers, and page numbers across a document's pages + * and removes every copy but the first, so a footer repeated on 140 pages does + * not land mid-sentence in most retrieval chunks while one copy stays + * searchable. + * + * Only horizontal text takes part: a side-footer printed as rotated text never + * enters a band because `readItemGeometry` drops rotated items, so it survives + * on every page. Known limitation. + */ + +import type { PdfLine } from '@/lib/file-parsers/pdf-lines' + +export interface PdfPageLines { + lines: PdfLine[] + /** Page height in PDF user space; absent when the page could not report it. */ + pageHeight?: number +} + +/** Fraction of the page height at the top and bottom where furniture lives. */ +const FURNITURE_BAND_RATIO = 0.12 + +/** A page's first or last line counts as a folio candidate only this close to the page edge. */ +const EDGE_LINE_RATIO = 0.25 + +/** Furniture is short; longer repeated band text is a table header or real prose. */ +const MAX_FURNITURE_CHARS = 120 + +/** Lines whose baselines differ by at most this many points share one furniture row. */ +const SAME_BASELINE_TOLERANCE = 2 + +/** Minimum repeats for a key to count as furniture on longer documents. */ +const MIN_FURNITURE_REPEATS = 3 + +/** Fraction of pages a key must cover when it never runs on consecutive pages. */ +const FURNITURE_PAGE_FRACTION = 0.5 + +/** Consecutive-page run that marks furniture even when its total count is modest. */ +const MIN_FURNITURE_STREAK = 3 + +/** + * A band line whose nearest interior neighbour lies within this many line + * pitches is part of the text flow — a repeated table header, not furniture + * separated from the body by a margin gap. + */ +const FLOW_PITCH_RATIO = 1.5 + +/** Same flow test expressed against the taller of the two lines, for dense tables whose pitch is tiny. */ +const FLOW_HEIGHT_RATIO = 2 + +/** Baseline steps needed before a page's median step is trusted as its line pitch. */ +const MIN_PITCH_STEPS = 3 + +const PAGE_NUMBER_PATTERNS = [ + /^page\s*\d{1,4}(\s*(of|\/)\s*\d{1,4})?$/i, + /^\d{1,4}\s*(of|\/)\s*\d{1,4}$/i, + /^[-–—]\s*\d+\s*[-–—]$/, +] as const + +/** A bare number is a page number only when the document could have that many pages. */ +const BARE_NUMBER = /^\d{1,4}$/ + +/** Strictly formed roman numeral; `mix`, `civil`, or `vivid` never match. */ +const ROMAN_NUMERAL = /^m{0,3}(cm|cd|d?c{0,3})(xc|xl|l?x{0,3})(ix|iv|v?i{0,3})$/i + +const ROMAN_VALUES: Record = { i: 1, v: 5, x: 10, l: 50, c: 100, d: 500, m: 1000 } + +const TRAILING_HYPHEN = /\p{L}-$/u + +/** Only the tail of a line is tested for a hyphen, keeping the check linear on long lines. */ +const HYPHEN_TAIL_CHARS = 2 +const LEADING_LOWERCASE = /^\p{Ll}/u + +const EDGE_PUNCTUATION = /^[\p{P}\p{S}]+|[\p{P}\p{S}]+$/gu + +type Band = 'top' | 'bottom' + +interface BandLine { + band: Band + y: number +} + +interface BandGroup { + band: Band + /** Indices into the page's `lines`. */ + indices: number[] + text: string + y: number + height: number +} + +/** One distinct baseline of a page, in geometric order. */ +interface Baseline { + y: number + height: number + band: Band | undefined +} + +interface KeyOccurrences { + pages: number[] + groups: Array<{ page: number; group: BandGroup }> +} + +/** Returns each page's lines with repeated furniture and page numbers removed. */ +export function suppressFurniture(pages: readonly PdfPageLines[]): PdfLine[][] { + const drops = pages.map(() => new Set()) + const occurrences = new Map() + + const interiorKeys = collectInteriorKeys(pages) + + pages.forEach((page, pageIndex) => { + const seen = new Set() + const baselines = pageBaselines(page) + const pitch = baselinePitch(baselines) + for (const index of edgeLineIndices(page)) { + if (sharesBaseline(page.lines, index)) continue + const line = page.lines[index] + if (!isPageNumber(line.text, pages.length)) continue + const y = line.y as number + const half: Band = y > (page.pageHeight as number) / 2 ? 'top' : 'bottom' + const folio = { band: half, indices: [index], text: line.text, y, height: line.height } + if (isInTextFlow(folio, baselines, pitch)) continue + drops[pageIndex].add(index) + } + for (const group of bandGroups(page)) { + if (isPageNumber(group.text, pages.length)) { + for (const index of group.indices) drops[pageIndex].add(index) + continue + } + if (group.text.length > MAX_FURNITURE_CHARS) continue + const normalized = normalizeFurnitureText(group.text) + if (normalized.length === 0) continue + if (interiorKeys.has(normalized)) continue + if (isHyphenOrphan(page.lines, group)) continue + if (isInTextFlow(group, baselines, pitch)) continue + const key = `${group.band}|${normalized}` + const entry = occurrences.get(key) ?? { pages: [], groups: [] } + if (!seen.has(key)) { + seen.add(key) + entry.pages.push(pageIndex) + } + entry.groups.push({ page: pageIndex, group }) + occurrences.set(key, entry) + } + }) + + const threshold = furnitureThreshold(pages.length) + for (const entry of occurrences.values()) { + const count = entry.pages.length + const byFrequency = threshold !== undefined && count >= threshold + const byStreak = + count >= MIN_FURNITURE_REPEATS && longestRun(entry.pages) >= MIN_FURNITURE_STREAK + if (!byFrequency && !byStreak) continue + const firstPage = entry.pages[0] + for (const { page, group } of entry.groups) { + if (page === firstPage) continue + for (const index of group.indices) drops[page].add(index) + } + } + + return pages.map((page, pageIndex) => + drops[pageIndex].size === 0 + ? page.lines + : page.lines.filter((_, index) => !drops[pageIndex].has(index)) + ) +} + +/** + * Lowercases each word, strips its edge punctuation, maps digit runs to `#`, and + * sorts the words so facing-page footers that mirror their layout (`6 Chapter + * 1 … Publication 17 (2025)` vs `Publication 17 (2025) Chapter 1 … 7`) share + * one key. + */ +export function normalizeFurnitureText(text: string): string { + const words = text + .toLowerCase() + .split(/\s+/) + .map((word) => word.replace(EDGE_PUNCTUATION, '').replace(/\d+/g, '#')) + .filter((word) => word.length > 0) + words.sort() + return words.join(' ') +} + +/** + * Whether a band line is nothing but a page number. A bare number or roman + * numeral qualifies only when it does not exceed `pageCount`, so form and + * publication numbers listed near the page edge survive, and a roman numeral + * additionally needs a multi-page document. + */ +export function isPageNumber(text: string, pageCount: number): boolean { + const compact = text.replace(/\s+/g, ' ').trim() + if (BARE_NUMBER.test(compact)) return Number(compact) <= pageCount + if (ROMAN_NUMERAL.test(compact)) { + const value = romanValue(compact) + return pageCount >= 2 && value >= 1 && value <= pageCount + } + return PAGE_NUMBER_PATTERNS.some((pattern) => pattern.test(compact)) +} + +function romanValue(numeral: string): number { + let total = 0 + const letters = numeral.toLowerCase() + for (let i = 0; i < letters.length; i++) { + const value = ROMAN_VALUES[letters[i]] + const next = ROMAN_VALUES[letters[i + 1]] ?? 0 + total += value < next ? -value : value + } + return total +} + +/** Repeat count that marks a key as furniture, or undefined when the document is too short. */ +export function furnitureThreshold(pageCount: number): number | undefined { + if (pageCount < 2) return undefined + if (pageCount === 2) return 2 + return Math.max(MIN_FURNITURE_REPEATS, Math.ceil(FURNITURE_PAGE_FRACTION * pageCount)) +} + +/** + * The first and last non-blank lines of a page when they sit within the outer + * quarter of the page. A folio printed inside a wide margin sits outside the + * band, but it is still the edge of the page's text; a table cell mid-page is + * not. + */ +function edgeLineIndices(page: PdfPageLines): number[] { + const { pageHeight } = page + if (pageHeight === undefined || !(pageHeight > 0)) return [] + const indices: number[] = [] + let first = -1 + let last = -1 + page.lines.forEach((line, index) => { + if (line.text.trim().length === 0) return + if (first === -1) first = index + last = index + }) + const nearEdge = (index: number): boolean => { + const y = page.lines[index].y + return ( + y !== undefined && + (y <= EDGE_LINE_RATIO * pageHeight || y >= (1 - EDGE_LINE_RATIO) * pageHeight) + ) + } + if (first !== -1 && nearEdge(first)) indices.push(first) + if (last !== -1 && last !== first && nearEdge(last)) indices.push(last) + return indices +} + +/** Groups consecutive band lines that share a baseline into one furniture row. */ +function bandGroups(page: PdfPageLines): BandGroup[] { + const { lines, pageHeight } = page + if (pageHeight === undefined || !(pageHeight > 0)) return [] + const groups: BandGroup[] = [] + let current: BandGroup | undefined + + lines.forEach((line, index) => { + const placement = bandOf(line, pageHeight) + if (!placement) { + current = undefined + return + } + const { band, y } = placement + if (current && current.band === band && Math.abs(current.y - y) <= SAME_BASELINE_TOLERANCE) { + current.indices.push(index) + current.text = `${current.text} ${line.text.trim()}` + current.height = Math.max(current.height, line.height) + return + } + current = { band, indices: [index], text: line.text.trim(), y, height: line.height } + groups.push(current) + }) + + return groups +} + +/** + * Keys of every line outside the bands, so a band line that also occurs in + * body positions — a table header on a page where the table starts mid-page — + * is recognised as content rather than furniture. + */ +function collectInteriorKeys(pages: readonly PdfPageLines[]): Set { + const keys = new Set() + for (const page of pages) { + const { pageHeight } = page + if (pageHeight === undefined || !(pageHeight > 0)) continue + for (const line of page.lines) { + if (bandOf(line, pageHeight) !== undefined) continue + if (line.text.length > MAX_FURNITURE_CHARS) continue + const key = normalizeFurnitureText(line.text) + if (key.length > 0) keys.add(key) + } + } + return keys +} + +/** A group ending in a hyphen, or continuing a hyphenated word, must not leave an orphan. */ +function isHyphenOrphan(lines: readonly PdfLine[], group: BandGroup): boolean { + if (TRAILING_HYPHEN.test(group.text.slice(-HYPHEN_TAIL_CHARS))) return true + const previous = lines[group.indices[0] - 1] + return ( + previous !== undefined && + LEADING_LOWERCASE.test(group.text) && + TRAILING_HYPHEN.test(previous.text.trimEnd().slice(-HYPHEN_TAIL_CHARS)) + ) +} + +/** Distinct baselines of a page sorted from top to bottom. */ +function pageBaselines(page: PdfPageLines): Baseline[] { + const { pageHeight } = page + if (pageHeight === undefined || !(pageHeight > 0)) return [] + const sorted = page.lines + .filter((line) => line.y !== undefined) + .map((line) => ({ + y: line.y as number, + height: line.height, + band: bandOf(line, pageHeight)?.band, + })) + .sort((a, b) => b.y - a.y) + const baselines: Baseline[] = [] + for (const entry of sorted) { + const last = baselines[baselines.length - 1] + if (last && Math.abs(last.y - entry.y) <= SAME_BASELINE_TOLERANCE) { + last.height = Math.max(last.height, entry.height) + continue + } + baselines.push({ ...entry }) + } + return baselines +} + +/** + * Lower median of the vertical steps between distinct baselines; 0 when there + * are fewer than `MIN_PITCH_STEPS`, since on a near-empty page a margin gap is + * as likely to be the median as a line pitch. + */ +function baselinePitch(baselines: readonly Baseline[]): number { + const steps: number[] = [] + for (let i = 1; i < baselines.length; i++) steps.push(baselines[i - 1].y - baselines[i].y) + if (steps.length < MIN_PITCH_STEPS) return 0 + steps.sort((a, b) => a - b) + return steps[Math.floor((steps.length - 1) / 2)] +} + +/** + * Whether a group reaches an interior line through steps no larger than a line + * pitch. A running header, footer, or folio is cut off from the body by a + * margin gap; a repeated table header runs straight into its rows, and a table + * cell near the page edge sits at the row pitch. + */ +function isInTextFlow(group: BandGroup, baselines: readonly Baseline[], pitch: number): boolean { + let index = baselines.findIndex((entry) => Math.abs(entry.y - group.y) <= SAME_BASELINE_TOLERANCE) + if (index === -1) return false + const step = group.band === 'top' ? 1 : -1 + while (true) { + const current = baselines[index] + const next = baselines[index + step] + if (!next) return false + const gap = Math.abs(current.y - next.y) + const limit = Math.max( + FLOW_PITCH_RATIO * pitch, + FLOW_HEIGHT_RATIO * Math.max(current.height, next.height) + ) + if (gap > limit) return false + if (next.band === undefined) return true + index += step + } +} + +/** Whether a neighbouring line in reading order shares this line's baseline. */ +function sharesBaseline(lines: readonly PdfLine[], index: number): boolean { + const y = lines[index].y + if (y === undefined) return false + return [lines[index - 1], lines[index + 1]].some( + (neighbour) => + neighbour?.y !== undefined && Math.abs(neighbour.y - y) <= SAME_BASELINE_TOLERANCE + ) +} + +function bandOf(line: PdfLine, pageHeight: number): BandLine | undefined { + if (line.y === undefined) return undefined + if (line.y >= (1 - FURNITURE_BAND_RATIO) * pageHeight) return { band: 'top', y: line.y } + if (line.y <= FURNITURE_BAND_RATIO * pageHeight) return { band: 'bottom', y: line.y } + return undefined +} + +/** Longest run of consecutive page indices in an ascending list. */ +function longestRun(pages: readonly number[]): number { + let best = 0 + let run = 0 + for (let i = 0; i < pages.length; i++) { + run = i > 0 && pages[i] === pages[i - 1] + 1 ? run + 1 : 1 + if (run > best) best = run + } + return best +} diff --git a/apps/sim/lib/file-parsers/pdf-lines.test.ts b/apps/sim/lib/file-parsers/pdf-lines.test.ts new file mode 100644 index 00000000000..0c50a51ebe1 --- /dev/null +++ b/apps/sim/lib/file-parsers/pdf-lines.test.ts @@ -0,0 +1,320 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + collectCompounds, + collectWords, + dominantLineHeight, + headingMarkersViable, + joinLines, + MAX_PDF_LINES, + normalizePdfWhitespace, + type PdfLine, + PdfLineBuilder, + readItemGeometry, +} from '@/lib/file-parsers/pdf-lines' + +const BODY = 11 + +/** Body lines at a 14.4pt pitch starting at the given baseline. */ +function paragraph(texts: string[], top: number, height = BODY): PdfLine[] { + return texts.map((text, index) => ({ text, y: top - index * 14.4, height })) +} + +describe('joinLines', () => { + it('separates lines with \\n and paragraphs with \\n\\n from the baseline pitch', () => { + const lines = [ + ...paragraph(['First paragraph line one', 'first paragraph line two'], 700), + ...paragraph(['Second paragraph line one', 'second paragraph line two'], 700 - 14.4 - 20.4), + ] + + expect(joinLines(lines, { headingMarkers: false })).toBe( + 'First paragraph line one\nfirst paragraph line two\n\nSecond paragraph line one\nsecond paragraph line two' + ) + }) + + it('breaks a paragraph where the line height changes between heading and body', () => { + const lines: PdfLine[] = [ + { text: 'Heading', y: 700, height: 15.4 }, + { text: 'Body line', y: 700 - 15.5, height: BODY }, + ] + + expect(joinLines(lines, { headingMarkers: false })).toBe('Heading\n\nBody line') + }) + + it('joins cells that share a baseline with a space', () => { + const lines: PdfLine[] = [ + { text: 'SKU', y: 600, height: BODY }, + { text: 'HW-1021', y: 600.2, height: BODY }, + { text: 'Next row', y: 600 - 14.4, height: BODY }, + ] + + expect(joinLines(lines, { headingMarkers: false })).toBe('SKU HW-1021\nNext row') + }) + + it('rejoins a wrapped table cell to its row on a short upward return', () => { + const lines: PdfLine[] = [ + { text: 'HW-1000 Rack unit model', y: 669.5, height: BODY }, + { text: 'D0', y: 655.1, height: BODY }, + { text: '$65,918.68 6 weeks', y: 669.5, height: BODY }, + { text: 'HW-1001 Blade unit', y: 635.9, height: BODY }, + { text: 'model E1', y: 621.5, height: BODY }, + { text: '$942,425.74 11 weeks', y: 635.9, height: BODY }, + ] + + expect(joinLines(lines, { headingMarkers: false })).toBe( + 'HW-1000 Rack unit model D0 $65,918.68 6 weeks\n\nHW-1001 Blade unit model E1 $942,425.74 11 weeks' + ) + }) + + it('starts a paragraph when the text returns upward to a new column', () => { + const lines: PdfLine[] = [ + ...paragraph(['Column one ends here.'], 100), + ...paragraph(['Column two starts here.'], 700), + ] + + expect(joinLines(lines, { headingMarkers: false })).toBe( + 'Column one ends here.\n\nColumn two starts here.' + ) + }) + + it('falls back to single line breaks when lines carry no geometry', () => { + const lines: PdfLine[] = [ + { text: 'one', height: 0 }, + { text: 'two', height: 0 }, + ] + + expect(joinLines(lines)).toBe('one\ntwo') + }) + + it('prefixes short oversized lines with a heading marker only when markers are requested', () => { + const lines: PdfLine[] = [ + { text: 'Memo: Office Relocation Timeline', y: 692, height: 15.4 }, + ...paragraph(['Body text follows the title.'], 676.6), + ] + + expect(joinLines(lines, { bodyHeight: BODY, headingMarkers: true })).toBe( + '## Memo: Office Relocation Timeline\n\nBody text follows the title.' + ) + expect(joinLines(lines, { bodyHeight: BODY })).toBe( + 'Memo: Office Relocation Timeline\n\nBody text follows the title.' + ) + }) + + it('does not mark a heading candidate inside a run of more than three same-height lines', () => { + const tall = paragraph( + ['Slide bullet one', 'Slide bullet two', 'Slide bullet three', 'Slide bullet four'], + 700, + 15.4 + ) + const single: PdfLine[] = [ + { text: 'Real heading', y: 700, height: 15.4 }, + ...paragraph(['Body text follows the heading here.'], 684), + ] + + expect(joinLines(tall, { bodyHeight: BODY, headingMarkers: true })).not.toContain('## ') + expect(joinLines(single, { bodyHeight: BODY, headingMarkers: true })).toMatch( + /^## Real heading/ + ) + }) + + it('joins fifty thousand lines in linear time', () => { + const lines: PdfLine[] = Array.from({ length: 50_000 }, (_, i) => ({ + text: i % 2 === 0 ? `line ${i} ends with hyphen-` : `ated continuation ${i}`, + y: 1_000_000 - i * 14.4, + height: BODY, + })) + const started = performance.now() + + const text = joinLines(lines, { words: new Set(['hyphenated']) }) + + expect(performance.now() - started).toBeLessThan(1000) + expect(text).toContain('ends with hyphenated continuation') + }) + + it('keeps a multi-line heading together by scaling the pitch with its height', () => { + const lines: PdfLine[] = [ + { text: 'Do I Have To', y: 735.9, height: 15 }, + { text: 'File a Return?', y: 719.9, height: 15 }, + ...paragraph(['You must file a federal income tax return if you', 'are a citizen'], 701.8, 8), + ...paragraph(['a resident of Puerto Rico'], 701.8 - 2 * 9.5, 8), + ] + + expect(joinLines(lines, { bodyHeight: 8, headingMarkers: false })).toBe( + 'Do I Have To\nFile a Return?\n\nYou must file a federal income tax return if you\nare a citizen\na resident of Puerto Rico' + ) + }) + + describe('dehyphenation', () => { + it('removes a line-end hyphen when the document shows the joined word', () => { + const lines = paragraph(['archived by the Infra-', 'structure team.'], 627.4) + const words = collectWords([{ text: 'The Infrastructure team owns it.', height: BODY }]) + + expect(joinLines(lines, { words, headingMarkers: false })).toBe( + 'archived by the Infrastructure team.' + ) + }) + + it('keeps an unknown line-end hyphen rather than inventing a word', () => { + const lines = paragraph(['we ship high-', 'quality builds'], 627.4) + const words = collectWords(lines) + + expect(joinLines(lines, { words, headingMarkers: false })).toBe('we ship high-quality builds') + expect(joinLines(lines, { headingMarkers: false })).toBe('we ship high-quality builds') + }) + + it('keeps the hyphen when the compound appears intact elsewhere in the document', () => { + const lines = paragraph(['we compare attention-', 'based models with others'], 700) + const compounds = collectCompounds([{ text: 'Attention-based models win.', height: BODY }]) + + expect(joinLines(lines, { compounds, headingMarkers: false })).toBe( + 'we compare attention-based models with others' + ) + }) + + it('keeps the hyphen when the next line starts with a capital or the break is a paragraph', () => { + expect( + joinLines(paragraph(['the English-', 'German pair'], 700), { headingMarkers: false }) + ).toBe('the English-\nGerman pair') + + const acrossParagraphs: PdfLine[] = [ + ...paragraph(['a first line', 'a second line', 'ends with a dash-'], 700), + ...paragraph(['lowercase start'], 700 - 2 * 14.4 - 30), + ] + expect(joinLines(acrossParagraphs, { headingMarkers: false })).toBe( + 'a first line\na second line\nends with a dash-\n\nlowercase start' + ) + }) + + it('always removes a soft hyphen at a line break', () => { + const lines = paragraph(['Infra­', 'Structure'], 700) + + expect(joinLines(lines, { headingMarkers: false })).toBe('InfraStructure') + }) + }) +}) + +describe('PdfLineBuilder', () => { + it('stops splitting lines at the ceiling and keeps the overflow text', () => { + const builder = new PdfLineBuilder() + for (let i = 0; i < MAX_PDF_LINES; i++) { + builder.append('x', { x: 0, y: i, width: 1, height: 1 }) + builder.endLine() + } + builder.append('overflow', { x: 0, y: -1, width: 1, height: 1 }) + builder.endLine() + builder.append('tail', { x: 0, y: -2, width: 1, height: 1 }) + + const lines = builder.finish() + + expect(lines).toHaveLength(MAX_PDF_LINES) + expect(lines[lines.length - 1].text).toBe('x overflow tail') + }) + + it('starts a new line on a baseline change even without hasEOL', () => { + const builder = new PdfLineBuilder() + builder.append('and', { x: 100, y: 700, width: 20, height: 11 }) + const separator = builder.separatorBefore('CAUTION', { x: 100, y: 680, width: 50, height: 11 }) + + expect(separator).toBe('\n') + }) + + it('starts a new cell on a backwards x-move along one baseline', () => { + const builder = new PdfLineBuilder() + builder.append('EOL 2027', { x: 400, y: 700, width: 40, height: 11 }) + + expect(builder.separatorBefore('HW-1021', { x: 120, y: 700, width: 40, height: 11 })).toBe('\n') + }) + + it('inserts a space across a word-sized gap and nothing across a tight one', () => { + const builder = new PdfLineBuilder() + builder.append('Table', { x: 100, y: 700, width: 30, height: 11 }) + + expect(builder.separatorBefore('Caption', { x: 136, y: 700, width: 40, height: 11 })).toBe(' ') + expect(builder.separatorBefore('s', { x: 130.5, y: 700, width: 5, height: 11 })).toBe('') + builder.append(' ', { x: 130, y: 700, width: 4, height: 0 }) + expect(builder.separatorBefore('Caption', { x: 140, y: 700, width: 40, height: 11 })).toBe('') + }) + + it('records the baseline and dominant height of each line and drops blank lines', () => { + const builder = new PdfLineBuilder() + builder.append('•', { x: 90, y: 592.5, width: 3.9, height: 12.6 }) + builder.append(' ', { x: 93.9, y: 592.5, width: 5.5, height: 0 }) + builder.append('Confirm desk allocations', { x: 99.4, y: 592.5, width: 157, height: 11 }) + builder.endLine() + builder.append(' ') + builder.endLine() + + expect(builder.finish()).toEqual([{ text: '• Confirm desk allocations', y: 592.5, height: 11 }]) + }) +}) + +describe('readItemGeometry', () => { + it('returns undefined for missing, rotated, or non-ltr items', () => { + expect(readItemGeometry({ str: 'x' })).toBeUndefined() + expect(readItemGeometry({ str: 'x', transform: [0, 1, -1, 0, 10, 20] })).toBeUndefined() + expect( + readItemGeometry({ str: 'x', transform: [1, 0, 0, 1, 10, 20], dir: 'rtl' }) + ).toBeUndefined() + expect(readItemGeometry({ str: 'x', transform: [1, 0, 0, 1, 'a', 20] })).toBeUndefined() + }) + + it('reads placement from a horizontal transform', () => { + expect( + readItemGeometry({ + str: 'x', + transform: [1, 0, 0, 1, 10, 20], + width: 5, + height: 11, + dir: 'ltr', + }) + ).toEqual({ x: 10, y: 20, width: 5, height: 11 }) + }) +}) + +describe('helpers', () => { + it('collapses blanks without destroying line structure', () => { + expect(normalizePdfWhitespace('a b \n c\n\n\n\nd\t e')).toBe('a b\nc\n\nd e') + }) + + it('picks the character-weighted modal height as body height', () => { + expect( + dominantLineHeight([ + { text: 'Heading', height: 15.4 }, + { text: 'A long body line of text', height: 11 }, + { text: 'Another body line', height: 11.02 }, + ]) + ).toBe(11) + }) + + it('weighs only prose-like lines when the document has any, so table text cannot become the body', () => { + const cells = Array.from({ length: 200 }, (_, i) => ({ text: `${i} 4,512 7%`, height: 7.5 })) + const prose = [ + { text: 'This sentence is long enough and has enough words to count as prose.', height: 11 }, + ] + + expect(dominantLineHeight([...cells, ...prose])).toBe(11) + expect(dominantLineHeight(cells)).toBe(7.5) + }) + + it('disables heading markers when too many lines would qualify', () => { + const bullets = Array.from({ length: 6 }, () => ({ text: 'Bullet', height: 15 })) + const body = Array.from({ length: 4 }, () => ({ text: 'Body', height: 11 })) + + expect(headingMarkersViable([...bullets, ...body], 11)).toBe(false) + expect(headingMarkersViable([bullets[0], ...body], 11)).toBe(true) + }) + + it('collects words of three to forty letters and caps the set', () => { + const words = collectWords([ + { text: `ab abc ${'x'.repeat(41)} ${'y'.repeat(40)}`, height: BODY }, + ]) + expect(words).toEqual(new Set(['abc', 'y'.repeat(40)])) + + const letters = (n: number): string => + n < 26 ? String.fromCharCode(97 + n) : letters(Math.floor(n / 26)) + letters(n % 26) + const unique = Array.from({ length: 200_050 }, (_, i) => `w${letters(i)}z`).join(' ') + expect(collectWords([{ text: unique, height: BODY }]).size).toBe(200_000) + }) +}) diff --git a/apps/sim/lib/file-parsers/pdf-lines.ts b/apps/sim/lib/file-parsers/pdf-lines.ts new file mode 100644 index 00000000000..070765f9aa5 --- /dev/null +++ b/apps/sim/lib/file-parsers/pdf-lines.ts @@ -0,0 +1,484 @@ +/** + * Rebuilds lines and paragraphs from pdf.js text items. + * + * pdf.js only flags `hasEOL` when its own heuristics notice a line change; it + * resets that state when it recurses into a Form XObject and stays silent on a + * backwards x-move along one baseline, so items glue together without a + * separator. The builder here derives separators from item geometry instead and + * keeps each line's baseline and height so paragraph breaks, same-row cells, + * headings, and running furniture can be recovered afterwards. + */ + +/** One text item as pdf.js streams it; every field is untrusted. */ +export interface PdfTextItem { + str?: unknown + hasEOL?: unknown + transform?: unknown + width?: unknown + height?: unknown + dir?: unknown +} + +/** Horizontal, left-to-right placement of one item in PDF user space. */ +export interface PdfItemGeometry { + x: number + y: number + width: number + height: number +} + +/** A reconstructed line of one page. */ +export interface PdfLine { + text: string + /** Baseline in PDF user space (origin bottom-left); absent when the source carried no geometry. */ + y?: number + /** Height of the line's dominant item; 0 when unknown. */ + height: number +} + +export type PdfLineSeparator = '' | ' ' | '\n' + +export interface JoinLinesOptions { + /** Lowercase `a-b` compounds seen intact in the document; a line break on their hyphen keeps it. */ + compounds?: ReadonlySet + /** + * Lowercase words seen in the document. A line-end hyphen is dropped only when + * the joined word occurs elsewhere, so `high-` / `quality` keeps its hyphen + * while `Infra-` / `structure` rejoins when `Infrastructure` appears intact. + */ + words?: ReadonlySet + /** Dominant body-text height for the document; enables heading markers and heading pitch scaling. */ + bodyHeight?: number + /** Prefixes short, oversized lines with `## ` so Markdown-aware chunkers split on them. */ + headingMarkers?: boolean +} + +/** + * Whether `## ` heading prefixes are emitted by default. Off: on documents + * dominated by footnote, table, or form text the estimated body height is too + * small and prose becomes headings, which `TextChunker` then splits per line. + * The code path stays available through `JoinLinesOptions.headingMarkers`. + */ +export const PDF_HEADING_MARKERS_ENABLED = false + +/** + * Ceiling on reconstructed lines per page. Past it the builder stops splitting + * and appends to the last line, so a page of one-character lines cannot turn + * the per-line bookkeeping into hundreds of megabytes. + */ +export const MAX_PDF_LINES = 500_000 + +/** Ceiling on the document word set used for dehyphenation. */ +const MAX_COLLECTED_WORDS = 200_000 + +/** Words longer than this are noise (base64, hashes) and never hyphenation halves. */ +const MAX_COLLECTED_WORD_CHARS = 40 + +/** Prose-like lines (long, several words) alone decide the body text height. */ +const PROSE_MIN_CHARS = 40 +const PROSE_MIN_WORDS = 6 + +/** A heading candidate inside a longer run of same-height lines is body text, not a heading. */ +const MAX_HEADING_RUN = 3 + +/** A line at least this many times taller than body text is a heading candidate. */ +const HEADING_HEIGHT_RATIO = 1.15 + +/** Headings are short; longer oversized lines are pull quotes or callouts. */ +const HEADING_MAX_CHARS = 120 + +/** + * When more than this share of a document's lines would become headings the + * "body" height is really a bullet or caption size (slide decks), so markers + * would only add noise. + */ +const MAX_HEADING_LINE_FRACTION = 0.3 + +/** Line gap beyond this multiple of the page's line pitch is a paragraph break. */ +const PARAGRAPH_PITCH_RATIO = 1.3 + +/** Height change between adjacent lines beyond this fraction marks a heading/body boundary. */ +const HEIGHT_CHANGE_RATIO = 0.2 + +/** Lines whose baselines differ by less than this fraction of their height share a row. */ +const SAME_ROW_RATIO = 0.3 + +/** An upward return of at most this many pitches rejoins a wrapped table cell to its row. */ +const ROW_RETURN_PITCHES = 3 + +/** Baseline shift beyond this fraction of the reference height starts a new line. */ +const LINE_SHIFT_RATIO = 0.5 + +/** Backwards x-move beyond this fraction of the reference height starts a new line or cell. */ +const BACKWARDS_MOVE_RATIO = 0.5 + +/** Forward gap beyond this fraction of the reference height is an inter-word space. */ +const WORD_GAP_RATIO = 0.1 + +const SOFT_HYPHEN = '\u00AD' +const TRAILING_HYPHEN = /(\p{L}+)-$/u + +/** + * Only the tail of a line is inspected for a hyphenated word: an unanchored + * `(\p{L}+)-$` retried from every position of a megabyte-long line is + * quadratic, and no hyphenation half is longer than this. + */ +const HYPHEN_TAIL_CHARS = 64 +const LEADING_LOWERCASE_WORD = /^(\p{Ll}\p{L}*)/u +const LETTER = /\p{L}/u +const WORD_TOKEN = /\p{L}+/gu +const LEADING_WHITESPACE = /^\s/ +const TRAILING_WHITESPACE = /\s$/ + +/** + * Reads an item's placement, or undefined when the item is rotated, vertical, + * right-to-left, or carries no usable transform — those fall back to pdf.js's + * own `hasEOL` line breaks. + */ +export function readItemGeometry(item: PdfTextItem): PdfItemGeometry | undefined { + const transform = item.transform + if (!Array.isArray(transform) || transform.length < 6) return undefined + const [, skewY, skewX, , x, y] = transform as unknown[] + if ( + !isFiniteNumber(skewY) || + !isFiniteNumber(skewX) || + !isFiniteNumber(x) || + !isFiniteNumber(y) + ) { + return undefined + } + if (skewY !== 0 || skewX !== 0) return undefined + if (item.dir !== undefined && item.dir !== 'ltr') return undefined + return { + x, + y, + width: isFiniteNumber(item.width) ? item.width : 0, + height: isFiniteNumber(item.height) ? item.height : 0, + } +} + +/** Accumulates positioned items into lines for one page. */ +export class PdfLineBuilder { + private readonly lines: PdfLine[] = [] + private parts: string[] = [] + private lineY: number | undefined + private dominantHeight = 0 + private dominantLength = -1 + private prevEndX: number | undefined + private prevY = 0 + private lineHeight = 0 + + /** + * Separator the geometry rules call for before `str`; '' at line start or + * when either side lacks geometry. + */ + separatorBefore(str: string, geometry: PdfItemGeometry | undefined): PdfLineSeparator { + if (!geometry || this.prevEndX === undefined) return '' + const height = geometry.height || this.lineHeight + const ref = Math.max(height, this.lineHeight, 1) + if (Math.abs(geometry.y - this.prevY) > LINE_SHIFT_RATIO * ref) return '\n' + const gap = geometry.x - this.prevEndX + if (gap < -BACKWARDS_MOVE_RATIO * ref) return '\n' + if (gap > WORD_GAP_RATIO * ref && !this.endsWithWhitespace() && !LEADING_WHITESPACE.test(str)) + return ' ' + return '' + } + + append(str: string, geometry?: PdfItemGeometry): void { + if (str.length > 0) { + this.parts.push(str) + const visibleLength = str.trim().length + if (geometry && visibleLength > this.dominantLength) { + this.dominantLength = visibleLength + this.dominantHeight = geometry.height || this.lineHeight + } + } + if (!geometry) return + if (this.lineY === undefined && str.length > 0) this.lineY = geometry.y + this.prevEndX = geometry.x + geometry.width + this.prevY = geometry.y + this.lineHeight = geometry.height || this.lineHeight + } + + /** + * Closes the current line; whitespace-only lines are dropped. Past + * `MAX_PDF_LINES` the line stays open and later text joins it with a space. + */ + endLine(): void { + if (this.lines.length >= MAX_PDF_LINES) { + if (this.parts.length > 0) this.parts.push(' ') + return + } + const text = this.parts.join('') + if (text.trim().length > 0) { + this.lines.push({ text, y: this.lineY, height: this.dominantHeight }) + } + this.parts = [] + this.lineY = undefined + this.dominantHeight = 0 + this.dominantLength = -1 + this.prevEndX = undefined + this.prevY = 0 + this.lineHeight = 0 + } + + finish(): PdfLine[] { + if (this.lines.length >= MAX_PDF_LINES && this.parts.length > 0) { + const last = this.lines[this.lines.length - 1] + last.text = `${last.text} ${this.parts.join('')}` + this.parts = [] + } + this.endLine() + return this.lines + } + + private endsWithWhitespace(): boolean { + const last = this.parts[this.parts.length - 1] + return last !== undefined && TRAILING_WHITESPACE.test(last) + } +} + +/** Collapses runs of blanks without destroying line and paragraph breaks. */ +export function normalizePdfWhitespace(text: string): string { + return text + .replace(/[^\S\n]+/g, ' ') + .replace(/ ?\n ?/g, '\n') + .replace(/\n{3,}/g, '\n\n') +} + +/** + * Hyphenated compounds that appear intact inside a line, lowercased. Scans + * outward from each hyphen rather than matching `\p{L}+-\p{L}+`, which retries + * from every letter of a long hyphen-free line. + */ +export function collectCompounds(lines: Iterable): Set { + const compounds = new Set() + for (const line of lines) { + const text = line.text + for (let at = text.indexOf('-'); at !== -1; at = text.indexOf('-', at + 1)) { + const start = letterRunStart(text, at) + const end = letterRunEnd(text, at + 1) + if (start < at && end > at + 1) compounds.add(text.slice(start, end).toLowerCase()) + } + } + return compounds +} + +/** Start index of the run of letters ending just before `index`, at most `HYPHEN_TAIL_CHARS` long. */ +function letterRunStart(text: string, index: number): number { + let start = index + while (start > 0 && index - start < HYPHEN_TAIL_CHARS && LETTER.test(text[start - 1])) start-- + return start +} + +/** End index (exclusive) of the run of letters starting at `index`, at most `HYPHEN_TAIL_CHARS` long. */ +function letterRunEnd(text: string, index: number): number { + let end = index + while (end < text.length && end - index < HYPHEN_TAIL_CHARS && LETTER.test(text[end])) end++ + return end +} + +/** + * Lowercase words of three to `MAX_COLLECTED_WORD_CHARS` letters seen anywhere + * in the document, capped at `MAX_COLLECTED_WORDS` entries. + */ +export function collectWords(lines: Iterable): Set { + const words = new Set() + for (const line of lines) { + for (const match of line.text.matchAll(WORD_TOKEN)) { + const word = match[0] + if (word.length < 3 || word.length > MAX_COLLECTED_WORD_CHARS) continue + words.add(word.toLowerCase()) + if (words.size >= MAX_COLLECTED_WORDS) return words + } + } + return words +} + +/** + * Character-weighted modal height of the document's prose-like lines; falls + * back to every line when nothing reads as prose. 0 when unknown. + */ +export function dominantLineHeight(lines: Iterable): number { + const prose = new Map() + const all = new Map() + for (const line of lines) { + if (line.height <= 0) continue + const key = Math.round(line.height * 10) / 10 + const text = line.text.trim() + all.set(key, (all.get(key) ?? 0) + text.length) + if (isProseLike(text)) prose.set(key, (prose.get(key) ?? 0) + text.length) + } + const weights = prose.size > 0 ? prose : all + let best = 0 + let bestWeight = 0 + for (const [height, weight] of weights) { + if (weight > bestWeight) { + best = height + bestWeight = weight + } + } + return best +} + +/** + * Joins one page's lines into text with `\n` between lines, `\n\n` between + * paragraphs, and a space between cells that share a row, dehyphenating words + * that a line break split. + */ +export function joinLines(lines: readonly PdfLine[], options: JoinLinesOptions = {}): string { + if (lines.length === 0) return '' + const pitch = medianPitch(lines) + const bodyHeight = options.bodyHeight ?? 0 + const headingMarkers = options.headingMarkers ?? PDF_HEADING_MARKERS_ENABLED + const compounds = options.compounds + const words = options.words + const headingRuns = headingMarkers ? sameHeightRuns(lines) : undefined + + /** + * Output segments; the last one is always the text of the line being + * built, so hyphen checks and joins only ever touch one line's worth of + * string instead of the whole page. + */ + const parts: string[] = [decorate(lines[0], bodyHeight, headingRuns?.[0] ?? 0)] + for (let i = 1; i < lines.length; i++) { + const line = lines[i] + const separator = separatorBetween(lines, i, pitch, bodyHeight) + const last = parts[parts.length - 1] + if (last.endsWith(SOFT_HYPHEN)) { + parts[parts.length - 1] = last.slice(0, -1) + line.text + continue + } + if (separator === '\n') { + const joined = dehyphenate(last, line.text, compounds, words) + if (joined !== undefined) { + parts[parts.length - 1] = joined + continue + } + } + parts.push(separator) + parts.push(separator === ' ' ? line.text : decorate(line, bodyHeight, headingRuns?.[i] ?? 0)) + } + return parts.join('') +} + +/** Prefixes a heading candidate with `## `; `run` is 0 when markers are off. */ +function decorate(line: PdfLine, bodyHeight: number, run: number): string { + if (run > 0 && run <= MAX_HEADING_RUN && isHeadingCandidate(line, bodyHeight)) { + return `## ${line.text.trimStart()}` + } + return line.text +} + +function isHeadingCandidate(line: PdfLine, bodyHeight: number): boolean { + return ( + bodyHeight > 0 && + line.height >= HEADING_HEIGHT_RATIO * bodyHeight && + line.text.trim().length < HEADING_MAX_CHARS + ) +} + +function isProseLike(text: string): boolean { + return text.length >= PROSE_MIN_CHARS && text.split(/\s+/).length >= PROSE_MIN_WORDS +} + +/** Length of the run of consecutive same-height lines each line belongs to. */ +function sameHeightRuns(lines: readonly PdfLine[]): number[] { + const runs = new Array(lines.length) + let start = 0 + for (let i = 1; i <= lines.length; i++) { + if (i < lines.length && Math.abs(lines[i].height - lines[start].height) < 0.05) continue + for (let j = start; j < i; j++) runs[j] = i - start + start = i + } + return runs +} + +/** + * Whether heading markers make sense for a document: false when so many lines + * qualify that the dominant height is not the body text. + */ +export function headingMarkersViable(lines: Iterable, bodyHeight: number): boolean { + let total = 0 + let candidates = 0 + for (const line of lines) { + if (line.height <= 0) continue + total++ + if (isHeadingCandidate(line, bodyHeight)) candidates++ + } + return total === 0 || candidates / total <= MAX_HEADING_LINE_FRACTION +} + +/** + * Joins `next` onto `out` across a hyphen that ended the line, or undefined + * when the break is not a hyphenation. The hyphen is removed only when the + * document itself shows the joined word; a compound seen intact keeps it, and + * an unknown pair keeps it too, because `high-quality` split at a line end is + * far more common in real documents than a word the document never repeats. + */ +function dehyphenate( + out: string, + next: string, + compounds: ReadonlySet | undefined, + words: ReadonlySet | undefined +): string | undefined { + if (!out.endsWith('-')) return undefined + const head = TRAILING_HYPHEN.exec(out.slice(-HYPHEN_TAIL_CHARS)) + const tail = LEADING_LOWERCASE_WORD.exec(next) + if (!head || !tail) return undefined + const compound = `${head[1]}-${tail[1]}`.toLowerCase() + if (compounds?.has(compound)) return out + next + const joined = `${head[1]}${tail[1]}`.toLowerCase() + if (words?.has(joined)) return out.slice(0, -1) + next + return out + next +} + +function separatorBetween( + lines: readonly PdfLine[], + index: number, + pitch: number, + bodyHeight: number +): ' ' | '\n' | '\n\n' { + const a = lines[index - 1] + const b = lines[index] + if (a.y === undefined || b.y === undefined) return '\n' + const dy = a.y - b.y + const maxHeight = Math.max(a.height, b.height) + if (maxHeight > 0 ? Math.abs(dy) < SAME_ROW_RATIO * maxHeight : dy === 0) return ' ' + if (dy < 0) return isRowReturn(dy, pitch) ? ' ' : '\n\n' + const next = lines[index + 1] + if (next?.y !== undefined && isRowReturn(b.y - next.y, pitch)) return ' ' + if (maxHeight > 0 && Math.abs(a.height - b.height) > HEIGHT_CHANGE_RATIO * maxHeight) + return '\n\n' + const scale = bodyHeight > 0 ? Math.max(1, maxHeight / bodyHeight) : 1 + if (pitch > 0 && dy > PARAGRAPH_PITCH_RATIO * pitch * scale) return '\n\n' + return '\n' +} + +/** A short upward jump returns to a table row whose earlier cell wrapped onto extra lines. */ +function isRowReturn(dy: number, pitch: number): boolean { + return dy < 0 && pitch > 0 && -dy <= ROW_RETURN_PITCHES * pitch +} + +/** + * Lower median of the downward baseline steps between consecutive lines; 0 when + * there is none. The lower median keeps a two-step page treating its larger + * step as the paragraph gap rather than the pitch. + */ +function medianPitch(lines: readonly PdfLine[]): number { + const steps: number[] = [] + for (let i = 1; i < lines.length; i++) { + const a = lines[i - 1].y + const b = lines[i].y + if (a === undefined || b === undefined) continue + const dy = a - b + if (dy > 0) steps.push(dy) + } + if (steps.length === 0) return 0 + steps.sort((left, right) => left - right) + return steps[Math.floor((steps.length - 1) / 2)] +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) +} diff --git a/apps/sim/lib/file-parsers/pdf-parser-structure.test.ts b/apps/sim/lib/file-parsers/pdf-parser-structure.test.ts new file mode 100644 index 00000000000..65056208e27 --- /dev/null +++ b/apps/sim/lib/file-parsers/pdf-parser-structure.test.ts @@ -0,0 +1,266 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockOpenPdfDocument } = vi.hoisted(() => ({ + mockOpenPdfDocument: vi.fn(), +})) + +vi.mock('@/lib/file-parsers/pdfjs-server', () => ({ + openPdfDocument: mockOpenPdfDocument, +})) + +import { MAX_PDF_TEXT_CHARS, PdfParser } from '@/lib/file-parsers/pdf-parser' + +const PAGE_HEIGHT = 792 +const BODY = 11 +const PITCH = 14.4 +const PARAGRAPH_GAP = 20.4 + +interface PositionedItem { + str: string + hasEOL: boolean + transform: number[] + width: number + height: number + dir: 'ltr' +} + +interface BareItem { + str: string + hasEOL: boolean +} + +type StreamItem = PositionedItem | BareItem + +/** A positioned item the way pdf.js emits it for horizontal text. */ +function item(str: string, x: number, y: number, height = BODY): PositionedItem { + return { + str, + hasEOL: false, + transform: [height, 0, 0, height, x, y], + width: str.length * 5, + height, + dir: 'ltr', + } +} + +/** pdf.js marks a line change with an empty item positioned on the next baseline. */ +function eol(x: number, y: number): PositionedItem { + return { str: '', hasEOL: true, transform: [0, 0, 0, 0, x, y], width: 0, height: 0, dir: 'ltr' } +} + +/** Body lines at the in-paragraph pitch, each preceded by pdf.js's EOL marker. */ +function paragraph(texts: string[], top: number, x = 90): PositionedItem[] { + return texts.flatMap((text, index) => { + const y = top - index * PITCH + return [eol(x, y), item(text, x, y)] + }) +} + +function buildPage(items: StreamItem[]) { + const read = vi + .fn() + .mockResolvedValueOnce({ value: { items }, done: false }) + .mockResolvedValue({ done: true }) + return { + cleanup: vi.fn(), + getViewport: () => ({ height: PAGE_HEIGHT }), + streamTextContent: () => ({ + getReader: () => ({ read, cancel: vi.fn().mockResolvedValue(undefined) }), + }), + } +} + +function pdfWithPages(pages: StreamItem[][]) { + const built = pages.map(buildPage) + return { + numPages: pages.length, + getPage: vi.fn(async (pageNumber: number) => built[pageNumber - 1]), + destroy: vi.fn().mockResolvedValue(undefined), + } +} + +function pageWithFurniture(body: PositionedItem[], pageNumber: number): PositionedItem[] { + return [ + item('ACME Corp — Internal Use Only', 373, 729), + ...body, + eol(90, 55), + item(`Confidential draft, do not distribute — Page ${pageNumber} of 3`, 90, 55), + ] +} + +describe('PdfParser structure reconstruction', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('rebuilds paragraphs, headings, hyphenation, fused XObject text, and furniture', async () => { + const firstParagraph = [ + 'The revised rollout was flagged on 4 June by the Platform team. A rollback', + 'path exists and was rehearsed twice during the dry run. Stakeholders should', + 'review the attached appendix before the next checkpoint, and the owning', + 'team retains sign-off authority for scope changes above five percent.', + 'Exceptions require written approval from a director or above. This', + 'supersedes the guidance the Infrastructure team circulated on 14 March.', + ] + const p1Top = 676.6 + const p2Top = p1Top - 5 * PITCH - PARAGRAPH_GAP + const p3Top = p2Top - 2 * PITCH - PARAGRAPH_GAP + const p4Top = p3Top - 2 * PITCH - PARAGRAPH_GAP + const cautionY = p4Top - 34.6 + const formY = cautionY - PITCH + + const pageOne = pageWithFurniture( + [ + eol(90, 692), + item('Memo: Office Relocation Timeline', 90, 692, 15.4), + ...paragraph(firstParagraph, p1Top), + eol(90, p2Top), + item('The capacity model was archived on 25 June by the Infra', 90, p2Top), + item('-', 365, p2Top), + ...paragraph( + [ + 'structure team. Historical figures were restated to align with the model.', + 'Open questions are tracked in the shared register and reviewed weekly.', + ], + p2Top - PITCH + ), + ...paragraph( + [ + 'We compared attention-', + 'based models with attention-based baselines on the same hardware.', + 'Latency stayed under the objective for most sampled requests.', + ], + p3Top + ), + ...paragraph(['Keep records that support an item of income'], p4Top), + item('CAUTION', 320, cautionY), + item('Form 8815', 90, formY), + item('RECORDS', 160, formY), + ], + 1 + ) + const pageTwo = pageWithFurniture(paragraph(['Second page body text.'], p1Top), 2) + const pageThree = pageWithFurniture(paragraph(['Third page body text.'], p1Top), 3) + mockOpenPdfDocument.mockResolvedValueOnce(pdfWithPages([pageOne, pageTwo, pageThree])) + + const result = await new PdfParser().parseBuffer(Buffer.from('%PDF-1.4'), { + pdfTextMode: 'complete', + }) + + expect(result.content).toBe( + [ + 'ACME Corp — Internal Use Only', + '', + 'Memo: Office Relocation Timeline', + '', + ...firstParagraph, + '', + 'The capacity model was archived on 25 June by the Infrastructure team. Historical figures were restated to align with the model.', + 'Open questions are tracked in the shared register and reviewed weekly.', + '', + 'We compared attention-based models with attention-based baselines on the same hardware.', + 'Latency stayed under the objective for most sampled requests.', + '', + 'Keep records that support an item of income', + '', + 'CAUTION', + 'Form 8815 RECORDS', + '', + 'Confidential draft, do not distribute — Page 1 of 3', + '', + 'Second page body text.', + '', + 'Third page body text.', + ].join('\n') + ) + expect(result.metadata).toMatchObject({ pageCount: 3, truncated: false }) + }) + + it('keeps preview mode output structured as well', async () => { + mockOpenPdfDocument.mockResolvedValueOnce( + pdfWithPages([ + paragraph(['First line.', 'Second line.'], 700), + paragraph(['Next page.'], 700), + ]) + ) + + const result = await new PdfParser().parseBuffer(Buffer.from('%PDF-1.4')) + + expect(result.content).toBe('First line.\nSecond line.\n\nNext page.') + expect(result.metadata).toMatchObject({ pageCount: 2, truncated: false }) + }) + + it('still parses when a page cannot report its viewport', async () => { + const pdf = pdfWithPages([ + [...paragraph(['Header'], 760), ...paragraph(['Body one.'], 700)], + [...paragraph(['Header'], 760), ...paragraph(['Body two.'], 700)], + [...paragraph(['Header'], 760), ...paragraph(['Body three.'], 700)], + ]) + for (let pageNumber = 1; pageNumber <= 3; pageNumber++) { + const page = await pdf.getPage(pageNumber) + page.getViewport = () => { + throw new Error('no viewport') + } + } + mockOpenPdfDocument.mockResolvedValueOnce(pdf) + + const result = await new PdfParser().parseBuffer(Buffer.from('%PDF-1.4'), { + pdfTextMode: 'complete', + }) + + expect(result.content).toBe('Header\nBody one.\n\nHeader\nBody two.\n\nHeader\nBody three.') + }) + + it('keeps the free separator when the character budget cuts an item short', async () => { + const first = item('A'.repeat(MAX_PDF_TEXT_CHARS - 5), 90, 700) + const second = item('tail text', first.transform[4] + first.width + 6, 700) + mockOpenPdfDocument.mockResolvedValueOnce(pdfWithPages([[first, second]])) + + const result = await new PdfParser().parseBuffer(Buffer.from('%PDF-1.4')) + + expect(result.content.slice(MAX_PDF_TEXT_CHARS - 8, MAX_PDF_TEXT_CHARS)).toBe('AAA tail') + expect(result.metadata?.truncated).toBe(true) + }) + + it('flags preview output as truncated when paragraph breaks push it past the budget', async () => { + const long = MAX_PDF_TEXT_CHARS - 20 + mockOpenPdfDocument.mockResolvedValueOnce( + pdfWithPages([ + [ + item('A'.repeat(long), 90, 700), + eol(90, 700 - PITCH), + item('B'.repeat(9), 90, 700 - PITCH), + eol(90, 700 - 4 * PITCH), + item('C'.repeat(9), 90, 700 - 4 * PITCH), + ], + ]) + ) + + const result = await new PdfParser().parseBuffer(Buffer.from('%PDF-1.4')) + + expect(result.content).toContain('\n[... PDF text truncated at parser limits') + expect(result.metadata?.truncated).toBe(true) + expect(result.content.indexOf('[...')).toBe(MAX_PDF_TEXT_CHARS + 1) + }) + + it('falls back to hasEOL line breaks when items carry no geometry', async () => { + mockOpenPdfDocument.mockResolvedValueOnce( + pdfWithPages([ + [ + { str: 'alpha', hasEOL: true }, + { str: 'beta', hasEOL: false }, + { str: 'gamma', hasEOL: false }, + ], + ]) + ) + + const result = await new PdfParser().parseBuffer(Buffer.from('%PDF-1.4'), { + pdfTextMode: 'complete', + }) + + expect(result.content).toBe('alpha\nbetagamma') + }) +}) diff --git a/apps/sim/lib/file-parsers/pdf-parser.test.ts b/apps/sim/lib/file-parsers/pdf-parser.test.ts index 114746cc8d6..36a31010acf 100644 --- a/apps/sim/lib/file-parsers/pdf-parser.test.ts +++ b/apps/sim/lib/file-parsers/pdf-parser.test.ts @@ -5,6 +5,7 @@ import { deflateSync } from 'zlib' import { describe, expect, it } from 'vitest' import { MAX_PDF_TEXT_CHARS, PdfParser } from '@/lib/file-parsers/pdf-parser' import { openPdfDocument } from '@/lib/file-parsers/pdfjs-server' +import type { FileParseResult } from '@/lib/file-parsers/types' /** * Builds a single-page PDF that draws 64 characters per repeat from a @@ -122,6 +123,19 @@ function assemblePdf(objects: Buffer[], trailerEntries = ''): Buffer { return Buffer.concat(chunks) } +/** Repeats needed to exceed `MAX_PDF_TEXT_CHARS`; 64 characters per repeat. */ +const BOMB_REPEATS = 200_000 + +/** Evaluating the bomb takes pdf.js about two minutes; both bomb tests share one parse. */ +const BOMB_TIMEOUT_MS = 300_000 + +let bombParse: Promise | undefined + +function parseBomb(): Promise { + bombParse ??= new PdfParser().parseBuffer(buildTextBombPdf(BOMB_REPEATS)) + return bombParse +} + describe('PdfParser', () => { it('preloads the server worker instead of relying on a runtime-relative worker path', async () => { const previousWorker: unknown = Reflect.get(globalThis, 'pdfjsWorker') @@ -144,22 +158,30 @@ describe('PdfParser', () => { } }) - it('bounds extracted text from a compression-bomb PDF instead of exhausting the heap', async () => { - const bomb = buildTextBombPdf(200_000) - expect(bomb.length).toBeLessThan(200 * 1024) + it( + 'bounds extracted text from a compression-bomb PDF instead of exhausting the heap', + async () => { + const bomb = buildTextBombPdf(BOMB_REPEATS) + expect(bomb.length).toBeLessThan(200 * 1024) - const result = await new PdfParser().parseBuffer(bomb) + const result = await parseBomb() - expect(result.metadata?.truncated).toBe(true) - expect(result.metadata?.warning).toMatch(/parser limit/i) - expect(result.content.length).toBeLessThanOrEqual(MAX_PDF_TEXT_CHARS + 200) - }, 120_000) + expect(result.metadata?.truncated).toBe(true) + expect(result.metadata?.warning).toMatch(/parser limit/i) + expect(result.content.length).toBeLessThanOrEqual(MAX_PDF_TEXT_CHARS + 200) + }, + BOMB_TIMEOUT_MS + ) - it('marks truncated content inline so callers reading only content can see it', async () => { - const result = await new PdfParser().parseBuffer(buildTextBombPdf(200_000)) + it( + 'marks truncated content inline so callers reading only content can see it', + async () => { + const result = await parseBomb() - expect(result.content).toMatch(/\[\.\.\. PDF text truncated at parser limits.* \.\.\.\]/) - }, 120_000) + expect(result.content).toMatch(/\[\.\.\. PDF text truncated at parser limits.* \.\.\.\]/) + }, + BOMB_TIMEOUT_MS + ) it('extracts a real multi-page PDF past the preview budget completely', async () => { const result = await new PdfParser().parseBuffer(buildLargeTypesetPdf(60), { @@ -203,7 +225,8 @@ describe('PdfParser', () => { it('preserves the password-required error for encrypted PDFs', async () => { await expect(new PdfParser().parseBuffer(buildEncryptedPdf())).rejects.toMatchObject({ - name: 'PasswordException', + name: 'FileParserError', + code: 'encrypted_file', }) }) }) diff --git a/apps/sim/lib/file-parsers/pdf-parser.ts b/apps/sim/lib/file-parsers/pdf-parser.ts index c0921d67a84..66f24009326 100644 --- a/apps/sim/lib/file-parsers/pdf-parser.ts +++ b/apps/sim/lib/file-parsers/pdf-parser.ts @@ -1,7 +1,23 @@ import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' +import { sleep } from '@sim/utils/helpers' import type { PDFDocumentProxy, PDFPageProxy } from 'pdfjs-dist/types/src/pdf' import { FileParserError } from '@/lib/file-parsers/errors' +import { type PdfPageLines, suppressFurniture } from '@/lib/file-parsers/pdf-furniture' +import { + collectCompounds, + collectWords, + dominantLineHeight, + headingMarkersViable, + joinLines, + normalizePdfWhitespace, + PDF_HEADING_MARKERS_ENABLED, + type PdfItemGeometry, + type PdfLine, + PdfLineBuilder, + type PdfTextItem, + readItemGeometry, +} from '@/lib/file-parsers/pdf-lines' import { openPdfDocument } from '@/lib/file-parsers/pdfjs-server' import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types' import { sanitizeTextForUTF8, truncationNotice } from '@/lib/file-parsers/utils' @@ -27,6 +43,20 @@ export const MAX_COMPLETE_PDF_PAGE_CHARS = 250_000 /** Wall-clock ceiling for extracting text from a whole document. */ const PDF_EXTRACTION_TIMEOUT_MS = 60_000 +/** + * Upper bound on what line reconstruction adds per line after the budget is + * spent: a two-character paragraph break plus a three-character heading marker. + * The complete-mode byte ceiling therefore trips slightly earlier than it did + * when pages were flattened to one line, by at most this many bytes per line. + */ +const MAX_LINE_DECORATION_BYTES = 5 + +/** Pages assembled between event-loop yields, so a long document cannot block the loop. */ +const ASSEMBLY_YIELD_EVERY_PAGES = 32 + +/** Pages are joined with a paragraph break. */ +const PAGE_SEPARATOR = '\n\n' + const PDF_TRUNCATION_WARNING = 'PDF text extraction stopped at a parser limit and is incomplete' const PDF_READ_DEADLINE_REACHED = Symbol('PDF_READ_DEADLINE_REACHED') @@ -34,11 +64,11 @@ const PDF_READ_DEADLINE_REACHED = Symbol('PDF_READ_DEADLINE_REACHED') const PDF_PARSER_SOURCE = 'unpdf' interface TextContentChunk { - items?: Array<{ str?: unknown; hasEOL?: unknown }> + items?: PdfTextItem[] } interface PageExtraction { - text: string + lines: PdfLine[] /** Characters consumed from the caller's budget. */ used: number /** False when a budget stopped the read before the page was exhausted. */ @@ -141,7 +171,7 @@ async function readPageWithinBudget( .streamTextContent() .getReader() as ReadableStreamDefaultReader - const parts: string[] = [] + const builder = new PdfLineBuilder() let remaining = budget let completed = false let dropped = false @@ -180,16 +210,24 @@ async function readPageWithinBudget( for (const item of value?.items ?? []) { if (typeof item?.str !== 'string') continue - const piece = item.hasEOL === true ? `${item.str}\n` : item.str - if (piece.length > remaining) { - parts.push(piece.slice(0, remaining)) + const str = item.str + const hasEOL = item.hasEOL === true + const geometry = readItemGeometry(item) + const separator = str.length > 0 ? builder.separatorBefore(str, geometry) : '' + /** Only text and pdf.js's own line breaks count, exactly as before geometry separators existed. */ + const cost = str.length + (hasEOL ? 1 : 0) + if (cost > remaining) { + appendTruncated(builder, separator, str, geometry, remaining) remaining = 0 dropped = true break } - if (piece.length > 0) parts.push(piece) - remaining -= piece.length + if (separator === '\n') builder.endLine() + else if (separator.length > 0) builder.append(separator) + builder.append(str, geometry) + if (hasEOL) builder.endLine() + remaining -= cost } } } finally { @@ -201,7 +239,76 @@ async function readPageWithinBudget( } } - return { text: parts.join(''), used: budget - remaining, completed, deadlineReached } + return { lines: builder.finish(), used: budget - remaining, completed, deadlineReached } +} + +/** Applies the free separator, then as much of `str` as `remaining` allows, mirroring the old `slice(0, remaining)`. */ +function appendTruncated( + builder: PdfLineBuilder, + separator: string, + str: string, + geometry: PdfItemGeometry | undefined, + remaining: number +): void { + if (remaining <= 0) return + if (separator === '\n') builder.endLine() + else if (separator.length > 0) builder.append(separator) + builder.append(str.slice(0, remaining), geometry) +} + +/** Page height in user space, or undefined when the page cannot report a viewport. */ +function readPageHeight(page: PDFPageProxy): number | undefined { + if (typeof page.getViewport !== 'function') return undefined + try { + const height = page.getViewport({ scale: 1 }).height + return Number.isFinite(height) && height > 0 ? height : undefined + } catch { + return undefined + } +} + +/** Bytes a page's lines can occupy in the output once joined and decorated. */ +function estimatePageBytes(lines: readonly PdfLine[]): number { + let bytes = 0 + for (const line of lines) { + bytes += Buffer.byteLength(line.text, 'utf8') + MAX_LINE_DECORATION_BYTES + } + return bytes +} + +/** + * Turns the collected pages into text: repeated furniture is dropped, lines are + * joined into paragraphs, hyphenation is undone, and pages are separated by a + * paragraph break. Yields to the event loop periodically and honours `signal` + * between pages, since this runs after the streaming budgets have stopped. + */ +async function assemblePages( + pages: readonly PdfPageLines[], + complete: boolean, + signal: AbortSignal | undefined +): Promise { + signal?.throwIfAborted() + const filteredPages = suppressFurniture(pages) + const allLines = filteredPages.flat() + const bodyHeight = dominantLineHeight(allLines) + const options = { + compounds: collectCompounds(allLines), + words: collectWords(allLines), + bodyHeight, + headingMarkers: PDF_HEADING_MARKERS_ENABLED && headingMarkersViable(allLines, bodyHeight), + } + const pageTexts: string[] = [] + for (const [index, lines] of filteredPages.entries()) { + if (index > 0 && index % ASSEMBLY_YIELD_EVERY_PAGES === 0) { + await sleep(0) + signal?.throwIfAborted() + } + const joined = joinLines(lines, options) + const text = complete ? normalizePdfWhitespace(sanitizeTextForUTF8(joined)).trim() : joined + if (text.length > 0) pageTexts.push(text) + } + const text = pageTexts.join(PAGE_SEPARATOR) + return complete ? text : normalizePdfWhitespace(text).trim() } function completeExtractionLimit(message: string): FileParserError { @@ -217,7 +324,7 @@ async function extractTextWithinBudget( const complete = options.pdfTextMode === 'complete' const totalPages = pdf.numPages const pageLimit = Math.min(totalPages, MAX_PDF_PAGES) - const pageTexts: string[] = [] + const pages: PdfPageLines[] = [] let remainingChars = MAX_PDF_TEXT_CHARS let outputBytes = 0 @@ -250,6 +357,7 @@ async function extractTextWithinBudget( } const page = pageResult + const pageHeight = readPageHeight(page) let extraction: PageExtraction try { extraction = await readPageWithinBudget( @@ -262,26 +370,25 @@ async function extractTextWithinBudget( page.cleanup() } - const { text, used, completed } = extraction + const { lines, used, completed } = extraction if (!complete) remainingChars -= used /** A page stopped before yielding text must not count as read or add a separator. */ - if (completed || text.length > 0) { + if (completed || used > 0) { pagesRead++ if (complete) { - const normalized = sanitizeTextForUTF8(text.replace(/\s+/g, ' ')).trim() - if (normalized.length > 0) { - outputBytes += Buffer.byteLength(normalized, 'utf8') + (pageTexts.length > 0 ? 1 : 0) + if (lines.length > 0) { + outputBytes += estimatePageBytes(lines) + (pages.length > 0 ? PAGE_SEPARATOR.length : 0) if (outputBytes > MAX_COMPLETE_PDF_TEXT_BYTES) { throw completeExtractionLimit( `PDF text exceeds the safe ${MAX_COMPLETE_PDF_TEXT_BYTES.toLocaleString()}-byte output limit.` ) } - pageTexts.push(normalized) + pages.push({ lines, pageHeight }) } } else { - pageTexts.push(text) + pages.push({ lines, pageHeight }) } } @@ -298,8 +405,16 @@ async function extractTextWithinBudget( } } + let text = await assemblePages(pages, complete, signal) + + /** Paragraph breaks land after the budget is spent; trimming that overflow is a truncation too. */ + if (!complete && text.length > MAX_PDF_TEXT_CHARS) { + text = text.slice(0, MAX_PDF_TEXT_CHARS) + truncated = true + } + return { - text: complete ? pageTexts.join(' ') : pageTexts.join('\n').replace(/\s+/g, ' '), + text, totalPages, pagesRead, truncated, diff --git a/apps/sim/lib/file-parsers/pdfjs-server.test.ts b/apps/sim/lib/file-parsers/pdfjs-server.test.ts index 5983c47734b..ed5112d2e7b 100644 --- a/apps/sim/lib/file-parsers/pdfjs-server.test.ts +++ b/apps/sim/lib/file-parsers/pdfjs-server.test.ts @@ -27,6 +27,7 @@ vi.mock('pdfjs-dist/legacy/build/pdf.worker.mjs', () => ({ WorkerMessageHandler: workerMessageHandler, })) +import { FileParserError } from '@/lib/file-parsers/errors' import { openPdfDocument } from '@/lib/file-parsers/pdfjs-server' describe('openPdfDocument', () => { @@ -76,4 +77,26 @@ describe('openPdfDocument', () => { resolveLoading?.({ destroy: lateDocumentDestroy }) await vi.waitFor(() => expect(lateDocumentDestroy).toHaveBeenCalledOnce()) }) + + it.each([ + ['InvalidPDFException', 'Invalid PDF structure.', 'invalid_format'], + ['FormatError', 'Bad XRef entry', 'invalid_format'], + ['PasswordException', 'No password given', 'encrypted_file'], + ])('maps the pdf.js %s to a typed parser failure', async (name, message, code) => { + const pdfjsError = Object.assign(new Error(message), { name }) + mockGetDocument.mockReturnValueOnce({ promise: Promise.reject(pdfjsError) }) + + const error = await openPdfDocument(new Uint8Array([1])).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(FileParserError) + expect(error).toMatchObject({ code }) + expect((error as FileParserError).cause).toBe(pdfjsError) + }) + + it('leaves an unrecognized pdf.js failure untyped so it stays retryable', async () => { + const unknownError = new Error('worker crashed') + mockGetDocument.mockReturnValueOnce({ promise: Promise.reject(unknownError) }) + + await expect(openPdfDocument(new Uint8Array([1]))).rejects.toBe(unknownError) + }) }) diff --git a/apps/sim/lib/file-parsers/pdfjs-server.ts b/apps/sim/lib/file-parsers/pdfjs-server.ts index e1c0274a5f3..b4e4a6a9b94 100644 --- a/apps/sim/lib/file-parsers/pdfjs-server.ts +++ b/apps/sim/lib/file-parsers/pdfjs-server.ts @@ -1,4 +1,5 @@ import type { PDFDocumentLoadingTask, PDFDocumentProxy } from 'pdfjs-dist/types/src/pdf' +import { FileParserError } from '@/lib/file-parsers/errors' let pdfRuntime: Promise | undefined @@ -70,6 +71,26 @@ function waitForLoadingTask( }) } +/** pdf.js exception classes that mean the bytes are not a readable PDF. */ +const INVALID_PDF_ERROR_NAMES = new Set(['InvalidPDFException', 'FormatError']) + +/** + * pdf.js reports failures as its own exception classes whose `name` survives + * the worker boundary. Untyped, they classify as transient and are retried + * forever; this is the single choke point every pdf.js caller shares, so the + * mapping to the parser code taxonomy lives here. + */ +function toTypedPdfError(error: unknown): unknown { + if (!(error instanceof Error)) return error + if (error.name === 'PasswordException') { + return new FileParserError('encrypted_file', 'This PDF is password-protected', error) + } + if (INVALID_PDF_ERROR_NAMES.has(error.name)) { + return new FileParserError('invalid_format', `Invalid PDF: ${error.message}`, error) + } + return error +} + /** Open a PDF with the server-compatible pdf.js build and hardened defaults. */ export async function openPdfDocument( data: Uint8Array, @@ -85,5 +106,9 @@ export async function openPdfDocument( useSystemFonts: true, }) - return waitForLoadingTask(loadingTask, signal) + try { + return await waitForLoadingTask(loadingTask, signal) + } catch (error) { + throw toTypedPdfError(error) + } } diff --git a/apps/sim/lib/file-parsers/pptx-parser.test.ts b/apps/sim/lib/file-parsers/pptx-parser.test.ts index 93cca1433be..d0465696eef 100644 --- a/apps/sim/lib/file-parsers/pptx-parser.test.ts +++ b/apps/sim/lib/file-parsers/pptx-parser.test.ts @@ -1,45 +1,35 @@ /** * @vitest-environment node */ -import { describe, expect, it, vi } from 'vitest' - -const { mockParseOfficeText } = vi.hoisted(() => ({ - mockParseOfficeText: vi.fn(), -})) - -vi.mock('@/lib/file-parsers/officeparser-module', () => ({ - parseOfficeText: mockParseOfficeText, -})) - +import { describe, expect, it } from 'vitest' import type { FileParserError } from '@/lib/file-parsers/errors' import { PptxParser } from '@/lib/file-parsers/pptx-parser' -describe('PptxParser', () => { - it('classifies encrypted legacy presentations before degraded extraction', async () => { - const libraryError = new Error('File is password-protected') - mockParseOfficeText.mockRejectedValueOnce(libraryError) - const legacyOleBuffer = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]) +const LEGACY_OLE_BUFFER = Buffer.concat([ + Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]), + Buffer.alloc(2048), +]) - const result = new PptxParser().parseBuffer(legacyOleBuffer) +describe('PptxParser', () => { + it('rejects a legacy OLE .ppt as unsupported rather than scraping its bytes', async () => { + await expect( + new PptxParser().parseBuffer(LEGACY_OLE_BUFFER) + ).rejects.toMatchObject({ code: 'unsupported_type' }) + }) - await expect(result).rejects.toMatchObject({ - code: 'encrypted_file', - cause: libraryError, - }) + it('rejects bytes that are neither a package nor an OLE container', async () => { + await expect( + new PptxParser().parseBuffer(Buffer.from('random presentation bytes')) + ).rejects.toMatchObject({ code: 'invalid_format' }) }) - it('preserves cancellation instead of degrading to scraped bytes', async () => { + it('preserves cancellation instead of classifying the container', async () => { const controller = new AbortController() const abortError = new DOMException('The operation was aborted', 'AbortError') - mockParseOfficeText.mockImplementationOnce(async () => { - controller.abort(abortError) - throw abortError - }) + controller.abort(abortError) await expect( - new PptxParser().parseBuffer(Buffer.from('legacy presentation'), { - signal: controller.signal, - }) + new PptxParser().parseBuffer(LEGACY_OLE_BUFFER, { signal: controller.signal }) ).rejects.toBe(abortError) }) }) diff --git a/apps/sim/lib/file-parsers/pptx-parser.ts b/apps/sim/lib/file-parsers/pptx-parser.ts index db7c50aa37d..8c8614d8a70 100644 --- a/apps/sim/lib/file-parsers/pptx-parser.ts +++ b/apps/sim/lib/file-parsers/pptx-parser.ts @@ -1,14 +1,22 @@ import { existsSync } from 'fs' import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' -import { FileParserError, isEncryptedOfficeParserError } from '@/lib/file-parsers/errors' -import { parseOfficeText } from '@/lib/file-parsers/officeparser-module' +import { FileParserError, isFileParserError } from '@/lib/file-parsers/errors' +import { isEncryptedOoxmlContainer, isOle2Container } from '@/lib/file-parsers/ooxml-encryption' +import { extractPresentationText } from '@/lib/file-parsers/ooxml-presentation' import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types' import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' -import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard' +import { assertOoxmlArchiveWithinLimits, isZipShaped } from '@/lib/file-parsers/zip-guard' const logger = createLogger('PptxParser') +/** + * Extracts presentation text. PresentationML packages go through the slide XML + * walker, which keeps table rows together and skips layout placeholders. An OLE + * container is either an encrypted OOXML package, reported as such, or a legacy + * `.ppt`, which has no pure-JS extractor and is rejected as unsupported rather + * than scraped for printable bytes. + */ export class PptxParser implements FileParser { async parseFile(filePath: string, options: FileParseOptions = {}): Promise { if (!filePath) { @@ -35,78 +43,66 @@ export class PptxParser implements FileParser { assertOoxmlArchiveWithinLimits(buffer) - try { - const result = await parseOfficeText(buffer, options) - - if (!result || typeof result !== 'string') { - return this.fallbackExtraction(buffer) - } + if (isZipShaped(buffer)) { + return this.parsePackage(buffer, options) + } - const content = sanitizeTextForUTF8(result.trim()) + if (isOle2Container(buffer)) { + this.rejectOleContainer(buffer) + } - logger.info('PowerPoint parsing completed successfully with officeparser') + throw new FileParserError( + 'invalid_format', + 'The file is neither a PowerPoint package nor a legacy PowerPoint binary' + ) + } - return { - content: content, - metadata: { - characterCount: content.length, - extractionMethod: 'officeparser', - }, - } - } catch (extractError) { + private async parsePackage(buffer: Buffer, options: FileParseOptions): Promise { + let extracted: string + try { + extracted = await extractPresentationText(buffer, options) + } catch (error) { options.signal?.throwIfAborted() - if (isEncryptedOfficeParserError(extractError)) { - throw new FileParserError( - 'encrypted_file', - 'This presentation is encrypted or password-protected', - extractError - ) - } - - const isZipFile = buffer.length >= 2 && buffer[0] === 0x50 && buffer[1] === 0x4b - if (!isZipFile) { - logger.warn('officeparser failed for legacy PowerPoint, using fallback:', extractError) - return this.fallbackExtraction(buffer) - } - + if (isFileParserError(error)) throw error throw new FileParserError( 'invalid_format', 'The PowerPoint container could not be read', - extractError + error ) } - } - - private fallbackExtraction(buffer: Buffer): FileParseResult { - logger.info('Using fallback text extraction for PowerPoint file') - const text = buffer.toString('utf8', 0, Math.min(buffer.length, 200000)) - - const readableText = text - .match(/[\x20-\x7E\s]{4,}/g) - ?.filter( - (chunk) => - chunk.trim().length > 10 && - /[a-zA-Z]/.test(chunk) && - !/^[\x00-\x1F]*$/.test(chunk) && - !/^[^\w\s]*$/.test(chunk) + const content = sanitizeTextForUTF8(extracted.trim()) + if (!content) { + throw new FileParserError( + 'no_extractable_text', + 'No text could be extracted from this presentation' ) - .join(' ') - .replace(/\s+/g, ' ') - .trim() - - const content = readableText - ? sanitizeTextForUTF8(readableText) - : 'Unable to extract text from PowerPoint file. Please ensure the file contains readable text content.' + } return { content, metadata: { - extractionMethod: 'fallback', - degraded: true, characterCount: content.length, - warning: 'Basic text extraction used', + extractionMethod: 'ooxml-walker', }, } } + + /** + * Neither OLE shape has a reader here: officeparser 5 only throws a generic + * error for both, so the encrypted case is recognized from the container's + * own stream directory instead. + */ + private rejectOleContainer(buffer: Buffer): never { + if (isEncryptedOoxmlContainer(buffer)) { + throw new FileParserError( + 'encrypted_file', + 'This presentation is encrypted or password-protected' + ) + } + throw new FileParserError( + 'unsupported_type', + 'Legacy .ppt presentations are not supported. Save the file as .pptx and retry.' + ) + } } diff --git a/apps/sim/lib/file-parsers/registry.test.ts b/apps/sim/lib/file-parsers/registry.test.ts index 6c7c0769e40..05c205a1e81 100644 --- a/apps/sim/lib/file-parsers/registry.test.ts +++ b/apps/sim/lib/file-parsers/registry.test.ts @@ -36,7 +36,6 @@ const ALL_SUPPORTED_TYPES: SupportedFileType[] = [ 'html', 'htm', 'pptx', - 'ppt', 'pptm', 'potx', 'odt', @@ -81,9 +80,11 @@ describe('file parser registry', () => { /** * Formats with no bundled extractor must not claim support. `rtf` especially: * `DocParser`'s plaintext branch would pass its control words through as prose. + * Legacy `ppt` was registered once and only ever produced scraped placeholder + * prose, so it is refused up front with the unsupported-type message instead. */ it('does not claim formats with no extractor', () => { - for (const extension of ['rtf', 'msg', 'eml', 'pages', 'key', 'one', 'vsdx', 'png']) { + for (const extension of ['rtf', 'msg', 'eml', 'pages', 'key', 'one', 'vsdx', 'png', 'ppt']) { expect(isSupportedFileType(extension), `unexpectedly claims .${extension}`).toBe(false) } }) diff --git a/apps/sim/lib/file-parsers/sheet-display-text.test.ts b/apps/sim/lib/file-parsers/sheet-display-text.test.ts new file mode 100644 index 00000000000..906381edc40 --- /dev/null +++ b/apps/sim/lib/file-parsers/sheet-display-text.test.ts @@ -0,0 +1,289 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import * as XLSX from 'xlsx' +import { + generalNumberText, + isoDateText, + isTimeOnlyFormat, + normalizeSheetDisplayText, +} from '@/lib/file-parsers/sheet-display-text' +import { XlsxParser } from '@/lib/file-parsers/xlsx-parser' + +/** + * Every date below is built with `Date.UTC` and asserted as an ISO slice, so + * the expectations hold whatever `TZ` the runner has. The suite is also run + * under `TZ=Asia/Tokyo` and `TZ=America/Los_Angeles` from the CLI to prove the + * parser itself is zone-independent: a `String(date)` rendering would print + * the runner's zone and a serial-to-local conversion would shift the day. + */ +function typedSheet(): XLSX.WorkSheet { + const sheet = XLSX.utils.aoa_to_sheet([ + ['Issued', 'At', 'Rate', 'Amount', 'Paid', 'Total', 'Card', 'Sum', 'Note'], + ]) + sheet.A2 = { t: 'd', v: new Date(Date.UTC(2026, 2, 4)), z: 'm/d/yyyy' } + sheet.B2 = { t: 'd', v: new Date(Date.UTC(2026, 2, 4, 12)), z: 'm/d/yyyy h:mm' } + sheet.C2 = { t: 'n', v: 0.085, z: '0.0%' } + sheet.D2 = { t: 'n', v: 1250, z: '"$"#,##0.00' } + sheet.E2 = { t: 'b', v: true } + sheet.F2 = { t: 'n', v: 2500, f: 'D2*2', z: '"$"#,##0.00' } + sheet.G2 = { t: 'n', v: 4111111111111111 } + sheet.H2 = { t: 'n', v: 0.1 + 0.2 } + sheet.I2 = { t: 's', v: 'left\tright' } + sheet['!ref'] = 'A1:I2' + return sheet +} + +function typedWorkbook(bookType: XLSX.BookType, date1904 = false): Buffer { + const book = XLSX.utils.book_new() + XLSX.utils.book_append_sheet(book, typedSheet(), 'Ledger') + if (date1904) book.Workbook = { WBProps: { date1904: true } } + return XLSX.write(book, { type: 'buffer', bookType }) as Buffer +} + +function dataRow(content: string): string[] { + const lines = content.split('\n') + return lines[lines.length - 1].split('\t') +} + +describe('XlsxParser display text', () => { + it('indexes the text a user sees rather than the stored value', async () => { + const result = await new XlsxParser().parseBuffer(typedWorkbook('xlsx')) + + expect(dataRow(result.content)).toEqual([ + '2026-03-04', + '2026-03-04T12:00:00', + '8.5%', + '$1,250.00', + 'TRUE', + '$2,500.00', + '4111111111111111', + '0.3', + 'left right', + ]) + }) + + it('renders dates from a date1904 workbook identically', async () => { + const result = await new XlsxParser().parseBuffer(typedWorkbook('xlsx', true)) + + expect(dataRow(result.content).slice(0, 2)).toEqual(['2026-03-04', '2026-03-04T12:00:00']) + }) + + /** + * A time-of-day serial lands on 1899-12-31 in a 1900 workbook and on + * 1904-01-01 in a 1904 one, so the decision must come from the format, not + * the epoch date. Elapsed formats are durations Excel shows as `30:00`. + */ + describe.each([ + ['xlsx', false], + ['xlsx', true], + ['xls', false], + ['xls', true], + ['xlsb', false], + ['xlsb', true], + ] as const)('time cells in %s (date1904: %s)', (bookType, date1904) => { + function timeWorkbook(): Buffer { + const sheet = XLSX.utils.aoa_to_sheet([['Clock', 'Meridiem', 'Elapsed', 'Minutes', 'Month']]) + sheet.A2 = { t: 'n', v: 0.520821759, z: 'h:mm:ss' } + sheet.B2 = { t: 'n', v: 0.75, z: 'hh:mm AM/PM' } + sheet.C2 = { t: 'n', v: 1.25, z: '[h]:mm' } + sheet.D2 = { t: 'n', v: 0.5, z: '[mm]:ss' } + sheet.E2 = { t: 'n', v: 46085, z: 'mmm' } + sheet['!ref'] = 'A1:E2' + const book = XLSX.utils.book_new() + XLSX.utils.book_append_sheet(book, sheet, 'Times') + if (date1904) book.Workbook = { WBProps: { date1904: true } } + return XLSX.write(book, { type: 'buffer', bookType }) as Buffer + } + + it('rounds float serials to the second instead of truncating', async () => { + const sheet = XLSX.utils.aoa_to_sheet([['When', 'Clock']]) + sheet.A2 = { t: 'n', v: 45366.572916666664, z: 'yyyy-mm-dd h:mm' } + sheet.B2 = { t: 'n', v: 0.6041666666666666, z: 'h:mm' } + sheet['!ref'] = 'A1:B2' + const book = XLSX.utils.book_new() + XLSX.utils.book_append_sheet(book, sheet, 'Times') + if (date1904) book.Workbook = { WBProps: { date1904: true } } + const buffer = XLSX.write(book, { type: 'buffer', bookType }) as Buffer + + const result = await new XlsxParser().parseBuffer(buffer) + + const row = dataRow(result.content) + expect(row[0]).toMatch(/^\d{4}-\d{2}-\d{2}T13:45:00$/) + expect(row[1]).toBe('14:30:00') + }) + + it('renders time-only cells as times and elapsed cells as durations', async () => { + const result = await new XlsxParser().parseBuffer(timeWorkbook()) + + const row = dataRow(result.content) + expect(row.slice(0, 4)).toEqual(['12:29:59', '18:00:00', '30:00', '720:00']) + expect(row[4]).toMatch(/^\d{4}-\d{2}-\d{2}$/) + }) + }) + + /** + * The SheetJS ODS writer stores each serial as the cell text, so an elapsed + * cell reads back its serial; time-only cells still come from the format. + */ + it.each([false, true])('renders time-only cells from ods (date1904: %s)', async (date1904) => { + const sheet = XLSX.utils.aoa_to_sheet([['Clock']]) + sheet.A2 = { t: 'n', v: 0.520821759, z: 'h:mm:ss' } + sheet['!ref'] = 'A1:A2' + const book = XLSX.utils.book_new() + XLSX.utils.book_append_sheet(book, sheet, 'Times') + if (date1904) book.Workbook = { WBProps: { date1904: true } } + const buffer = XLSX.write(book, { type: 'buffer', bookType: 'ods' }) as Buffer + + const result = await new XlsxParser().parseBuffer(buffer) + + expect(dataRow(result.content)).toEqual(['12:29:59']) + }) + + it.each(['xls', 'xlsb'] as const)('renders the same display text from %s', async (bookType) => { + const result = await new XlsxParser().parseBuffer(typedWorkbook(bookType)) + + expect(dataRow(result.content).slice(0, 6)).toEqual([ + '2026-03-04', + '2026-03-04T12:00:00', + '8.5%', + '$1,250.00', + 'TRUE', + '$2,500.00', + ]) + }) + + /** + * The SheetJS ODS writer emits each number's stored value as the cell text + * the reader then trusts, so only dates, booleans and General numbers can be + * asserted through a round trip. + */ + it('renders ISO dates from an ods round trip', async () => { + const result = await new XlsxParser().parseBuffer(typedWorkbook('ods')) + + const row = dataRow(result.content) + expect(row.slice(0, 2)).toEqual(['2026-03-04', '2026-03-04T12:00:00']) + expect(row[4]).toBe('TRUE') + expect(row[6]).toBe('4111111111111111') + }) + + it('keeps the sampled metadata on display text as well', async () => { + const result = await new XlsxParser().parseBuffer(typedWorkbook('xlsx')) + + const sampled = result.metadata?.sampledData as string[][] + expect(sampled[1].slice(0, 4)).toEqual([ + '2026-03-04', + '2026-03-04T12:00:00', + '8.5%', + '$1,250.00', + ]) + }) +}) + +describe('isoDateText', () => { + it('drops a midnight time and keeps a non-midnight one without a zone suffix', () => { + expect(isoDateText(new Date(Date.UTC(2026, 2, 4)))).toBe('2026-03-04') + expect(isoDateText(new Date(Date.UTC(2026, 2, 4, 12, 30, 15)))).toBe('2026-03-04T12:30:15') + }) + + it('renders an invalid date as empty text', () => { + expect(isoDateText(new Date(Number.NaN))).toBe('') + }) + + it('rounds the sub-second drift of a float serial to the nearest second', () => { + const datetime = XLSX.SSF.parse_date_code(45366.572916666664) + const time = XLSX.SSF.parse_date_code(0.6041666666666666) + const toDate = (d: XLSX.SSF.DateObject) => + new Date(Date.UTC(d.y, d.m - 1, d.d, d.H, d.M, d.S, Math.round(d.u * 1000))) + + expect(isoDateText(new Date(Date.UTC(2024, 2, 15, 13, 44, 59, 999)), 'yyyy-mm-dd h:mm')).toBe( + '2024-03-15T13:45:00' + ) + expect(isoDateText(toDate(datetime), 'yyyy-mm-dd h:mm')).toBe('2024-03-15T13:45:00') + expect(isoDateText(toDate(time), 'h:mm')).toBe('14:30:00') + expect(isoDateText(new Date(Date.UTC(2024, 2, 15, 23, 59, 59, 700)), 'yyyy-mm-dd')).toBe( + '2024-03-16' + ) + }) + + it('renders a formatless date before 1900 as a time of day', () => { + expect(isoDateText(new Date(Date.UTC(1899, 11, 30, 0, 30, 0)))).toBe('00:30:00') + expect(isoDateText(new Date(Date.UTC(1899, 11, 31, 13, 5, 9)))).toBe('13:05:09') + }) + + it('decides time of day from the format whatever the epoch date', () => { + expect(isoDateText(new Date(Date.UTC(1904, 0, 1, 12, 29, 59)), 'h:mm:ss')).toBe('12:29:59') + expect(isoDateText(new Date(Date.UTC(1899, 11, 31, 12, 0, 0)), 'yyyy-mm-dd')).toBe( + '1899-12-31T12:00:00' + ) + }) +}) + +describe('isTimeOnlyFormat', () => { + it.each(['h:mm:ss', 'hh:mm AM/PM', 'h:mm', 'mm:ss', '[$-409]h:mm:ss', 'hh"h"mm'])( + 'treats %s as time only', + (format) => { + expect(isTimeOnlyFormat(format)).toBe(true) + } + ) + + it.each(['m/d/yyyy', 'yyyy-mm-dd hh:mm', 'mmm', 'd-mmm', 'mmmm yyyy', 'General'])( + 'treats %s as a date', + (format) => { + expect(isTimeOnlyFormat(format)).toBe(false) + } + ) +}) + +describe('generalNumberText', () => { + it('keeps integers exact and rounds fractions to 15 significant digits', () => { + expect(generalNumberText(4111111111111111)).toBe('4111111111111111') + expect(generalNumberText(Number.MAX_SAFE_INTEGER)).toBe('9007199254740991') + expect(generalNumberText(0.1 + 0.2)).toBe('0.3') + expect(generalNumberText(1063.8425)).toBe('1063.8425') + expect(generalNumberText(1.22464679914735e-16)).toBe('1.22464679914735e-16') + }) +}) + +describe('normalizeSheetDisplayText', () => { + it('rewrites dates and General numbers on a sparse sheet and leaves formatted cells alone', () => { + const sheet = XLSX.utils.aoa_to_sheet([['a']]) + sheet.A1 = { t: 'd', v: new Date(Date.UTC(2026, 2, 4)), z: 'm/d/yyyy', w: '3/4/2026' } + sheet.B1 = { t: 'n', v: 4111111111111111, z: 'General', w: '4.11111E+15' } + sheet.C1 = { t: 'n', v: 1250, z: '"$"#,##0.00', w: '$1,250.00' } + sheet.D1 = { t: 'n', v: 9, w: '9' } + sheet['!ref'] = 'A1:D1' + + normalizeSheetDisplayText(sheet, XLSX.utils.decode_range('A1:C1'), XLSX.utils) + + expect(sheet.A1.w).toBe('2026-03-04') + expect(sheet.B1.w).toBe('4111111111111111') + expect(sheet.C1.w).toBe('$1,250.00') + expect(sheet.D1.w).toBe('9') + }) + + it('keeps the rendered duration of an elapsed-time cell', () => { + const sheet = XLSX.utils.aoa_to_sheet([['a']]) + sheet.A1 = { t: 'd', v: new Date(Date.UTC(1900, 0, 1, 6)), z: '[h]:mm', w: '30:00' } + sheet.B1 = { t: 'd', v: new Date(Date.UTC(1899, 11, 31, 12)), z: '[mm]:ss', w: '720:00' } + sheet['!ref'] = 'A1:B1' + + normalizeSheetDisplayText(sheet, XLSX.utils.decode_range('A1:B1'), XLSX.utils) + + expect(sheet.A1.w).toBe('30:00') + expect(sheet.B1.w).toBe('720:00') + }) + + it('touches only the window on a dense sheet', () => { + const sheet = XLSX.utils.aoa_to_sheet([['a']], { dense: true }) + const inside = { t: 'd', v: new Date(Date.UTC(2026, 2, 4)), w: '3/4/2026' } as XLSX.CellObject + const outside = { t: 'd', v: new Date(Date.UTC(2026, 2, 5)), w: '3/5/2026' } as XLSX.CellObject + sheet['!data'] = [[inside], [outside]] + + normalizeSheetDisplayText(sheet, XLSX.utils.decode_range('A1:A1'), XLSX.utils) + + expect(inside.w).toBe('2026-03-04') + expect(outside.w).toBe('3/5/2026') + }) +}) diff --git a/apps/sim/lib/file-parsers/sheet-display-text.ts b/apps/sim/lib/file-parsers/sheet-display-text.ts new file mode 100644 index 00000000000..e9a16134381 --- /dev/null +++ b/apps/sim/lib/file-parsers/sheet-display-text.ts @@ -0,0 +1,143 @@ +import type { CellAddress, CellObject, Range, WorkSheet } from 'xlsx' + +/** + * Read options that make a workbook's cells carry the text a user sees in + * Excel rather than the values Excel stores. + * + * `cellDates` parses date serials into `Date` objects whose UTC fields are the + * calendar fields, independent of the process time zone. `cellNF` keeps each + * cell's number format so General-formatted numbers can be told apart from + * currency, percent and date cells. + */ +export const SHEET_DISPLAY_READ_OPTIONS = { + cellDates: true, + cellNF: true, +} as const + +interface CellLookup { + encode_cell: (address: CellAddress) => string +} + +/** Excel shows 15 significant digits for a General-formatted number. */ +const GENERAL_SIGNIFICANT_DIGITS = 15 + +const ELAPSED_TOKEN = /\[(h+|m+|s+)\]/i + +/** + * Strips the parts of a number format that carry no date tokens: quoted + * literals, backslash escapes, bracketed colour/condition/elapsed sections and + * the AM/PM markers whose `m` is not a month. + */ +function dateTokensOf(format: string): string { + return format + .replace(/"[^"]*"/g, '') + .replace(/\\./g, '') + .replace(/\[[^\]]*\]/g, '') + .replace(/am\/pm|a\/p/gi, '') + .toLowerCase() +} + +/** + * Whether a date format shows a time of day and nothing else, such as + * `h:mm:ss` or `hh:mm AM/PM`. Any `y` or `d` token is a date, and so is an `m` + * run that is not next to hours or seconds, which is how Excel tells a month + * from minutes. + */ +export function isTimeOnlyFormat(format: string): boolean { + const tokens = dateTokensOf(format) + if (/[yd]/.test(tokens)) return false + if (!/[hms]/.test(tokens)) return false + for (const match of tokens.matchAll(/m+/g)) { + const before = tokens.slice(0, match.index).replace(/[:\s]+$/, '') + const after = tokens.slice(match.index + match[0].length).replace(/^[:\s]+/, '') + const isMinutes = before.endsWith('h') || after.startsWith('s') + if (!isMinutes) return false + } + return true +} + +const MS_PER_SECOND = 1000 + +/** + * Excel dates carry no zone. Emit the UTC fields SheetJS parsed the serial + * into, without a trailing `Z`, and drop the time when it is midnight. + * + * A float serial such as `45366.572916666664` parses to `13:44:59.999`, so + * the instant is rounded to the nearest second first; a value that rounds up + * to midnight is a whole date. + * + * A time-of-day cell is decided from its format, because the epoch date its + * serial lands on differs between 1900 and 1904 workbooks. Without a format, + * a date before 1900 can only be a fraction of a day and is shown as a time. + */ +export function isoDateText(parsed: Date, format?: string): string { + if (Number.isNaN(parsed.getTime())) return '' + const date = new Date(Math.round(parsed.getTime() / MS_PER_SECOND) * MS_PER_SECOND) + const iso = date.toISOString() + const timeOnly = format === undefined ? date.getUTCFullYear() < 1900 : isTimeOnlyFormat(format) + if (timeOnly) return iso.slice(11, 19) + return iso.endsWith('T00:00:00.000Z') ? iso.slice(0, 10) : iso.slice(0, 19) +} + +/** + * Renders a General-formatted number the way Excel displays it: integers in + * full, so 16-digit identifiers keep every digit, and fractions rounded to 15 + * significant digits, so `=0.1+0.2` reads `0.3`. + */ +export function generalNumberText(value: number): string { + if (Number.isInteger(value)) return String(value) + return String(Number(value.toPrecision(GENERAL_SIGNIFICANT_DIGITS))) +} + +function isGeneralFormat(format: unknown): boolean { + return format === undefined || format === 'General' +} + +/** + * Rewrites the display text of the cells that `sheet_to_json({ raw: false })` + * would otherwise render badly, within the bounded window only. + * + * `raw: false` returns `cell.w` verbatim. The file's `w` is right for currency, + * percent, boolean and text cells, but not for dates (locale-shaped, such as + * `3/4/2026`) or General-formatted numbers (Excel's 11-character rendering + * turns `4111111111111111` into `4.11111E+15`, losing digits of numeric IDs). + * Dates become ISO text and General numbers print as Excel displays them. + * + * Elapsed-time formats (`[h]:mm`, `[mm]:ss`) are durations, not moments; + * `cellDates` still parses them into a `Date`, so their `w` (`30:00`) is kept. + * + * A number with no format at all is treated as General too. Every other + * number keeps the text the file rendered for it, so a LibreOffice workbook + * indexes as LibreOffice showed it (`0,5` in a German locale). + * + * Works on dense (`!data`) and sparse (address-keyed) worksheets so the same + * pass serves the indexing parser and the Files viewer. + */ +export function normalizeSheetDisplayText( + worksheet: WorkSheet, + window: Range, + utils: CellLookup +): void { + const data = worksheet['!data'] + const lastRow = data ? Math.min(window.e.r, data.length - 1) : window.e.r + + for (let r = window.s.r; r <= lastRow; r++) { + const denseRow = data?.[r] + if (data && !denseRow) continue + + for (let c = window.s.c; c <= window.e.c; c++) { + const cell: CellObject | undefined = denseRow + ? denseRow[c] + : (worksheet[utils.encode_cell({ r, c })] as CellObject | undefined) + if (!cell) continue + + if (cell.t === 'd' && cell.v instanceof Date) { + const format = typeof cell.z === 'string' ? cell.z : undefined + if (format !== undefined && ELAPSED_TOKEN.test(format)) continue + cell.w = isoDateText(cell.v, format) + } else if (cell.t === 'n' && typeof cell.v === 'number' && isGeneralFormat(cell.z)) { + cell.w = generalNumberText(cell.v) + } + } + } +} diff --git a/apps/sim/lib/file-parsers/sniff.test.ts b/apps/sim/lib/file-parsers/sniff.test.ts new file mode 100644 index 00000000000..5e94bc89698 --- /dev/null +++ b/apps/sim/lib/file-parsers/sniff.test.ts @@ -0,0 +1,409 @@ +/** + * @vitest-environment node + */ +import JSZip from 'jszip' +import { describe, expect, it } from 'vitest' +import * as XLSX from 'xlsx' +import { parseBuffer } from '@/lib/file-parsers' +import { FileParserError } from '@/lib/file-parsers/errors' +import { reconcileParserRoute, type SniffedKind, sniffFileKind } from '@/lib/file-parsers/sniff' + +const OLE2_HEADER = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]) + +function oleBinary(): Buffer { + return Buffer.concat([OLE2_HEADER, Buffer.alloc(2048, 0)]) +} + +function pngBinary(): Buffer { + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.from(Array.from({ length: 4000 }, (_, index) => (index * 7919) % 256)), + ]) +} + +async function zipWith(entries: Record, storedMimetype?: string): Promise { + const zip = new JSZip() + if (storedMimetype) zip.file('mimetype', storedMimetype, { compression: 'STORE' }) + for (const [name, content] of Object.entries(entries)) zip.file(name, content) + return zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }) as Promise +} + +function buildDocx(text: string): Promise { + return zipWith({ + '[Content_Types].xml': + '', + '_rels/.rels': + '', + 'word/document.xml': `${text}`, + }) +} + +describe('sniffFileKind', () => { + it('recognizes a PDF by its header at the start, after optional BOM or whitespace', () => { + expect(sniffFileKind(Buffer.from('%PDF-1.7\n%\xe2\xe3\xcf\xd3\n'))).toBe('pdf') + expect(sniffFileKind(Buffer.from('\n %PDF-1.4'))).toBe('pdf') + expect( + sniffFileKind(Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('%PDF-1.4')])) + ).toBe('pdf') + }) + + /** + * Some PDFs carry junk before the header, which pdf.js tolerates, so a declared + * `.pdf` keeps the 1 KiB search window. Under any other extension the signature + * must be at the start: a `.txt` that merely mentions "%PDF-1.4" is text. + */ + it('searches the first KiB for the PDF header only under a declared .pdf extension', () => { + const junkThenPdf = Buffer.concat([Buffer.alloc(200, 0x41), Buffer.from('%PDF-1.4')]) + const mention = Buffer.from( + 'The file starts with the magic string %PDF-1.4 followed by objects.' + ) + + expect(sniffFileKind(junkThenPdf, 'pdf')).toBe('pdf') + expect(sniffFileKind(junkThenPdf)).toBe('text') + expect(sniffFileKind(junkThenPdf, 'txt')).toBe('text') + expect(sniffFileKind(mention, 'txt')).toBe('text') + expect(sniffFileKind(mention, 'pdf')).toBe('pdf') + expect( + sniffFileKind(Buffer.concat([Buffer.alloc(2000, 0x41), Buffer.from('%PDF-1.4')]), 'pdf') + ).toBe('text') + }) + + it('recognizes an OLE2 compound file', () => { + expect(sniffFileKind(oleBinary())).toBe('ole2') + }) + + it('classifies Office packages by their central-directory part names', async () => { + expect(sniffFileKind(await zipWith({ 'word/document.xml': '' }))).toBe('docx') + expect( + sniffFileKind(await zipWith({ '[Content_Types].xml': '', 'xl/workbook.xml': '' })) + ).toBe('xlsx') + expect(sniffFileKind(await zipWith({ 'ppt/presentation.xml': '

      ' }))).toBe('pptx') + }) + + it('classifies OpenDocument packages by the stored mimetype entry', async () => { + expect( + sniffFileKind( + await zipWith({ 'content.xml': '' }, 'application/vnd.oasis.opendocument.text') + ) + ).toBe('odt') + expect( + sniffFileKind( + await zipWith({ 'content.xml': '' }, 'application/vnd.oasis.opendocument.spreadsheet') + ) + ).toBe('ods') + expect( + sniffFileKind( + await zipWith({ 'content.xml': '' }, 'application/vnd.oasis.opendocument.presentation') + ) + ).toBe('odp') + }) + + it('classifies SheetJS-written workbooks the way the spreadsheet parser expects', () => { + const wb = XLSX.utils.book_new() + XLSX.utils.book_append_sheet(wb, XLSX.utils.aoa_to_sheet([['a'], ['b']]), 'S') + + expect(sniffFileKind(XLSX.write(wb, { type: 'buffer', bookType: 'xlsx' }) as Buffer)).toBe( + 'xlsx' + ) + expect(sniffFileKind(XLSX.write(wb, { type: 'buffer', bookType: 'xlsb' }) as Buffer)).toBe( + 'xlsx' + ) + expect(sniffFileKind(XLSX.write(wb, { type: 'buffer', bookType: 'ods' }) as Buffer)).toBe('ods') + expect(sniffFileKind(XLSX.write(wb, { type: 'buffer', bookType: 'xls' }) as Buffer)).toBe( + 'ole2' + ) + }) + + it('reports an unrecognized archive as zip', async () => { + expect(sniffFileKind(await zipWith({ 'readme.txt': 'hi' }))).toBe('zip') + }) + + it('reports NUL-bearing bytes without a UTF-16 layout as binary', () => { + expect(sniffFileKind(pngBinary())).toBe('binary') + expect(sniffFileKind(Buffer.from('abc\0def'))).toBe('binary') + }) + + it('treats UTF-16 text as text, with or without a BOM', () => { + expect( + sniffFileKind(Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from('Hello', 'utf16le')])) + ).toBe('text') + expect(sniffFileKind(Buffer.from('Hello UTF-16 without a BOM', 'utf16le'))).toBe('text') + }) + + it('recognizes an HTML document by its opening tag after optional BOM and whitespace', () => { + expect(sniffFileKind(Buffer.from('x'))).toBe('html') + expect(sniffFileKind(Buffer.from('\n

      x

      '))).toBe('html') + expect( + sniffFileKind( + Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('

      x

      ')]) + ) + ).toBe('html') + expect(sniffFileKind(Buffer.from('

      fragment, not a document

      '))).toBe('text') + }) + + it('recognizes RTF by its opening group', () => { + expect(sniffFileKind(Buffer.from('{\\rtf1\\ansi\\deff0 {\\fonttbl} Hello}'))).toBe('rtf') + expect(sniffFileKind(Buffer.from(' {\\rtf1 not at offset zero}'))).toBe('text') + }) + + it('reports plain text and Latin-1 text as text', () => { + expect(sniffFileKind(Buffer.from('Vendor list\nBloomberg\n'))).toBe('text') + expect(sniffFileKind(Buffer.from('Caf\xe9 r\xe9sum\xe9', 'latin1'))).toBe('text') + }) +}) + +describe('reconcileParserRoute', () => { + it.each<[string, SniffedKind]>([ + ['pdf', 'pdf'], + ['docx', 'docx'], + ['docm', 'docx'], + ['xlsx', 'xlsx'], + ['xls', 'ole2'], + ['xlsx', 'ole2'], + ['ods', 'ods'], + ['pptx', 'pptx'], + ['odt', 'odt'], + ['odp', 'odp'], + ['doc', 'ole2'], + ['txt', 'text'], + ['csv', 'text'], + ['html', 'html'], + ['html', 'text'], + ['md', 'text'], + ])('keeps the .%s route when the bytes are %s', (extension, kind) => { + expect(reconcileParserRoute(extension, kind)).toEqual({ extension }) + }) + + it.each<[string, SniffedKind, string]>([ + ['xlsx', 'text', 'csv'], + ['xls', 'text', 'csv'], + ['txt', 'html', 'html'], + ['md', 'html', 'html'], + ['docx', 'pdf', 'pdf'], + ['txt', 'pdf', 'pdf'], + ['xlsx', 'docx', 'docx'], + ['doc', 'docx', 'docx'], + ['pdf', 'docx', 'docx'], + ['docx', 'xlsx', 'xlsx'], + ['docx', 'pptx', 'pptx'], + ['docx', 'odt', 'odt'], + ['odt', 'ods', 'ods'], + ['txt', 'odp', 'odp'], + ['docx', 'ole2', 'doc'], + ['doc', 'text', 'txt'], + ['docx', 'text', 'txt'], + ['pptx', 'text', 'txt'], + ['pdf', 'text', 'txt'], + ['odt', 'text', 'txt'], + ])('re-routes .%s holding %s to the %s parser with a warning', (extension, kind, route) => { + expect(reconcileParserRoute(extension, kind)).toEqual({ + extension: route, + detectedType: kind, + warning: expect.stringContaining(`parsed as .${route} instead of .${extension}`), + }) + }) + + /** An HTML error page saved as structured data is an error, not a document. */ + it.each(['csv', 'json', 'jsonl', 'yaml', 'yml'])( + 'rejects an HTML document under .%s as invalid_format', + (extension) => { + expect(() => reconcileParserRoute(extension, 'html')).toThrow( + expect.objectContaining({ code: 'invalid_format' }) + ) + } + ) + + it.each(['doc', 'docx', 'txt', 'pdf', 'xlsx', 'unknown'])( + 'rejects RTF under .%s as unsupported_type', + (extension) => { + expect(() => reconcileParserRoute(extension, 'rtf')).toThrow( + expect.objectContaining({ + code: 'unsupported_type', + message: expect.stringContaining('RTF'), + }) + ) + } + ) + + /** + * A NUL byte in a text file is not proof of a container: the decoder handles + * UTF-16 and Windows-1252 and the sanitizer strips stray NULs, so the declared + * text route is kept rather than refusing the file. + */ + it.each(['txt', 'csv', 'md', 'json'])( + 'keeps the .%s route for NUL-bearing text bytes', + (extension) => { + expect(reconcileParserRoute(extension, 'binary')).toEqual({ extension }) + } + ) + + it.each<[string, SniffedKind]>([ + ['csv', 'zip'], + ['txt', 'ole2'], + ['docx', 'binary'], + ['pdf', 'binary'], + ['pdf', 'ole2'], + ['odt', 'ole2'], + ])('rejects .%s holding %s as invalid_format', (extension, kind) => { + const error = (() => { + try { + reconcileParserRoute(extension, kind) + return null + } catch (caught) { + return caught + } + })() + + expect(error).toBeInstanceOf(FileParserError) + expect(error).toMatchObject({ code: 'invalid_format' }) + }) + + it('rejects a legacy OLE binary under a PowerPoint extension as unsupported_type', () => { + expect(() => reconcileParserRoute('pptx', 'ole2')).toThrow( + expect.objectContaining({ code: 'unsupported_type' }) + ) + }) + + it('keeps an unrecognised archive on a spreadsheet or Word route for the parser to judge', () => { + expect(reconcileParserRoute('xlsx', 'zip')).toEqual({ extension: 'xlsx' }) + expect(reconcileParserRoute('docx', 'zip')).toEqual({ extension: 'docx' }) + expect(() => reconcileParserRoute('txt', 'zip')).toThrow( + expect.objectContaining({ code: 'invalid_format' }) + ) + }) + + it('keeps an unknown binary layout on the SheetJS and legacy Word routes', () => { + expect(reconcileParserRoute('xls', 'binary')).toEqual({ extension: 'xls' }) + expect(reconcileParserRoute('doc', 'binary')).toEqual({ extension: 'doc' }) + expect(() => reconcileParserRoute('docx', 'binary')).toThrow( + expect.objectContaining({ code: 'invalid_format' }) + ) + }) + + it('leaves an extension with no known family alone', () => { + expect(reconcileParserRoute('unknown', 'binary')).toEqual({ extension: 'unknown' }) + }) +}) + +describe('parseBuffer reconciles the extension with the sniffed bytes', () => { + it('parses CSV bytes labelled .xlsx as CSV and keeps their UTF-8 intact', async () => { + const result = await parseBuffer(Buffer.from('name,city\nAna,Araújo\n'), 'xlsx') + + expect(result.content).toContain('Araújo') + expect(result.content).not.toContain('Ã') + expect(result.metadata).toMatchObject({ + detectedType: 'text', + warning: expect.stringContaining('parsed as .csv instead of .xlsx'), + }) + }) + + it('strips markup from an HTML document labelled .txt', async () => { + const result = await parseBuffer( + Buffer.from('

      Memo

      Body text

      '), + 'txt' + ) + + expect(result.content).toContain('Body text') + expect(result.content.toLowerCase()).not.toContain(' { + const result = await parseBuffer(await buildDocx('Office Relocation'), 'xlsx') + + expect(result.content).toContain('Office Relocation') + expect(result.metadata?.detectedType).toBe('docx') + }) + + it('extracts a docx labelled .doc through the Word parser without degrading', async () => { + const result = await parseBuffer(await buildDocx('Office Relocation'), 'doc') + + expect(result.content).toContain('Office Relocation') + expect(result.metadata?.degraded).toBeFalsy() + }) + + it('keeps plain text labelled .docx as text with a warning', async () => { + const result = await parseBuffer(Buffer.from('Vendor list\nBloomberg\n'), 'docx') + + expect(result.content).toContain('Bloomberg') + expect(result.metadata?.warning).toContain('parsed as .txt instead of .docx') + }) + + it('rejects a PNG labelled .doc with a typed error instead of placeholder prose', async () => { + await expect(parseBuffer(pngBinary(), 'doc')).rejects.toMatchObject({ + name: 'FileParserError', + code: 'invalid_format', + }) + }) + + it('rejects RTF bytes under .doc and .docx instead of indexing control words', async () => { + const rtf = Buffer.from( + '{\\rtf1\\ansi{\\fonttbl\\f0\\fswiss Helvetica;}\\f0\\pard Hello, world.\\par}' + ) + + for (const extension of ['doc', 'docx']) { + await expect(parseBuffer(rtf, extension)).rejects.toMatchObject({ + name: 'FileParserError', + code: 'unsupported_type', + }) + } + }) + + it('rejects an HTML error page saved as .json', async () => { + await expect( + parseBuffer(Buffer.from('403 Forbidden'), 'json') + ).rejects.toMatchObject({ code: 'invalid_format' }) + }) + + it('decodes a .csv containing a stray NUL byte instead of refusing it', async () => { + const result = await parseBuffer(Buffer.from('name,city\nAna,Lisboa\x00\n'), 'csv') + + expect(result.content).toContain('Lisboa') + expect(result.metadata?.detectedType).toBeUndefined() + }) + + it('keeps a .txt that mentions the PDF magic string as text', async () => { + const result = await parseBuffer( + Buffer.from('Every PDF begins with %PDF-1.4 or similar.'), + 'txt' + ) + + expect(result.content).toContain('Every PDF begins with') + expect(result.metadata?.detectedType).toBeUndefined() + }) + + it('rejects an OLE binary labelled .txt', async () => { + await expect(parseBuffer(oleBinary(), 'txt')).rejects.toMatchObject({ code: 'invalid_format' }) + }) + + it('rejects a legacy OLE deck labelled .pptx as unsupported', async () => { + await expect(parseBuffer(oleBinary(), 'pptx')).rejects.toMatchObject({ + code: 'unsupported_type', + }) + }) + + it('refuses the .ppt extension before sniffing', async () => { + await expect(parseBuffer(oleBinary(), 'ppt')).rejects.toMatchObject({ + code: 'unsupported_type', + }) + }) + + it('surfaces a truncated OOXML archive as a typed invalid_format failure', async () => { + const truncated = (await buildDocx('Office Relocation')).subarray(0, 200) + + await expect(parseBuffer(truncated, 'docx')).rejects.toMatchObject({ + name: 'FileParserError', + code: 'invalid_format', + }) + }) + + it('decodes a Latin-1 text file and reports the encoding', async () => { + const result = await parseBuffer( + Buffer.from('Caf\xe9 r\xe9sum\xe9 na\xefve \xa3 42', 'latin1'), + 'txt' + ) + + expect(result.content).toBe('Café résumé naïve £ 42') + expect(result.metadata).toMatchObject({ encoding: 'windows-1252', characterCount: 22 }) + }) +}) diff --git a/apps/sim/lib/file-parsers/sniff.ts b/apps/sim/lib/file-parsers/sniff.ts new file mode 100644 index 00000000000..365b4c4b66f --- /dev/null +++ b/apps/sim/lib/file-parsers/sniff.ts @@ -0,0 +1,379 @@ +import { FileParserError } from '@/lib/file-parsers/errors' +import { isEncryptedOoxmlContainer } from '@/lib/file-parsers/ooxml-encryption' +import { decodeTextBuffer, detectBomlessUtf16 } from '@/lib/file-parsers/utils' +import { isZipShaped } from '@/lib/file-parsers/zip-guard' + +/** + * What the bytes of a buffer look like, independent of the caller-supplied + * extension. `zip` is a ZIP archive that is none of the recognized Office + * containers; `ole2` is any OLE compound file (legacy `.doc`/`.xls`/`.ppt`); + * `encrypted-ooxml` is an OLE compound file wrapping a password-protected + * `.docx`/`.xlsx`/`.pptx` package. + */ +export type SniffedKind = + | 'pdf' + | 'docx' + | 'xlsx' + | 'pptx' + | 'odt' + | 'ods' + | 'odp' + | 'zip' + | 'ole2' + | 'rtf' + | 'encrypted-ooxml' + | 'html' + | 'text' + | 'binary' + +const PDF_HEAD_WINDOW = 1024 +const TEXT_HEAD_WINDOW = 4096 +const PDF_SIGNATURE = Buffer.from('%PDF-', 'latin1') +const OLE2_SIGNATURE = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]) +/** An RTF file is a group opening with the `rtf` control word; nothing may precede it. */ +const RTF_SIGNATURE = Buffer.from('{\\rtf', 'latin1') + +const EOCD_SIGNATURE = 0x06054b50 +const EOCD_MIN_SIZE = 22 +const MAX_EOCD_COMMENT_SIZE = 0xffff +const ZIP64_EOCD_LOCATOR_SIGNATURE = 0x07064b50 +const ZIP64_EOCD_LOCATOR_SIZE = 20 +const ZIP64_EOCD_SIGNATURE = 0x06064b50 +const CENTRAL_DIRECTORY_HEADER_SIGNATURE = 0x02014b50 +const CENTRAL_DIRECTORY_HEADER_MIN_SIZE = 46 +const LOCAL_FILE_HEADER_MIN_SIZE = 30 +const COMPRESSION_METHOD_STORED = 0 +const UINT16_SENTINEL = 0xffff +const UINT32_SENTINEL = 0xffffffff +/** Enough to reach the first `word/`, `xl/` or `ppt/` part in any real package. */ +const MAX_INSPECTED_ENTRIES = 256 +const MAX_MIMETYPE_BYTES = 128 + +const ODF_MIMETYPES: Record = { + 'application/vnd.oasis.opendocument.text': 'odt', + 'application/vnd.oasis.opendocument.spreadsheet': 'ods', + 'application/vnd.oasis.opendocument.presentation': 'odp', +} + +interface ZipEntry { + name: string + compressionMethod: number + compressedSize: number + localHeaderOffset: number +} + +/** Same EOCD anchoring as the zip guard: only a record whose comment ends the buffer counts. */ +function findEocdOffset(buffer: Buffer): number { + const minStart = Math.max(0, buffer.length - EOCD_MIN_SIZE - MAX_EOCD_COMMENT_SIZE) + for (let offset = buffer.length - EOCD_MIN_SIZE; offset >= minStart; offset--) { + if (buffer.readUInt32LE(offset) !== EOCD_SIGNATURE) continue + const commentLength = buffer.readUInt16LE(offset + 20) + if (offset + EOCD_MIN_SIZE + commentLength === buffer.length) return offset + } + return -1 +} + +function locateCentralDirectory(buffer: Buffer, eocdOffset: number): number | null { + const entryCount = buffer.readUInt16LE(eocdOffset + 10) + const directoryOffset = buffer.readUInt32LE(eocdOffset + 16) + if (entryCount !== UINT16_SENTINEL && directoryOffset !== UINT32_SENTINEL) { + return directoryOffset + } + + const locatorOffset = eocdOffset - ZIP64_EOCD_LOCATOR_SIZE + if (locatorOffset < 0 || buffer.readUInt32LE(locatorOffset) !== ZIP64_EOCD_LOCATOR_SIGNATURE) { + return null + } + const zip64Eocd = buffer.readBigUInt64LE(locatorOffset + 8) + if (zip64Eocd > BigInt(buffer.length - 56)) return null + const zip64EocdOffset = Number(zip64Eocd) + if (buffer.readUInt32LE(zip64EocdOffset) !== ZIP64_EOCD_SIGNATURE) return null + const zip64DirectoryOffset = buffer.readBigUInt64LE(zip64EocdOffset + 48) + if (zip64DirectoryOffset > BigInt(buffer.length)) return null + return Number(zip64DirectoryOffset) +} + +/** + * Reads central-directory entry names without decompressing anything. Returns + * `null` for a buffer whose directory cannot be located. Bounded to the first + * {@link MAX_INSPECTED_ENTRIES} records so a large archive costs no more than a + * small one. + */ +function readZipEntries(buffer: Buffer): ZipEntry[] | null { + if (buffer.length < EOCD_MIN_SIZE) return null + const eocdOffset = findEocdOffset(buffer) + if (eocdOffset < 0) return null + const directoryOffset = locateCentralDirectory(buffer, eocdOffset) + if (directoryOffset === null) return null + + const entries: ZipEntry[] = [] + let cursor = directoryOffset + while ( + entries.length < MAX_INSPECTED_ENTRIES && + cursor + CENTRAL_DIRECTORY_HEADER_MIN_SIZE <= buffer.length && + buffer.readUInt32LE(cursor) === CENTRAL_DIRECTORY_HEADER_SIGNATURE + ) { + const fileNameLength = buffer.readUInt16LE(cursor + 28) + const extraFieldLength = buffer.readUInt16LE(cursor + 30) + const commentLength = buffer.readUInt16LE(cursor + 32) + const nameStart = cursor + CENTRAL_DIRECTORY_HEADER_MIN_SIZE + if (nameStart + fileNameLength > buffer.length) break + entries.push({ + name: buffer.toString('utf8', nameStart, nameStart + fileNameLength), + compressionMethod: buffer.readUInt16LE(cursor + 10), + compressedSize: buffer.readUInt32LE(cursor + 20), + localHeaderOffset: buffer.readUInt32LE(cursor + 42), + }) + cursor = nameStart + fileNameLength + extraFieldLength + commentLength + } + return entries +} + +/** The stored `mimetype` entry's bytes, which OpenDocument requires to be uncompressed. */ +function readStoredEntry(buffer: Buffer, entry: ZipEntry, maxBytes: number): string | null { + if (entry.compressionMethod !== COMPRESSION_METHOD_STORED || entry.compressedSize > maxBytes) { + return null + } + const headerOffset = entry.localHeaderOffset + if (headerOffset + LOCAL_FILE_HEADER_MIN_SIZE > buffer.length) return null + const fileNameLength = buffer.readUInt16LE(headerOffset + 26) + const extraFieldLength = buffer.readUInt16LE(headerOffset + 28) + const dataStart = headerOffset + LOCAL_FILE_HEADER_MIN_SIZE + fileNameLength + extraFieldLength + const dataEnd = dataStart + entry.compressedSize + if (dataEnd > buffer.length) return null + return buffer.toString('latin1', dataStart, dataEnd).trim() +} + +function classifyZip(buffer: Buffer): SniffedKind { + const entries = readZipEntries(buffer) + if (!entries) return 'zip' + + const mimetypeEntry = entries.find((entry) => entry.name === 'mimetype') + if (mimetypeEntry) { + const mimetype = readStoredEntry(buffer, mimetypeEntry, MAX_MIMETYPE_BYTES) + if (mimetype && mimetype in ODF_MIMETYPES) return ODF_MIMETYPES[mimetype] + } + + for (const { name } of entries) { + if (name.startsWith('word/')) return 'docx' + if (name.startsWith('xl/')) return 'xlsx' + if (name.startsWith('ppt/')) return 'pptx' + } + return 'zip' +} + +function hasUtf16Bom(head: Buffer): boolean { + return ( + head.length >= 2 && + ((head[0] === 0xff && head[1] === 0xfe) || (head[0] === 0xfe && head[1] === 0xff)) + ) +} + +function sniffTextKind(buffer: Buffer): SniffedKind { + const head = buffer.subarray(0, TEXT_HEAD_WINDOW) + if (head.includes(0) && !hasUtf16Bom(head) && detectBomlessUtf16(head) === null) { + return 'binary' + } + + const leading = decodeTextBuffer(head).text.trimStart().slice(0, 16).toLowerCase() + if (leading.startsWith('= OLE2_SIGNATURE.length && buffer.subarray(0, 8).equals(OLE2_SIGNATURE)) { + return isEncryptedOoxmlContainer(buffer) ? 'encrypted-ooxml' : 'ole2' + } + if (isZipShaped(buffer)) return classifyZip(buffer) + if (buffer.subarray(0, RTF_SIGNATURE.length).equals(RTF_SIGNATURE)) return 'rtf' + return sniffTextKind(buffer) +} + +/** The container family an extension promises, so a mismatch can be reconciled. */ +export type ExtensionFamily = + | 'pdf' + | 'word' + | 'sheet' + | 'presentation' + | 'opendocument' + | 'ole' + | 'text' + +const EXTENSION_FAMILIES: Record = { + pdf: 'pdf', + docx: 'word', + docm: 'word', + dotx: 'word', + xlsx: 'sheet', + xls: 'sheet', + xlsm: 'sheet', + xlsb: 'sheet', + xltx: 'sheet', + ods: 'sheet', + pptx: 'presentation', + pptm: 'presentation', + potx: 'presentation', + odt: 'opendocument', + odp: 'opendocument', + doc: 'ole', + txt: 'text', + md: 'text', + csv: 'text', + json: 'text', + jsonl: 'text', + yaml: 'text', + yml: 'text', + html: 'text', + htm: 'text', +} + +/** Sniffed kinds that are exactly what each family's parsers read. */ +const FAMILY_ACCEPTS: Record> = { + pdf: new Set(['pdf']), + word: new Set(['docx']), + sheet: new Set(['xlsx', 'ods', 'ole2']), + presentation: new Set(['pptx']), + opendocument: new Set(['odt', 'odp']), + ole: new Set(['ole2']), + text: new Set(['text', 'html']), +} + +/** Sniffed kinds that name their own parser regardless of the extension. */ +const KIND_ROUTES: Partial> = { + pdf: 'pdf', + docx: 'docx', + xlsx: 'xlsx', + pptx: 'pptx', + odt: 'odt', + ods: 'ods', + odp: 'odp', + html: 'html', +} + +export interface ParserRoute { + /** Registry key to parse with — the extension itself when the bytes agree with it. */ + extension: string + /** Set only when the route differs from the extension. */ + detectedType?: SniffedKind + warning?: string +} + +function invalidFormat(extension: string, kind: SniffedKind): FileParserError { + return new FileParserError( + 'invalid_format', + `File content does not match the .${extension} extension (detected ${kind}). Re-save it in a supported format and retry.` + ) +} + +/** + * Reconciles the caller-supplied extension with what the bytes are. Magic wins + * over the name, as in Tika and unstructured: when the sniffed kind has its own + * parser the route is overridden and a warning recorded; when it has none and + * the family disagrees, the buffer is rejected as `invalid_format` rather than + * fed to a parser that would emit mojibake or placeholder prose. + * + * Plain text under a binary extension keeps today's behavior of parsing as + * text (as CSV under a spreadsheet extension), and an OLE2 file under a modern + * Word extension is the legacy `.doc` parser's job. Legacy `.ppt` has no reader. + */ +export function reconcileParserRoute(extension: string, kind: SniffedKind): ParserRoute { + if (kind === 'rtf') { + throw new FileParserError( + 'unsupported_type', + 'RTF is not supported. Save the file as .docx and retry.' + ) + } + + const family = EXTENSION_FAMILIES[extension] + if (!family) return { extension } + + const override = (route: string): ParserRoute => ({ + extension: route, + detectedType: kind, + warning: `File content was detected as ${kind}; parsed as .${route} instead of .${extension}`, + }) + + /** + * Only `.txt` and `.md` may hold a whole HTML document — a `.md` that starts + * with `` is deliberately treated as HTML, since Markdown allows + * raw HTML and the parser strips the markup either way. Under the structured + * text extensions an HTML document is an error page saved as data. + */ + if (kind === 'html' && family === 'text' && extension !== 'html' && extension !== 'htm') { + if (extension === 'txt' || extension === 'md') return override('html') + throw invalidFormat(extension, kind) + } + if (FAMILY_ACCEPTS[family].has(kind)) return { extension } + + /** + * Ambiguous bytes stay on the declared route. An archive without a recognised + * layout may still be a workbook SheetJS reads (`xl/` is a convention, not a + * rule), and an unknown binary layout under a spreadsheet or legacy Word + * extension covers raw BIFF streams and other formats those parsers accept. + * Each of those parsers raises its own typed error when the bytes are not a + * document, so passing them through never yields scraped garbage. + */ + if (kind === 'zip' && family !== 'pdf' && family !== 'text') return { extension } + /** + * A NUL byte in a declared text file is not proof of a container either: the + * decoder recognises UTF-16 and Windows-1252 and the sanitizer strips stray + * NULs, so the text route is kept. Recognised containers under a text + * extension are still refused below. + */ + if (kind === 'binary' && (family === 'sheet' || family === 'ole' || family === 'text')) { + return { extension } + } + + if (kind === 'text') return override(family === 'sheet' ? 'csv' : 'txt') + if (kind === 'encrypted-ooxml') { + throw new FileParserError( + 'encrypted_file', + 'This document is encrypted or password-protected. Remove the password and retry.' + ) + } + if (kind === 'ole2') { + if (family === 'word') return override('doc') + if (family === 'presentation') { + throw new FileParserError( + 'unsupported_type', + 'Legacy binary PowerPoint (.ppt) files are not supported. Save the file as .pptx and retry.' + ) + } + throw invalidFormat(extension, kind) + } + + const route = KIND_ROUTES[kind] + if (route) return override(route) + throw invalidFormat(extension, kind) +} diff --git a/apps/sim/lib/file-parsers/txt-parser.ts b/apps/sim/lib/file-parsers/txt-parser.ts index 3bb9e377859..ff2996202aa 100644 --- a/apps/sim/lib/file-parsers/txt-parser.ts +++ b/apps/sim/lib/file-parsers/txt-parser.ts @@ -1,7 +1,7 @@ import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' import type { FileParseResult, FileParser } from '@/lib/file-parsers/types' -import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' +import { decodeTextBuffer, sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' const logger = createLogger('TxtParser') @@ -25,14 +25,16 @@ export class TxtParser implements FileParser { try { logger.info('Parsing buffer, size:', buffer.length) - const rawContent = buffer.toString('utf-8') - const result = sanitizeTextForUTF8(rawContent) + const decoded = decodeTextBuffer(buffer) + const result = sanitizeTextForUTF8(decoded.text) return { content: result, metadata: { characterCount: result.length, tokenCount: result.length / 4, + encoding: decoded.encoding, + ...(decoded.warning ? { warning: decoded.warning } : {}), }, } } catch (error) { diff --git a/apps/sim/lib/file-parsers/types.ts b/apps/sim/lib/file-parsers/types.ts index 834f2fc4632..36b059a3bc7 100644 --- a/apps/sim/lib/file-parsers/types.ts +++ b/apps/sim/lib/file-parsers/types.ts @@ -7,16 +7,16 @@ export interface FileParseMetadata { * True when no real extraction happened and `content` is best-effort scraped * bytes or a placeholder message rather than the document's text. * - * The legacy-format parsers (`doc`, `ppt`) deliberately never throw, so an - * interactive upload still shows the user something. An automated caller must - * not index that: it embeds ZIP internals or an English placeholder sentence as - * if it were document content. Such callers check this flag and skip the file. + * Set by extractors that can only return best-effort output, such as a + * spreadsheet whose cells are all blank. Legacy `.doc` and `.ppt` inputs used + * to fall through to a byte scrape reported this way; they now raise typed + * errors instead. An automated caller must not index degraded content, and + * every automated consumer checks this flag and skips the file. */ degraded?: boolean extractionMethod?: string warning?: string messages?: unknown[] - html?: string type?: string headers?: string[] totalRows?: number @@ -59,7 +59,6 @@ export type SupportedFileType = | 'html' | 'htm' | 'pptx' - | 'ppt' | 'pptm' | 'potx' | 'odt' diff --git a/apps/sim/lib/file-parsers/utils.test.ts b/apps/sim/lib/file-parsers/utils.test.ts index b01a1ca6d51..d1ac7957549 100644 --- a/apps/sim/lib/file-parsers/utils.test.ts +++ b/apps/sim/lib/file-parsers/utils.test.ts @@ -2,7 +2,15 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { sanitizeTextForUTF8, truncationNotice } from '@/lib/file-parsers/utils' +import { + decodeTextBuffer, + decodeWindows1252, + decodeWindows1252WithTable, + sanitizeTextForUTF8, + TRUNCATED_UTF8_WARNING, + truncationNotice, + WINDOWS_1252_WARNING, +} from '@/lib/file-parsers/utils' const LONE_HIGH = '\uD800' const LONE_LOW = '\uDC00' @@ -52,3 +60,109 @@ describe('truncationNotice', () => { ) }) }) + +describe('decodeTextBuffer', () => { + it('decodes clean UTF-8 without a warning', () => { + const decoded = decodeTextBuffer(Buffer.from('Café résumé 😀', 'utf8')) + + expect(decoded).toEqual({ text: 'Café résumé 😀', encoding: 'utf-8' }) + }) + + it('strips a UTF-8 BOM so it never reaches content or character counts', () => { + const decoded = decodeTextBuffer( + Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('Café ok')]) + ) + + expect(decoded.text).toBe('Café ok') + expect(decoded.text.length).toBe(7) + expect(decoded.encoding).toBe('utf-8') + }) + + it('decodes Latin-1 bytes as Windows-1252 instead of destroying accented characters', () => { + const decoded = decodeTextBuffer(Buffer.from('Caf\xe9 r\xe9sum\xe9 na\xefve \xa3 42', 'latin1')) + + expect(decoded.text).toBe('Café résumé naïve £ 42') + expect(decoded.encoding).toBe('windows-1252') + expect(decoded.warning).toBe(WINDOWS_1252_WARNING) + expect(sanitizeTextForUTF8(decoded.text)).toBe('Café résumé naïve £ 42') + }) + + it('decodes Windows-1252 smart quotes, dashes and the euro sign', () => { + const decoded = decodeTextBuffer( + Buffer.from([0x93, 0x53, 0x6d, 0x61, 0x72, 0x74, 0x94, 0x20, 0x96, 0x20, 0x80, 0x35]) + ) + + expect(decoded.text).toBe('“Smart” – €5') + expect(decoded.encoding).toBe('windows-1252') + }) + + it('decodes UTF-16LE with a BOM, including non-ASCII characters', () => { + const decoded = decodeTextBuffer( + Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from('Hello UTF-16 wörld €', 'utf16le')]) + ) + + expect(decoded).toEqual({ text: 'Hello UTF-16 wörld €', encoding: 'utf-16le' }) + }) + + it('decodes UTF-16BE with a BOM', () => { + const decoded = decodeTextBuffer(Buffer.from([0xfe, 0xff, 0x00, 0x48, 0x00, 0x69, 0x20, 0xac])) + + expect(decoded).toEqual({ text: 'Hi€', encoding: 'utf-16be' }) + }) + + it('recognizes BOM-less UTF-16LE text instead of reading it as NUL-riddled UTF-8', () => { + const decoded = decodeTextBuffer(Buffer.from('Hello UTF-16 world without a BOM', 'utf16le')) + + expect(decoded).toEqual({ text: 'Hello UTF-16 world without a BOM', encoding: 'utf-16le' }) + }) + + it('keeps the UTF-8 reading when only a trailing codepoint was truncated', () => { + const full = Buffer.from('Truncated download résumé 😀', 'utf8') + const decoded = decodeTextBuffer(full.subarray(0, full.length - 2)) + + expect(decoded.text).toBe('Truncated download résumé ') + expect(decoded.encoding).toBe('utf-8') + expect(decoded.warning).toBe(TRUNCATED_UTF8_WARNING) + }) + + it('does not mistake a Latin-1 file ending in an accented letter for truncated UTF-8', () => { + expect(decodeTextBuffer(Buffer.from('name: Caf\xe9', 'latin1'))).toMatchObject({ + text: 'name: Café', + encoding: 'windows-1252', + }) + expect(decodeTextBuffer(Buffer.from('Caf\xe9\n', 'latin1')).text).toBe('Café\n') + }) + + it('maps every Windows-1252 byte to the WHATWG code point on both decode paths', () => { + const everyByte = new Uint8Array(Array.from({ length: 256 }, (_, index) => index)) + const expectedC1 = + '\u20AC\u0081\u201A\u0192\u201E\u2026\u2020\u2021\u02C6\u2030\u0160\u2039\u0152\u008D\u017D\u008F' + + '\u0090\u2018\u2019\u201C\u201D\u2022\u2013\u2014\u02DC\u2122\u0161\u203A\u0153\u009D\u017E\u0178' + + for (const decode of [decodeWindows1252, decodeWindows1252WithTable]) { + const decoded = decode(everyByte) + expect(decoded.length).toBe(256) + expect(decoded.slice(0, 0x80)).toBe( + String.fromCharCode(...Array.from({ length: 0x80 }, (_, index) => index)) + ) + expect(decoded.slice(0x80, 0xa0)).toBe(expectedC1) + expect(decoded.slice(0xa0)).toBe(Buffer.from(everyByte.subarray(0xa0)).toString('latin1')) + } + }) + + it('decodes a large C1-heavy buffer in one bounded pass', () => { + const heavy = new Uint8Array(4 * 1024 * 1024).fill(0x93) + + const decoded = decodeWindows1252WithTable(heavy) + + expect(decoded.length).toBe(heavy.length) + expect(decoded.charCodeAt(0)).toBe(0x201c) + expect(decoded.charCodeAt(heavy.length - 1)).toBe(0x201c) + }) + + it('never emits a replacement character for single-byte input', () => { + const everyByte = Buffer.from(Array.from({ length: 256 }, (_, index) => index)) + + expect(decodeTextBuffer(everyByte).text).not.toContain('�') + }) +}) diff --git a/apps/sim/lib/file-parsers/utils.ts b/apps/sim/lib/file-parsers/utils.ts index 02832e688dc..17be6233539 100644 --- a/apps/sim/lib/file-parsers/utils.ts +++ b/apps/sim/lib/file-parsers/utils.ts @@ -24,3 +24,227 @@ export function sanitizeTextForUTF8(text: string): string { export function truncationNotice(detail: string): string { return `\n[... ${detail} ...]\n` } + +/** Character encodings {@link decodeTextBuffer} can produce. */ +export type TextEncodingLabel = 'utf-8' | 'utf-16le' | 'utf-16be' | 'windows-1252' + +export interface DecodedText { + text: string + encoding: TextEncodingLabel + /** Set when the bytes were not clean UTF-8 and a lossy or inferred decode was used. */ + warning?: string +} + +const strictUtf8Decoder = new TextDecoder('utf-8', { fatal: true }) +const utf16leDecoder = new TextDecoder('utf-16le') +const utf16beDecoder = new TextDecoder('utf-16be') + +/** + * WHATWG windows-1252: identical to Latin-1 except 0x80–0x9F, which hold the + * typographic characters (smart quotes, dashes, €, …) instead of C1 controls. + * Implemented here rather than via `TextDecoder('windows-1252')` because a + * Node build without full ICU silently falls back to Latin-1 for that label, + * so the same bytes would decode differently under test and in production. + */ +const WINDOWS_1252_C1 = [ + '\u20AC', + '\u0081', + '\u201A', + '\u0192', + '\u201E', + '\u2026', + '\u2020', + '\u2021', + '\u02C6', + '\u2030', + '\u0160', + '\u2039', + '\u0152', + '\u008D', + '\u017D', + '\u008F', + '\u0090', + '\u2018', + '\u2019', + '\u201C', + '\u201D', + '\u2022', + '\u2013', + '\u2014', + '\u02DC', + '\u2122', + '\u0161', + '\u203A', + '\u0153', + '\u009D', + '\u017E', + '\u0178', +] as const +/** Every byte value's windows-1252 code point; all of them are BMP, so one UTF-16 code unit each. */ +const WINDOWS_1252_CODE_UNITS = Uint16Array.from({ length: 256 }, (_, byte) => + byte >= 0x80 && byte <= 0x9f ? WINDOWS_1252_C1[byte - 0x80].charCodeAt(0) : byte +) +const WINDOWS_1252_SELF_TEST_BYTES = new Uint8Array([0x80, 0x93, 0x94, 0x9f, 0xe9]) +const WINDOWS_1252_SELF_TEST_TEXT = '\u20AC\u201C\u201D\u0178\u00E9' +const HOST_IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1 + +/** + * `TextDecoder('windows-1252')` when the runtime really implements it (checked + * once here — Bun 1.3 does; a Node build without full ICU accepts the label but + * decodes as Latin-1), otherwise the table decoder below. + */ +const nativeWindows1252Decoder = (() => { + try { + const decoder = new TextDecoder('windows-1252') + return decoder.encoding === 'windows-1252' && + decoder.decode(WINDOWS_1252_SELF_TEST_BYTES) === WINDOWS_1252_SELF_TEST_TEXT + ? decoder + : null + } catch { + return null + } +})() + +/** + * One pass mapping each byte to its UTF-16 code unit, then a single native + * UTF-16 decode. Peak transient memory is the 2-byte-per-input code-unit array; + * the earlier `toString('latin1')` + regex replace materialized several string + * copies and, on 100 MB of C1 bytes, took seconds and gigabytes. + */ +export function decodeWindows1252WithTable(buffer: Uint8Array): string { + const units = new Uint16Array(buffer.length) + for (let index = 0; index < buffer.length; index++) { + units[index] = WINDOWS_1252_CODE_UNITS[buffer[index]] + } + if (HOST_IS_LITTLE_ENDIAN) return utf16leDecoder.decode(units) + return utf16beDecoder.decode(units) +} + +export function decodeWindows1252(buffer: Uint8Array): string { + return nativeWindows1252Decoder + ? nativeWindows1252Decoder.decode(buffer) + : decodeWindows1252WithTable(buffer) +} + +const UTF8_BOM_LENGTH = 3 +const UTF16_BOM_LENGTH = 2 +/** A UTF-8 sequence is at most four bytes, so a truncated tail is at most three. */ +const MAX_TRUNCATED_UTF8_TAIL = 3 +const UTF16_HEURISTIC_SAMPLE_BYTES = 4096 + +export const TRUNCATED_UTF8_WARNING = 'Trailing bytes of an incomplete UTF-8 sequence were dropped' +export const WINDOWS_1252_WARNING = + 'File was not valid UTF-8; decoded as Windows-1252; the file may use another encoding' + +function stripLeadingBom(text: string): string { + return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text +} + +/** Declared length of the UTF-8 sequence a lead byte starts, or 0 for a non-lead byte. */ +function utf8SequenceLength(lead: number): number { + if (lead >= 0xc2 && lead <= 0xdf) return 2 + if (lead >= 0xe0 && lead <= 0xef) return 3 + if (lead >= 0xf0 && lead <= 0xf4) return 4 + return 0 +} + +/** + * Whether the last `tailLength` bytes look like the cut-off start of one UTF-8 + * sequence (a lead byte followed only by continuation bytes, shorter than the + * length the lead declares) AND the bytes before it already contain multi-byte + * UTF-8. Without that second condition a Latin-1 file that merely ends in an + * accented letter would be misread as truncated UTF-8 and lose the letter. + */ +function isTruncatedUtf8Tail(buffer: Uint8Array, tailLength: number): boolean { + const tailStart = buffer.length - tailLength + const declared = utf8SequenceLength(buffer[tailStart]) + if (declared === 0 || tailLength >= declared) return false + for (let index = tailStart + 1; index < buffer.length; index++) { + if ((buffer[index] & 0xc0) !== 0x80) return false + } + for (let index = 0; index < tailStart; index++) { + if (buffer[index] >= 0x80) return true + } + return false +} + +/** + * Whether a BOM-less buffer is laid out as UTF-16 ASCII-range text: one half of + * every byte pair is NUL while the other is not. Reports the byte order of the + * non-NUL half, or `null` when the sample does not fit either layout. + */ +export function detectBomlessUtf16(buffer: Uint8Array): 'utf-16le' | 'utf-16be' | null { + const sampleLength = Math.min(buffer.length, UTF16_HEURISTIC_SAMPLE_BYTES) & ~1 + if (sampleLength < 4) return null + + let evenNul = 0 + let oddNul = 0 + for (let index = 0; index < sampleLength; index += 2) { + if (buffer[index] === 0) evenNul++ + if (buffer[index + 1] === 0) oddNul++ + } + + const pairs = sampleLength / 2 + const highThreshold = pairs * 0.9 + const lowThreshold = pairs * 0.05 + if (oddNul >= highThreshold && evenNul <= lowThreshold) return 'utf-16le' + if (evenNul >= highThreshold && oddNul <= lowThreshold) return 'utf-16be' + return null +} + +/** + * Decodes text bytes without ever emitting U+FFFD for single-byte input. + * + * Order: a UTF-16 BOM wins; otherwise strict UTF-8 (which also consumes a UTF-8 + * BOM). When strict UTF-8 rejects the buffer it is retried with up to three + * trailing bytes removed, so a size-capped download cut mid-codepoint keeps its + * UTF-8 reading instead of falling to Windows-1252 wholesale. Only then is the + * whole buffer read as Windows-1252, which is a superset of Latin-1 and decodes + * every byte, so `sanitizeTextForUTF8` has nothing to delete. Never throws. + */ +export function decodeTextBuffer(buffer: Uint8Array): DecodedText { + if (buffer.length >= UTF16_BOM_LENGTH) { + if (buffer[0] === 0xff && buffer[1] === 0xfe) { + return { + text: stripLeadingBom(utf16leDecoder.decode(buffer.subarray(UTF16_BOM_LENGTH))), + encoding: 'utf-16le', + } + } + if (buffer[0] === 0xfe && buffer[1] === 0xff) { + return { + text: stripLeadingBom(utf16beDecoder.decode(buffer.subarray(UTF16_BOM_LENGTH))), + encoding: 'utf-16be', + } + } + } + + const bomlessUtf16 = detectBomlessUtf16(buffer) + if (bomlessUtf16 === 'utf-16le') { + return { text: utf16leDecoder.decode(buffer), encoding: 'utf-16le' } + } + if (bomlessUtf16 === 'utf-16be') { + return { text: utf16beDecoder.decode(buffer), encoding: 'utf-16be' } + } + + try { + return { text: strictUtf8Decoder.decode(buffer), encoding: 'utf-8' } + } catch { + for (let dropped = 1; dropped <= MAX_TRUNCATED_UTF8_TAIL; dropped++) { + if (buffer.length - dropped < UTF8_BOM_LENGTH) break + if (!isTruncatedUtf8Tail(buffer, dropped)) continue + try { + return { + text: strictUtf8Decoder.decode(buffer.subarray(0, buffer.length - dropped)), + encoding: 'utf-8', + warning: TRUNCATED_UTF8_WARNING, + } + } catch {} + } + } + + return { + text: decodeWindows1252(buffer), + encoding: 'windows-1252', + warning: WINDOWS_1252_WARNING, + } +} diff --git a/apps/sim/lib/file-parsers/xlsx-parser.ts b/apps/sim/lib/file-parsers/xlsx-parser.ts index 0095e6df0e6..f3eef4566c7 100644 --- a/apps/sim/lib/file-parsers/xlsx-parser.ts +++ b/apps/sim/lib/file-parsers/xlsx-parser.ts @@ -8,6 +8,10 @@ import { isEncryptedOfficeParserError, toFileParserError, } from '@/lib/file-parsers/errors' +import { + normalizeSheetDisplayText, + SHEET_DISPLAY_READ_OPTIONS, +} from '@/lib/file-parsers/sheet-display-text' import type { FileParseResult, FileParser } from '@/lib/file-parsers/types' import { sanitizeTextForUTF8, truncationNotice } from '@/lib/file-parsers/utils' import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard' @@ -78,6 +82,7 @@ export class XlsxParser implements FileParser { type: 'buffer', dense: true, // Use dense mode for better memory efficiency sheetStubs: false, // Don't create stub cells + ...SHEET_DISPLAY_READ_OPTIONS, }) return this.processWorkbook(workbook) @@ -162,13 +167,27 @@ export class XlsxParser implements FileParser { */ const lastPreviewRow = Math.min(range.e.r, range.s.r + CONFIG.MAX_PREVIEW_ROWS - 1) const lastPreviewColumn = Math.min(range.e.c, range.s.c + CONFIG.MAX_PREVIEW_COLUMNS - 1) + const previewRange = { + s: { r: range.s.r, c: range.s.c }, + e: { r: lastPreviewRow, c: lastPreviewColumn }, + } + + /** + * Indexed as the text a user sees, not the value Excel stores: `raw: false` + * emits each cell's formatted text, so `$1,250.00` and `20%` survive + * instead of `1250` and `0.2` — the same currency and percent text the + * Google Sheets and Excel connectors request, so a Drive export of a sheet + * indexes its numbers the way the connectors do. Dates and General numbers + * are rewritten first because their file-formatted text is locale-shaped + * or loses digits; dates therefore index as ISO text here where the + * connectors carry the locale text. + */ + normalizeSheetDisplayText(worksheet, previewRange, XLSX.utils) const sheetData = XLSX.utils.sheet_to_json(worksheet, { header: 1, blankrows: false, // Skip blank rows - range: { - s: { r: range.s.r, c: range.s.c }, - e: { r: lastPreviewRow, c: lastPreviewColumn }, - }, + raw: false, + range: previewRange, }) // Reported from the declared range, as before, so bounding the conversion @@ -291,7 +310,11 @@ export class XlsxParser implements FileParser { return '' } - let cellStr = String(cell) + /** + * A cell is one column: a tab or line break inside it (LibreOffice writes + * rendered text with embedded newlines) would otherwise split the row. + */ + let cellStr = String(cell).replace(/[\t\r\n]+/g, ' ') /** * Samples are previews; canonical content is bounded only by the aggregate diff --git a/apps/sim/lib/file-parsers/xlsx-preview-bound.test.ts b/apps/sim/lib/file-parsers/xlsx-preview-bound.test.ts index 297a6103fb1..1e0680fd2e6 100644 --- a/apps/sim/lib/file-parsers/xlsx-preview-bound.test.ts +++ b/apps/sim/lib/file-parsers/xlsx-preview-bound.test.ts @@ -36,6 +36,7 @@ describe('XlsxParser preview bound', () => { const options = toJson.mock.calls[0][1] as { range?: { s: { r: number; c: number }; e: { r: number; c: number } } defval?: unknown + raw?: boolean } /** @@ -61,6 +62,12 @@ describe('XlsxParser preview bound', () => { * silently defeated the `blankrows: false` sitting beside it. */ expect(options.defval).toBeUndefined() + + /** + * Display text, not stored values: without `raw: false` a date indexes as + * its serial and `20%` as `0.2`, unlike the Sheets and Excel connectors. + */ + expect(options.raw).toBe(false) }) it('caps a sheet with an inflated declared column range before conversion', async () => { diff --git a/apps/sim/lib/file-parsers/yaml-parser.test.ts b/apps/sim/lib/file-parsers/yaml-parser.test.ts index 08e24623620..2109933fb15 100644 --- a/apps/sim/lib/file-parsers/yaml-parser.test.ts +++ b/apps/sim/lib/file-parsers/yaml-parser.test.ts @@ -70,6 +70,28 @@ describe('parseYAMLBuffer', () => { await expect(parseYAMLBuffer(Buffer.from(bomb))).rejects.toBeInstanceOf(YamlComplexityError) }) + it('parses a multi-document stream as one document per item', async () => { + const stream = + 'apiVersion: v1\nkind: Service\nmetadata:\n name: web\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\n' + const result = await parseYAMLBuffer(Buffer.from(stream)) + const parsed = JSON.parse(result.content) as Array<{ kind: string }> + + expect(parsed.map((document) => document.kind)).toEqual(['Service', 'Deployment']) + expect(result.metadata).toMatchObject({ + type: 'yaml', + isArray: true, + itemCount: 2, + documentCount: 2, + }) + }) + + it('keeps a single document unwrapped and skips empty documents in a stream', async () => { + const result = await parseYAMLBuffer(Buffer.from('---\nname: solo\n---\n')) + + expect(JSON.parse(result.content)).toEqual({ name: 'solo' }) + expect(result.metadata).toMatchObject({ isArray: false, documentCount: 1 }) + }) + it('surfaces malformed YAML as an Invalid YAML error', async () => { await expect(parseYAMLBuffer(Buffer.from('key: "unterminated\n'))).rejects.toThrow( /Invalid YAML/ @@ -128,4 +150,16 @@ describe('assertYamlWithinLimits', () => { const astral = String.fromCodePoint(0x1f600).repeat(10 * 1024 * 1024) expect(() => assertYamlWithinLimits({ text: astral })).not.toThrow() }) + + it('decodes a BOM-prefixed Latin-1 YAML file without losing accented characters', async () => { + const bom = await parseYAMLBuffer( + Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('name: Café')]) + ) + const latin1 = await parseYAMLBuffer(Buffer.from('name: Caf\xe9', 'latin1')) + + expect(JSON.parse(bom.content)).toEqual({ name: 'Café' }) + expect(bom.metadata?.encoding).toBe('utf-8') + expect(JSON.parse(latin1.content)).toEqual({ name: 'Café' }) + expect(latin1.metadata?.encoding).toBe('windows-1252') + }) }) diff --git a/apps/sim/lib/file-parsers/yaml-parser.ts b/apps/sim/lib/file-parsers/yaml-parser.ts index 8823cc4f6d8..97b89d166a9 100644 --- a/apps/sim/lib/file-parsers/yaml-parser.ts +++ b/apps/sim/lib/file-parsers/yaml-parser.ts @@ -2,6 +2,7 @@ import { getErrorMessage } from '@sim/utils/errors' import * as yaml from 'js-yaml' import { FileParserError } from '@/lib/file-parsers/errors' import type { FileParseResult } from '@/lib/file-parsers/types' +import { type DecodedText, decodeTextBuffer } from '@/lib/file-parsers/utils' import { measureYamlExpansion, type YamlExpansionLimits } from '@/lib/file-parsers/yaml-limits' /** @@ -52,7 +53,11 @@ export function assertYamlWithinLimits(root: unknown): number { * Parse a YAML value into the shared `FileParseResult` shape after validating * that its expanded form stays within safe complexity limits. */ -function buildYamlResult(yamlData: unknown): FileParseResult { +function buildYamlResult( + yamlData: unknown, + decoded: DecodedText, + documentCount: number +): FileParseResult { if (yamlData === undefined) { throw new FileParserError('empty_input', 'Empty YAML input provided') } @@ -66,6 +71,9 @@ function buildYamlResult(yamlData: unknown): FileParseResult { keys: Array.isArray(yamlData) ? [] : Object.keys((yamlData as Record) || {}), itemCount: Array.isArray(yamlData) ? yamlData.length : undefined, depth, + documentCount, + encoding: decoded.encoding, + ...(decoded.warning ? { warning: decoded.warning } : {}), } return { @@ -79,19 +87,7 @@ function buildYamlResult(yamlData: unknown): FileParseResult { */ export async function parseYAML(filePath: string): Promise { const fs = await import('fs/promises') - const content = await fs.readFile(filePath, 'utf-8') - - try { - const yamlData = yaml.load(content) - return buildYamlResult(yamlData) - } catch (error) { - if (error instanceof FileParserError) throw error - throw new FileParserError( - 'invalid_format', - `Invalid YAML: ${getErrorMessage(error, 'Unknown error')}`, - error - ) - } + return parseYAMLBuffer(await fs.readFile(filePath)) } /** @@ -102,11 +98,21 @@ export async function parseYAMLBuffer(buffer: Buffer): Promise throw new FileParserError('empty_input', 'Empty buffer provided') } - const content = buffer.toString('utf-8') + const decoded = decodeTextBuffer(buffer) try { - const yamlData = yaml.load(content) - return buildYamlResult(yamlData) + /** + * A YAML file is a stream: Kubernetes manifests, Helm output and CI + * fixtures routinely hold several documents separated by `---`. A single + * document keeps its own shape; a multi-document stream becomes an array of + * documents, which the JSON/YAML chunker then splits one document per item. + */ + const documents = yaml + .loadAll(decoded.text) + .filter((document) => document !== undefined && document !== null) + const yamlData = + documents.length === 1 ? documents[0] : documents.length === 0 ? undefined : documents + return buildYamlResult(yamlData, decoded, documents.length) } catch (error) { if (error instanceof FileParserError) throw error throw new FileParserError( diff --git a/apps/sim/lib/internal/file/operations.ts b/apps/sim/lib/internal/file/operations.ts index 5ae07140097..9907e9103c0 100644 --- a/apps/sim/lib/internal/file/operations.ts +++ b/apps/sim/lib/internal/file/operations.ts @@ -438,6 +438,10 @@ const extractUserFileTextContent = async ( if (extension && isSupportedFileType(extension)) { try { const result = await parseBuffer(buffer, extension) + if (result.metadata?.degraded === true) { + /** Scraped or placeholder output is a failure, not the file's content. */ + throw new Error(result.metadata.warning ?? 'Parser returned degraded output') + } return { text: result.content ?? '', truncated: result.metadata?.truncated === true } } catch (error) { logger.warn('Falling back to raw text after parser failure', { diff --git a/apps/sim/lib/internal/file/parser.test.ts b/apps/sim/lib/internal/file/parser.test.ts index dc091099550..9df0f56b89f 100644 --- a/apps/sim/lib/internal/file/parser.test.ts +++ b/apps/sim/lib/internal/file/parser.test.ts @@ -462,6 +462,35 @@ describe('file parser operation', () => { ) }) + /** + * A parser that could only scrape bytes flags its output `degraded`; the tool + * must report that as a failure rather than hand placeholder prose to the model. + */ + it('reports degraded parser output as a failure instead of returning it as content', async () => { + setupFileApiMocks({ + cloudEnabled: false, + storageProvider: 'local', + authenticated: true, + }) + mockParseBuffer.mockResolvedValue({ + content: 'Unable to extract text from DOC file. Please convert to DOCX format.', + metadata: { degraded: true, warning: 'Basic text extraction used' }, + }) + const req = createMockRequest('POST', { + filePath: 'workspace/legacy.doc', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(false) + expect(data.error).toContain('Could not extract text from legacy.doc') + expect(data.error).toContain('Basic text extraction used') + expect(JSON.stringify(data)).not.toContain('Unable to extract text from DOC file') + expect(mockUploadExecutionFile).not.toHaveBeenCalled() + }) + it('should reject parser complexity limits instead of returning raw text', async () => { setupFileApiMocks({ cloudEnabled: true, diff --git a/apps/sim/lib/internal/file/parser.ts b/apps/sim/lib/internal/file/parser.ts index f47dc5e749f..3318736429b 100644 --- a/apps/sim/lib/internal/file/parser.ts +++ b/apps/sim/lib/internal/file/parser.ts @@ -893,6 +893,13 @@ async function handleLocalFile( }) const extension = path.extname(filename).toLowerCase().substring(1) const result = await parseBuffer(fileBuffer, extension, { signal }) + if (result.metadata?.degraded === true) { + return { + success: false, + error: degradedParseMessage(filename, result.metadata.warning), + filePath, + } + } const content = assertParsedContentWithinLimit(result.content, maxParsedOutputBytes) signal?.throwIfAborted() const hash = createHash('md5').update(fileBuffer).digest('hex') @@ -1085,6 +1092,13 @@ async function handleGenericTextBuffer( if (isSupportedFileType(extension)) { const result = await parseBuffer(fileBuffer, extension, { signal }) + if (result.metadata?.degraded === true) { + return { + success: false, + error: degradedParseMessage(filename, result.metadata.warning), + filePath: originalPath || filename, + } + } return { success: true, @@ -1189,6 +1203,15 @@ async function parseBufferAsPdf(buffer: Buffer, signal?: AbortSignal) { /** * Format bytes to human readable size */ +/** + * A parser that could not read the document but returned scraped bytes or a + * placeholder sentence flags the result `degraded`; that must reach the model + * as a failure, never as the file's content. + */ +function degradedParseMessage(filename: string, warning: string | undefined): string { + return `Could not extract text from ${filename}${warning ? `: ${warning}` : ''}` +} + function prettySize(bytes: number): string { if (bytes === 0) return '0 Bytes' diff --git a/apps/sim/lib/knowledge/documents/document-processing-error.test.ts b/apps/sim/lib/knowledge/documents/document-processing-error.test.ts index b90d8a1ba46..b89acf5f261 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-error.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-error.test.ts @@ -76,7 +76,7 @@ describe('document processing failure taxonomy', () => { expect(toPermanentDocumentProcessingError(error, 'Contract.doc')).toBe(error) }) - it.each(['Contract.doc', 'Budget.xls', 'Deck.ppt'])( + it.each(['Contract.doc', 'Budget.xls', 'Deck.pptx'])( 'classifies an unreadable legacy Office file as repairable: %s', (filename) => { const failure = classifyDocumentProcessingFailure( diff --git a/apps/sim/lib/knowledge/documents/document-processing-error.ts b/apps/sim/lib/knowledge/documents/document-processing-error.ts index cc2b1d26469..57f04655c06 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-error.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-error.ts @@ -135,7 +135,6 @@ const OFFICE_REPAIR_EXTENSIONS = new Set([ 'xlsm', 'xlsb', 'xltx', - 'ppt', 'pptx', 'pptm', 'potx', diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts index 2d50949bc48..a0691b43220 100644 --- a/apps/sim/lib/knowledge/documents/document-processor.ts +++ b/apps/sim/lib/knowledge/documents/document-processor.ts @@ -42,6 +42,7 @@ import { MistralOperationError } from '@/lib/internal/mistral/errors' import { mistralParseInputSchema } from '@/lib/internal/mistral/input' import { executeMistralParse } from '@/lib/internal/mistral/operations' import { + isPermanentDocumentProcessingError, MAX_DOCUMENT_CHUNKS, OcrRequestRejectedError, PermanentDocumentProcessingError, @@ -260,6 +261,13 @@ export async function processDocument( access.signal?.throwIfAborted() const { content, processingMethod } = parseResult const cloudUrl = 'cloudUrl' in parseResult ? parseResult.cloudUrl : undefined + if (parseResult.metadata?.detectedType || parseResult.metadata?.warning) { + logger.info('Parser reported a warning for the document', { + filename, + detectedType: parseResult.metadata.detectedType, + warning: parseResult.metadata.warning, + }) + } /** * Guards every parser, not just the file parsers: OCR reads a scanned page @@ -410,6 +418,18 @@ async function readEmbeddedPdfText( signal: access.signal, pdfTextMode: 'complete', }) + /** + * The parser re-routes by sniffed bytes, so an HTML error page or plain text + * saved as `.pdf` comes back as its decoded text. That would pass the text + * layer check and be indexed as the "PDF"; it is not one, and OCR would only + * fail on it terminally, so it is rejected here as an invalid file. + */ + if (parsed.metadata?.detectedType) { + throw new PermanentDocumentProcessingError( + 'invalid_file', + `This file is named as a PDF but contains ${parsed.metadata.detectedType} content. Upload the actual PDF and retry.` + ) + } if (parsed.metadata?.truncated) { throw new FileParserError( 'complexity_limit', @@ -444,6 +464,7 @@ async function readEmbeddedPdfText( } } catch (error) { access.signal?.throwIfAborted() + if (isPermanentDocumentProcessingError(error)) throw error if ( (error instanceof Error && error.name === 'PasswordException') || (isFileParserError(error) && error.code === 'encrypted_file') @@ -1256,10 +1277,9 @@ async function processMistralOCRInBatches( /** * Why a document could not be read, phrased for whoever has to act on it. * - * The `doc` and `ppt` parsers never throw: on a legacy OLE binary or a deck with - * no text they return a placeholder sentence or scraped archive bytes, which an - * interactive upload can show a user but an automated sync must never embed. They - * report that as `degraded`, and it is treated here exactly like empty output. + * A parser that could only produce a placeholder (today an all-blank workbook) + * reports `degraded`, which an interactive upload can show a user but an + * automated sync must never embed; it is treated here exactly like empty output. * Legacy formats get the concrete remedy, since re-saving genuinely fixes them — * the modern container is one the bundled parsers read. */ diff --git a/apps/sim/lib/knowledge/documents/parser-extension.test.ts b/apps/sim/lib/knowledge/documents/parser-extension.test.ts index 4d65abdfef5..d506fde1b5a 100644 --- a/apps/sim/lib/knowledge/documents/parser-extension.test.ts +++ b/apps/sim/lib/knowledge/documents/parser-extension.test.ts @@ -2,6 +2,7 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' +import { classifyDocumentProcessingFailure } from '@/lib/knowledge/documents/document-processing-error' import { resolveParserExtension } from '@/lib/knowledge/documents/parser-extension' describe('resolveParserExtension', () => { @@ -24,4 +25,29 @@ describe('resolveParserExtension', () => { resolveParserExtension('uber-message.unknown', 'application/octet-stream') ).toThrow('Unsupported file type') }) + + /** + * Documents stored before `.ppt` was dropped from the registry re-enter the + * pipeline through this resolver. A plain `Error` classified as transient and + * burned the retry budget; the typed code makes the failure permanent. + */ + it.each([ + ['Deck.ppt', 'application/vnd.ms-powerpoint'], + ['no-extension', 'application/octet-stream'], + ])('classifies an unresolvable %s as a permanent unsupported type', (filename, mimeType) => { + const error = (() => { + try { + resolveParserExtension(filename, mimeType) + return null + } catch (caught) { + return caught + } + })() + + expect(error).toMatchObject({ name: 'FileParserError', code: 'unsupported_type' }) + expect(classifyDocumentProcessingFailure(error, filename)).toMatchObject({ + disposition: 'permanent', + code: 'unsupported_file_type', + }) + }) }) diff --git a/apps/sim/lib/knowledge/documents/parser-extension.ts b/apps/sim/lib/knowledge/documents/parser-extension.ts index 2b7dbc6b1d6..dd174109f6b 100644 --- a/apps/sim/lib/knowledge/documents/parser-extension.ts +++ b/apps/sim/lib/knowledge/documents/parser-extension.ts @@ -1,4 +1,5 @@ import { isSupportedFileType } from '@/lib/file-parsers' +import { FileParserError } from '@/lib/file-parsers/errors' import { extractStorageKey, getExtensionFromMimeType, @@ -34,13 +35,21 @@ export function resolveParserExtension( return fallback } + /** + * Typed so the document pipeline classifies it as permanent: a plain `Error` + * read as transient and burned the retry budget on every stored `.ppt`. + */ if (filenameExtension) { - throw new Error( + throw new FileParserError( + 'unsupported_type', `Unsupported file type: ${filenameExtension}. Supported types are: ${SUPPORTED_EXTENSIONS_TEXT}` ) } - throw new Error(`Could not determine file type for ${filename || 'document'}`) + throw new FileParserError( + 'unsupported_type', + `Could not determine file type for ${filename || 'document'}` + ) } /** diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts index 1be20951578..723bf3bc6f3 100644 --- a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts +++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts @@ -454,6 +454,21 @@ describe('PDF OCR triage', () => { expect(result.metadata.processingMethod).toBe('mistral-ocr') }) + /** + * The parser re-routes by sniffed bytes, so an HTML error page saved as `.pdf` + * comes back as decoded text that would pass the text-layer check. It is not a + * PDF and OCR would fail on it terminally, so it must be rejected up front. + */ + it('rejects a non-PDF file named .pdf instead of indexing its text or sending it to OCR', async () => { + mockParseBuffer.mockResolvedValue({ + content: 'Access denied. Your request was blocked by the firewall. '.repeat(40), + metadata: { detectedType: 'html', warning: 'parsed as .html instead of .pdf' }, + }) + + await expect(parse()).rejects.toMatchObject({ code: 'invalid_file' }) + expect(mockExecuteMistralParse).not.toHaveBeenCalled() + }) + it('rejects password-protected PDFs before provider admission', async () => { mockParseBuffer.mockRejectedValue( Object.assign(new Error('Password needed'), { name: 'PasswordException' }) diff --git a/apps/sim/lib/uploads/utils/validation.ts b/apps/sim/lib/uploads/utils/validation.ts index a6bbd26681b..72680c87f41 100644 --- a/apps/sim/lib/uploads/utils/validation.ts +++ b/apps/sim/lib/uploads/utils/validation.ts @@ -23,7 +23,6 @@ export const SUPPORTED_DOCUMENT_EXTENSIONS = [ 'md', 'xlsx', 'xls', - 'ppt', 'pptx', 'html', 'htm', @@ -155,7 +154,6 @@ export const SUPPORTED_MIME_TYPES: Record 'application/x-excel', 'application/x-msexcel', ], - ppt: ['application/vnd.ms-powerpoint', 'application/powerpoint', 'application/x-mspowerpoint'], pptx: [ 'application/vnd.openxmlformats-officedocument.presentationml.presentation', 'application/octet-stream', diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-text.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-text.ts index c78ad9ca5c0..86b742ac42a 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-text.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-text.ts @@ -36,10 +36,9 @@ export interface ReadWorkspaceFileTextResult { /** True when a parser limit stopped extraction before the input was exhausted. */ truncated: boolean /** - * True when no real extraction happened and `text` is best-effort scraped - * bytes or a placeholder rather than the document's content. Surfaced rather - * than converted into an error because the legacy `doc`/`ppt` parsers - * deliberately never throw, and that behavior is characterization-tested. + * True when no real extraction happened and `text` is a placeholder rather + * than the document's content — today only an all-blank workbook. Legacy + * formats raise typed parser errors instead of degrading. */ degraded: boolean degradedReason: string | null diff --git a/apps/sim/package.json b/apps/sim/package.json index e5f5cc2f0dd..35600921107 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -251,6 +251,7 @@ "typebox": "1.1.38", "undici": "7.29.0", "unified": "11.0.5", + "word-extractor": "1.0.4", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", "y-protocols": "1.0.7", "yjs": "13.6.31", diff --git a/apps/sim/scripts/parser-eval/BENCHMARK-raw.md b/apps/sim/scripts/parser-eval/BENCHMARK-raw.md new file mode 100644 index 00000000000..7e0323053af --- /dev/null +++ b/apps/sim/scripts/parser-eval/BENCHMARK-raw.md @@ -0,0 +1,261 @@ +# Before/after benchmark + +961 files compared. Gate: recall −0.02, vocab recall −0.02, noise +0.02, glued +1, junk +0.5, ok→error (except intended), now-degraded; CHECK flags for count-aware word depletion (a reference word whose occurrences fell by more than half) and for ≥5 reference words present before and absent after. + +**Regressions: 53** + +| ext | n | ok before→after | typed errors b→a | degraded b→a | recall b→a | ref_vocab_recall b→a | precision b→a | noise b→a | glued b→a | lines b→a | repeated_lines b→a | page_number_lines b→a | junk b→a | chunks b→a | ms b→a | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| csv | 52 | 52→52 | 0→0 | 0→0 | 0.373→0.373 | 0.657→0.657 | 0.519→0.519 | 0.088→0.088 | 0.115→0.115 | 561.558→561.558 | 1.423→1.423 | 0.019→0.019 | 0.000→0.000 | 7.135→7.135 | 12.074→12.713 | +| doc | 51 | 51→41 | 0→10 | 51→0 | 0.522→0.794 | 0.537→0.935 | 0.426→0.891 | 0.753→0.003 | 0.043→0.000 | 1.000→14.000 | 0.000→0.829 | 0.000→0.390 | 0.000→0.000 | 1.157→1.146 | 1.407→0.743 | +| docx | 102 | 81→84 | 16→18 | 0→0 | 0.830→0.816 | 0.936→0.962 | 0.985→0.922 | 0.001→0.001 | 0.038→0.051 | 163.728→43.893 | 77.802→6.583 | 26.321→0.024 | 0.002→0.002 | 2.136→2.238 | 22.842→18.123 | +| html | 60 | 60→60 | 0→0 | 0→0 | 0.709→0.711 | 0.971→0.974 | 0.691→0.672 | 0.146→0.148 | 15.900→7.250 | 2078.383→1118.967 | 776.833→194.567 | 20.050→4.833 | 0.009→0.009 | 29.817→27.250 | 41.769→50.499 | +| json | 31 | 30→31 | 1→0 | 0→0 | 0.989→0.985 | 0.982→0.974 | 0.994→0.995 | 0.002→0.002 | 1.033→1.000 | 19655.467→19023.323 | 17489.333→16925.161 | 2.000→1.935 | 0.051→0.049 | 116.133→112.419 | 3.446→3.542 | +| md | 38 | 38→38 | 0→0 | 0→0 | 1.000→1.000 | 0.996→0.996 | 1.000→1.000 | 0.000→0.000 | 0.289→0.289 | 162.289→162.289 | 12.211→12.211 | 0.026→0.026 | 0.000→0.000 | 4.605→4.605 | 0.298→0.309 | +| odp | 27 | 13→14 | 14→13 | 0→0 | —→— | —→— | —→— | —→— | —→— | 3.615→3.429 | 0.231→0.214 | 0.231→0.071 | 0.000→0.000 | 1.000→1.000 | 0.804→0.594 | +| ods | 32 | 32→32 | 0→0 | 6→6 | 0.839→0.927 | 0.976→0.976 | 0.544→0.615 | 0.148→0.065 | 0.000→0.000 | 26.438→26.438 | 0.188→0.344 | 0.594→0.812 | 0.000→0.000 | 1.000→1.000 | 1.333→1.376 | +| odt | 44 | 33→32 | 11→12 | 0→0 | 0.892→0.861 | 0.867→0.970 | 0.838→0.844 | 0.169→0.066 | 0.296→0.037 | 6.182→5.719 | 0.727→0.438 | 0.576→0.031 | 0.000→0.000 | 1.061→1.125 | 0.638→0.453 | +| pdf | 190 | 190→190 | 0→0 | 0→0 | 0.990→0.989 | 0.977→0.976 | 0.983→0.975 | 0.011→0.008 | 3.158→1.234 | 0.968→1475.468 | 0.000→115.637 | 0.000→9.911 | 0.340→0.342 | 23.753→25.032 | 95.589→97.390 | +| ppt | 30 | 30→0 | 0→30 | 30→0 | —→— | —→— | —→— | —→— | —→— | 1.000→— | 0.000→— | 0.000→— | 0.000→— | 1.500→— | 0.993→— | +| pptx | 72 | 66→44 | 2→28 | 20→0 | 0.979→0.903 | 0.997→0.980 | 1.000→0.874 | 0.000→0.001 | 0.000→0.000 | 101.561→144.432 | 68.727→99.750 | 9.152→6.841 | 0.003→0.005 | 1.985→2.682 | 7.143→5.602 | +| txt | 42 | 42→42 | 0→0 | 0→0 | 0.986→1.000 | 0.966→0.974 | 0.984→1.000 | 0.003→0.000 | 8.405→5.071 | 8124.310→8124.310 | 461.381→461.190 | 0.119→0.119 | 0.000→0.000 | 126.381→126.643 | 11.934→12.034 | +| xls | 43 | 43→43 | 0→0 | 4→4 | 0.785→0.901 | 0.961→0.961 | 0.647→0.717 | 0.070→0.075 | 0.000→0.000 | 104.419→104.326 | 5.047→5.047 | 0.535→0.535 | 0.000→0.000 | 3.023→2.977 | 6.399→6.814 | +| xlsb | 17 | 17→17 | 0→0 | 1→1 | 1.000→1.000 | 0.948→0.933 | 0.671→0.621 | 0.041→0.066 | 0.000→0.000 | 27.824→27.824 | 2.000→2.000 | 2.882→2.882 | 0.509→0.509 | 1.000→1.000 | 1.326→1.206 | +| xlsm | 17 | 16→16 | 1→1 | 5→5 | 0.850→0.562 | 0.926→0.926 | 0.435→0.214 | 0.124→0.122 | 0.000→0.000 | 60.875→53.500 | 15.188→8.375 | 1.812→0.375 | 0.000→0.000 | 7.188→4.875 | 11.411→11.305 | +| xlsx | 71 | 68→68 | 0→3 | 11→11 | 0.909→0.912 | 0.981→0.983 | 0.639→0.659 | 0.108→0.109 | 0.045→0.045 | 90.706→89.868 | 2.176→1.706 | 1.015→0.897 | 0.342→0.342 | 3.088→3.132 | 10.536→10.031 | +| yaml | 42 | 39→42 | 3→0 | 0→0 | 0.829→0.838 | 0.858→0.866 | 0.412→0.393 | 0.009→0.009 | 0.154→0.143 | 209.641→201.190 | 121.205→115.905 | 0.333→0.310 | 0.000→0.000 | 1.872→1.810 | 0.268→0.263 | + +## Flags per file + +| file | flags | +|---|---| +| doc__lo__comments-nested.doc | REGRESSION:vocab-recall 0.9231->0.6154 | +| doc__lo__fdo77844.doc | CHECK:lost-words 7 (bugs cgi freedesktop https hyperlink org); REGRESSION:vocab-recall 0.9859->0.8873 | +| doc__lo__tdf127166_prstDash_Word97.doc | CHECK:length 316->116 (no reference) | +| doc__lo__tdf49102_mergedCellNumbering.doc | intended:ok->typed-error | +| doc__lo__tdf75539_relativeWidth.doc | CHECK:length 1392->49 (no reference) | +| doc__lo__tdf98284_softLockedFields.doc | REGRESSION:recall 0.5->0.0 | +| doc__loimp__image-lazy-read-0size.doc | intended:ok->typed-error | +| doc__poi__47304.doc | CHECK:length 668->14 (no reference) | +| doc__poi__52117.doc | intended:ok->typed-error | +| doc__poi__57843.doc | intended:ok->typed-error | +| doc__poi__Bug47958.doc | REGRESSION:recall 1.0->0.9474 | +| doc__poi__Bug50955.doc | intended:ok->typed-error | +| doc__poi__Bug60936.doc | intended:ok->typed-error | +| doc__poi__FloatingPictures.doc | CHECK:lost-words 6 (arabic date embed seq sheet yyyy); CHECK:depleted nasa:5->2 | +| doc__poi__HeaderWithMacros.doc | REGRESSION:recall 1.0->0.0 | +| doc__poi__Word6_sections.doc | intended:ok->typed-error | +| doc__poi__ca.kwsymphony.www_education_School_Concert_Seat_Booking_Form_2011-12.doc | REGRESSION:recall 1.0->0.9474 | +| doc__poi__clusterfuzz-testcase-minimized-POIHWPFFuzzer-4951943183990784.doc | intended:ok->typed-error | +| doc__poi__clusterfuzz-testcase-minimized-POIHWPFFuzzer-5832867957309440.doc | intended:ok->typed-error | +| doc__poi__simple-table2.doc | CHECK:length 801->118 (no reference) | +| doc__poi__testCroppedPictures.doc | CHECK:length 582->18 (no reference) | +| doc__poi__vector_image.doc | intended:ok->typed-error | +| doc__unstr__fake-doc-emphasized-text.doc | REGRESSION:recall 1.0->0.5; REGRESSION:vocab-recall 1.0->0.7143 | +| docx__lo__FDO76312.docx | REGRESSION:recall 1.0->0.5 | +| docx__lo__fdo76316.docx | REGRESSION:recall 1.0->0.6667 | +| docx__lo__n780563.docx | improved:error->ok | +| docx__lo__table-style-border.docx | improved:error->ok | +| docx__mammoth__tables.docx | REGRESSION:recall 0.4->0.2 | +| docx__poi__59030.docx | REGRESSION:recall 0.25->0.0 | +| docx__poi__TestTableColumns.docx | improved:error->ok | +| docx__poi__bug65649.docx | CHECK:lost-words 7 (noп видыработ длястроительства напроектносметные разделусправочника стоистьруб); REGRESSION:recall 1.0->0.6; REGRESSION:glued 2->3 | +| docx__poi__clusterfuzz-testcase-minimized-POIFuzzer-6709287337197568.docx | improved:untyped->typed | +| docx__poi__clusterfuzz-testcase-minimized-POIXWPFFuzzer-4961551840247808.docx | improved:untyped->typed | +| docx__poi__clusterfuzz-testcase-minimized-POIXWPFFuzzer-5564805011079168.docx | improved:untyped->typed | +| docx__poi__clusterfuzz-testcase-minimized-POIXWPFFuzzer-6442791109263360.docx | improved:untyped->typed | +| docx__poi__crash-517626e815e0afa9decd0ebb6d1dee63fb9907dd.docx | improved:untyped->typed | +| docx__poi__table_footnotes.docx | REGRESSION:recall 0.25->0.0 | +| html__mdn__Web_Accessibility_ARIA.html | REGRESSION:noise 0.2321->0.2891 | +| html__mdn__Web_CSS_CSS_grid_layout.html | REGRESSION:noise 0.5172->0.5859 | +| html__mdn__Web_CSS_flex.html | REGRESSION:noise 0.4002->0.4932 | +| html__mdn__Web_HTML_Element_table.html | REGRESSION:glued 6->7 | +| html__mdn__Web_HTTP_Status_404.html | REGRESSION:noise 0.539->0.6055 | +| html__mdn__Web_JavaScript_Guide_Introduction.html | REGRESSION:noise 0.2999->0.3235 | +| html__wiki__Amazon_rainforest.html | CHECK:depleted east:104->17 central:90->12 south:103->31 sea:71->15 | +| html__wiki__Antarctica.html | CHECK:depleted expedition:212->47 hms:134->24 pole:90->38 amundsen:61->16 | +| html__wiki__Bach.html | CHECK:depleted hymnal:36->9 den:31->6 svenska:25->5 christian:35->15 | +| html__wiki__Black_hole.html | CHECK:depleted formulation:11->5 friedmann:11->5 formalism:11->5 | +| html__wiki__Chess.html | REGRESSION:recall 0.3533->0.2333 | +| html__wiki__Climate_change.html | CHECK:depleted forcing:26->11 model:27->12 event:22->7 country:19->7 | +| html__wiki__Coffee.html | CHECK:depleted white:15->7 | +| html__wiki__French_Revolution.html | CHECK:depleted battle:126->44 louis:127->61 charles:57->13 françois:61->21 | +| html__wiki__Mount_Everest.html | CHECK:depleted highway:42->7 himal:30->10 range:27->12 kang:23->11 | +| html__wiki__Mozart.html | CHECK:depleted conductor:13->5 | +| html__wiki__Periodic_table.html | CHECK:depleted aufbau:16->6 heat:11->5 | +| html__wiki__Photosynthesis.html | CHECK:depleted vitamin:23->10 earliest:17->7 | +| html__wiki__Renaissance.html | CHECK:depleted art:284->140 school:152->52 historical:69->31 christianity:40->14 | +| html__wiki__Roman_Empire.html | CHECK:depleted duchy:244->42 ship:130->12 boat:120->10 republic:170->66 | +| html__wiki__Shakespeare.html | CHECK:depleted the:3036->1056 and:1024->447 king:475->86 hamlet:384->65 | +| html__wiki__World_War_II.html | CHECK:depleted battle:155->77 famine:16->6 | +| json__tsconfig__prettier_prettier.json | improved:error->ok | +| odp__lo__tdf157795.odp | improved:error->ok | +| ods__lo__cachedValue.ods | CHECK:length 785->406 (no reference) | +| ods__lo__formula-across-sheets.ods | REGRESSION:recall 1.0->0.2222; REGRESSION:noise 0.0->0.0435 | +| ods__lo__tdf160003_page_anchored_object.ods | CHECK:length 753->236 (no reference) | +| ods__lo__test_borders_export.ods | REGRESSION:recall 1.0->0.9 | +| odt__loexp__redlineTextFrame.odt | REGRESSION:ok->error | +| odt__loexp__tdf169882.odt | REGRESSION:glued 0->1 | +| odt__unstr__fake.odt | REGRESSION:recall 0.7143->0.1429 | +| pdf__arxiv__2606.13260.pdf | CHECK:depleted ˆfv:9->0 | +| pdf__arxiv__2606.21569.pdf | CHECK:depleted ˆβriv:23->0 | +| pdf__arxiv__2606.21840.pdf | CHECK:lost-words 43 (b22κpˆθpy b2κpˆθpy hpnq phpnq pˆζj pˆλkqkďr); CHECK:depleted rμ1:12->0 ˆc1:12->0 ˆθpy:11->0 ˆfμ0:9->0; REGRESSION:vocab-recall 0.9552->0.9286 | +| pdf__arxiv__2606.22035.pdf | CHECK:lost-words 54 (bdi bgi bhij bti bξi bσg); CHECK:depleted ˆgi:67->0 ˆθg:43->0 ˆθˆgi:26->0 ˆθg0:24->0; REGRESSION:vocab-recall 0.9834->0.954 | +| pdf__arxiv__2606.22230.pdf | CHECK:lost-words 11 (bgperm bκ3 bκ4 bλk channel14 conse9); CHECK:depleted bλk:6->0 bgperm:5->0 | +| pdf__arxiv__2606.22255.pdf | CHECK:lost-words 23 (vˆι ˆfi ˆgc ˆgl ˆlm ˆlι); CHECK:depleted ˇμj:39->0 ˇμ1:35->0 ˇμk:13->0 ˇθj:7->0 | +| pdf__arxiv__2606.26142.pdf | CHECK:lost-words 49 (2x2 erp0 ev0 evl evp0 ezl); CHECK:depleted ˆδq:46->0 ˆcq:37->0 ˆλk:19->0 ˆgg:11->0; REGRESSION:vocab-recall 0.9682->0.9353 | +| pdf__arxiv__2608.09558.pdf | CHECK:depleted bkh:34->0 | +| pdf__arxiv__2608.09561.pdf | CHECK:depleted lˆe:14->2 bgβm:8->0 | +| pdf__arxiv__2608.09623.pdf | CHECK:lost-words 17 (ddimensional dvol dμn dωref efk egε); CHECK:depleted ˆqε:31->0 preprint:31->4 august:29->2 dvol:11->0 | +| pdf__arxiv__2608.09736.pdf | CHECK:lost-words 21 (2h1 bckm bun bvn coˆut ea2); CHECK:depleted eh1:42->0 ehn:27->0 ˆcf:17->0 ˆcehz:16->0 | +| pdf__arxiv__2608.20598.pdf | CHECK:lost-words 15 (bdj bdmart bdw bdβ bsd bset); CHECK:depleted bdw:20->0 bsd:9->0 ˆsc:8->0 bdj:8->0 | +| pdf__arxiv__2608.20601.pdf | CHECK:lost-words 7 (chomper62 chomper67 fundamen2 tal ˆλ1 ˆλ20) | +| pdf__arxiv__2608.20610.pdf | CHECK:lost-words 6 (ˆgi ˆyit ˆσ2 ˆτi ˆτit ˆτt); CHECK:depleted ˆgi:11->0 ˆτt:6->0 ˆτit:5->0 | +| pdf__arxiv__2608.20641.pdf | CHECK:lost-words 6 (2tr propor12 tion ˆβ2 ˆβmt ˆψu); CHECK:depleted ˆβmt:10->0 | +| pdf__arxiv__2608.20727.pdf | CHECK:lost-words 21 (ba2 ban bat ben bpi bpperm); CHECK:depleted ben:22->0 bzi:10->0 bηt:10->0 dba:10->0 | +| pdf__arxiv__2608.20744.pdf | CHECK:lost-words 34 (2daug 2dn 2ex 2pn bdc bdn); CHECK:depleted bθa:39->0 bηn:39->0 bdn:19->0 bθc:16->0 | +| pdf__arxiv__2608.20922.pdf | CHECK:depleted preprint:6->2 | +| pdf__arxiv__2608.21480.pdf | CHECK:lost-words 8 (cus27 descrip32 probabili24 ties tive tomer) | +| pdf__arxiv__2609.06011.pdf | CHECK:lost-words 8 (ased dif2 ferent predic19 unbi8 xperc); CHECK:depleted xperc:6->0 xprop:6->0 | +| pdf__arxiv__2609.06025.pdf | CHECK:depleted ˆu0:12->1 | +| pdf__arxiv__2609.06660.pdf | CHECK:depleted ˆcp:10->0 | +| pdf__arxiv__2609.06721.pdf | CHECK:lost-words 6 (associ6 dy8 mˆt namically vˆt ˆqt); CHECK:depleted hangyu:17->3 qin:21->7 | +| pdf__arxiv__2609.07655.pdf | CHECK:depleted preprint:18->1 | +| pdf__arxiv__2609.07666.pdf | CHECK:depleted ˆgk:15->0 wang:10->3 yao:9->2 xie:10->3 | +| pdf__arxiv__2609.07681.pdf | CHECK:lost-words 23 (ˆein ˆgi ˆx1 ˆx1α1 ˆx2 ˆx2α2); CHECK:depleted ˆxt:35->0 ˆyt:14->0 ˆxτ:11->0 ˆxd:6->0 | +| pdf__arxiv__2609.07689.pdf | CHECK:depleted ˆρj:10->0 | +| pdf__arxiv__2609.07706.pdf | CHECK:lost-words 23 (bp1 bx0 bx1 bxi bxt bxtk); CHECK:depleted exi:26->0 bx1:14->0 extk:12->0 ept:7->0 | +| pdf__arxiv__2609.07888.pdf | CHECK:lost-words 19 (2ˆτl bhblb bqlms bvbetween bvl bvwithin); CHECK:depleted ˆτhl:84->0 ˆs2:24->0 ˆτl:21->0 bvl:9->0 | +| pdf__arxiv__2609.08335.pdf | CHECK:lost-words 24 (3př 96ˆσ bkpa1 cs2 erś erˆθ); CHECK:depleted erˆθ:7->0 ˆσ2:6->0 varpˆθ:6->0 ˆθω:5->0 | +| pdf__arxiv__2609.08411.pdf | CHECK:lost-words 7 (first10 outcome3 ˆmj ˆβc ˆβe ˆτij); CHECK:depleted ˆτij:6->0 | +| pdf__arxiv__2609.08615.pdf | CHECK:depleted why:8->2 fail:10->4 preprint:8->2 | +| pdf__arxiv__2609.09039.pdf | CHECK:lost-words 51 (4yi bg0 bg1 bgas bgi bgr); CHECK:depleted bτdm:28->0 bgi:10->0 bτdir:9->0 bvdir:8->0; REGRESSION:vocab-recall 0.988->0.9654 | +| pdf__arxiv__2609.09388.pdf | CHECK:lost-words 15 (brt bsys byi eqs essys eδt); CHECK:depleted byi:5->0 | +| pdf__arxiv__2609.09436.pdf | CHECK:lost-words 11 (zk2 ˆfn ˆzn ˆα1 ˆαk ˆαkf); CHECK:depleted ˆfn:65->0 ˆκn:17->0 ˆzn:10->0 | +| pdf__arxiv__2609.09488.pdf | CHECK:lost-words 22 (bm0 bqe bqg bvg bwg bwrj); CHECK:depleted bwrj:7->0 bθj:6->0 bηk:6->0 bφp:5->0 | +| pdf__arxiv__2609.09538.pdf | CHECK:lost-words 40 (2dε bf1 bfj bfm bqb brb); CHECK:depleted bγn:68->0 bμj:56->0 gonzálezsanz:52->23 chen:50->21; REGRESSION:vocab-recall 0.9845->0.9619 | +| pdf__arxiv__2609.09588.pdf | CHECK:depleted dependence:36->14 nunes:26->4 | +| pdf__arxiv__2609.09600.pdf | CHECK:lost-words 15 (bjp bw2 bwp bρp bσ2 bτp); CHECK:depleted bwp:13->0 bjp:8->0 bτp:8->0 bρp:6->0 | +| pdf__arxiv__2609.09831.pdf | CHECK:lost-words 62 (4αpμ0 bhi bγ2 bγδp bδ2 bζ2); CHECK:depleted ˆβdy:109->0 ηey:50->0 ˆθdy:49->0 eηr:39->0; REGRESSION:vocab-recall 0.9685->0.9436 | +| pdf__arxiv__2609.09981.pdf | CHECK:lost-words 15 (baβ bgβ bjβ bln bλβ bμn); CHECK:depleted bjβ:15->0 bgβ:12->0 bλβ:11->0 bωβ:9->0; REGRESSION:recall 0.97->0.9433 | +| pdf__arxiv__2609.10011.pdf | CHECK:lost-words 16 (ap7 ˆbf ˆge ˆgf ˆgs ˆh3); CHECK:depleted ˆgf:5->0 | +| pdf__arxiv__2609.10020.pdf | CHECK:lost-words 19 (bˆε dwˆε wˆε ˆcs ˆqs ˆβ1); CHECK:depleted bˆε:13->0 wˆε:13->0 ˆεt:11->0 ˆcs:10->0 | +| pdf__arxiv__2609.10086.pdf | CHECK:lost-words 8 (viii xii xiii തq0 തq1 തq2); CHECK:depleted eik:20->5 | +| pdf__arxiv__2609.10266.pdf | CHECK:depleted preprint:30->8 review:24->2 iclr:30->8 | +| pdf__arxiv__2609.10291.pdf | CHECK:lost-words 8 (erm ˆin ˆvn ˆαn ˆβn ˆηi); CHECK:depleted ˆθn:49->0 ˆvn:21->0 ˆσn:16->0 ˆηi:8->0 | +| pdf__arxiv__2609.10311.pdf | CHECK:depleted published:26->1 5th:26->1 lifelong:26->1 agents:26->1 | +| pdf__arxiv__2609.10321.pdf | CHECK:depleted submitted:13->2 elsevier:13->2 page:13->2 dargmax:9->0 | +| pdf__arxiv__2609.10371.pdf | CHECK:depleted ˆyi:5->0 | +| pdf__irs__i1040gi.pdf | CHECK:lost-words 5 (ble deduc93 employ12 ment nontaxa92); CHECK:depleted visit:80->11 need:126->58 continued:47->16; REGRESSION:recall 1.0->0.9333 | +| pdf__irs__p15.pdf | CHECK:depleted publication:74->18 | +| pdf__irs__p463.pdf | CHECK:depleted publication:86->32 | +| pdf__irs__p501.pdf | CHECK:depleted publication:44->15 | +| pdf__irs__p505.pdf | CHECK:depleted publication:63->19 chapter:66->28 | +| pdf__irs__p523.pdf | CHECK:depleted publication:37->12 | +| pdf__irs__p525.pdf | CHECK:depleted publication:52->12 | +| pdf__irs__p526.pdf | CHECK:depleted publication:54->16 | +| pdf__irs__p529.pdf | CHECK:depleted publication:26->10 december:18->2 page:18->2 | +| pdf__irs__p550.pdf | CHECK:depleted publication:132->27 trades:103->51; REGRESSION:recall 1.0->0.9567 | +| pdf__irs__p554.pdf | CHECK:depleted publication:47->18 chapter:41->20 | +| pdf__irs__p590a.pdf | CHECK:depleted 590a:64->8 publication:83->28 | +| pdf__irs__p596.pdf | CHECK:depleted publication:54->23 | +| pdf__irs__p970.pdf | CHECK:depleted publication:100->45 | +| pdf__nist__NIST.AI.100-1.pdf | CHECK:depleted nist:76->34 page:48->8 | +| pdf__nist__NIST.CSWP.29.pdf | CHECK:depleted cswp:35->10 framework:49->24 | +| pdf__nist__NIST.FIPS.197-upd1.pdf | CHECK:depleted fips:59->16 | +| pdf__nist__NIST.SP.800-171r3.pdf | CHECK:depleted 800171r3:125->30 protecting:146->51 controlled:147->52 unclassified:133->38 | +| pdf__nist__NIST.SP.800-207.pdf | CHECK:depleted architecture:107->52 | +| pdf__nist__NIST.SP.800-218.pdf | CHECK:depleted version:48->17 | +| pdf__nist__NIST.SP.800-37r2.pdf | CHECK:depleted page:182->30 chapter:102->22 appendix:114->42 three:77->17 | +| pdf__nist__NIST.SP.800-52r2.pdf | CHECK:depleted implementations:102->34 | +| pdf__nist__NIST.SP.800-53r5.pdf | CHECK:lost-words 14 (vii viii wellnist xii xiii xiv); CHECK:depleted rev:526->37 page:475->17 chapter:389->19 three:376->19 | +| pdf__nist__NIST.SP.800-61r3.pdf | CHECK:depleted 80061r3:52->11 april:47->6 cyber:61->20 | +| pdf__nist__NIST.SP.800-63b.pdf | CHECK:depleted digital:114->45 lifecycle:86->17 | +| pdf__nist__NIST.SP.800-88r2.pdf | CHECK:depleted 80088r2:52->13 guidelines:64->25 | +| pdf__nist__NIST.SP.800-90Ar1.pdf | CHECK:depleted rbgs:111->51 rev:109->52 | +| pdf__pdfjs__file_pdfjs_form.pdf | REGRESSION:vocab-recall 1.0->0.875 | +| pdf__slides__european-lisp-symposium_els-web_housel-slides.pdf | CHECK:depleted els:27->3 zürich:27->3 switzerland:27->3 | +| pdf__slides__jeremytammik_tbc_ar20462_angel_velez_ifc_slides.pdf | CHECK:depleted autodesk:149->49; REGRESSION:junk 11.63->12.69 | +| pdf__slides__wzpan_BeamerStyleSlides_slides.pdf | CHECK:depleted josephpan:11->3; CHECK:length 575->345 (no reference) | +| pdf__slides__xiangjjj_implicit_alignment_slides.pdf | CHECK:depleted implicit:61->21 for:60->20 june:44->4 uda:48->8 | +| ppt__lo__FillPatterns.ppt | intended:ok->typed-error | +| ppt__lo__fdo68594.ppt | intended:ok->typed-error | +| ppt__lo__indent_multiple_spacings.ppt | intended:ok->typed-error | +| ppt__lo__ppt-indentation-bullets.ppt | intended:ok->typed-error | +| ppt__lo__tdf115394.ppt | intended:ok->typed-error | +| ppt__lo__tdf122899_Arc_90_to_91_clockwise.ppt | intended:ok->typed-error | +| ppt__lo__tdf136911.ppt | intended:ok->typed-error | +| ppt__lo__tdf157636.ppt | intended:ok->typed-error | +| ppt__lo__tdf168736-1.ppt | intended:ok->typed-error | +| ppt__lo__tdf168786.ppt | intended:ok->typed-error | +| ppt__lo__tdf49561.ppt | intended:ok->typed-error | +| ppt__lo__tdf77747.ppt | intended:ok->typed-error | +| ppt__poi__119877_all_type_background_save_by_AOO.ppt | intended:ok->typed-error | +| ppt__poi__41246-2.ppt | intended:ok->typed-error | +| ppt__poi__44296.ppt | intended:ok->typed-error | +| ppt__poi__49648.ppt | intended:ok->typed-error | +| ppt__poi__54541_cropped_bitmap.ppt | intended:ok->typed-error | +| ppt__poi__60294.ppt | intended:ok->typed-error | +| ppt__poi__WithLinks.ppt | intended:ok->typed-error | +| ppt__poi__br.com.tvcamboriu.www_pps_Pensar_5b1_5d.ppt | intended:ok->typed-error | +| ppt__poi__bug53192.ppt | intended:ok->typed-error | +| ppt__poi__bug58159_headers-and-footers.ppt | intended:ok->typed-error | +| ppt__poi__bug60345_paperfigures.ppt | intended:ok->typed-error | +| ppt__poi__cf5f6fde99a8b3ea5a4946c258b7abad6f30b0c5.ppt | intended:ok->typed-error | +| ppt__poi__clusterfuzz-testcase-minimized-POIHSLFFuzzer-5018229722382336.ppt | intended:ok->typed-error | +| ppt__poi__clusterfuzz-testcase-minimized-POIHSLFFuzzer-6416153805979648.ppt | intended:ok->typed-error | +| ppt__poi__headers_footers.ppt | intended:ok->typed-error | +| ppt__poi__npe.ppt | intended:ok->typed-error | +| ppt__poi__ppt_with_png.ppt | intended:ok->typed-error | +| ppt__unstr__fake-power-point.ppt | intended:ok->typed-error | +| pptx__lo__activex_spinbutton.pptx | intended:ok->typed-error | +| pptx__lo__bnc870233_2.pptx | intended:ok->typed-error | +| pptx__lo__crop-to-shape.pptx | intended:ok->typed-error | +| pptx__lo__group-rot.pptx | intended:ok->typed-error | +| pptx__lo__shape-blur-effect.pptx | intended:ok->typed-error | +| pptx__lo__smartart-children.pptx | intended:ok->typed-error | +| pptx__lo__smartart-org-chart2.pptx | intended:ok->typed-error | +| pptx__lo__tdf111884.pptx | intended:ok->typed-error | +| pptx__lo__tdf125346.pptx | intended:ok->typed-error | +| pptx__lo__tdf134053_dashdot.pptx | intended:ok->typed-error | +| pptx__lo__tdf151767.pptx | intended:ok->typed-error | +| pptx__poi__54542_cropped_bitmap.pptx | intended:ok->typed-error | +| pptx__poi__EmbeddedVideo.pptx | intended:ok->typed-error | +| pptx__poi__aascu.org_hbcu_leadershipsummit_cooper_.pptx | REGRESSION:noise 0.0->0.0234 | +| pptx__poi__au.asn.aes.www_conferences_2011_presentations_Fri_20Room4Level4_20930_20Maloney.pptx | improved:untyped->typed | +| pptx__poi__bug54570.pptx | intended:ok->typed-error | +| pptx__poi__bug60715.pptx | intended:ok->typed-error | +| pptx__poi__chart-slide-bg.pptx | intended:ok->typed-error | +| pptx__poi__clusterfuzz-testcase-minimized-POIXSLFFuzzer-4838644450394112.pptx | improved:untyped->typed | +| pptx__poi__clusterfuzz-testcase-minimized-POIXSLFFuzzer-5471515212382208.pptx | improved:untyped->typed | +| pptx__poi__clusterfuzz-testcase-minimized-POIXSLFFuzzer-6254434927378432.pptx | improved:untyped->typed | +| pptx__poi__crash-57308ca363f5b71763c489d1b432aff009d4bc4f.pptx | intended:ok->typed-error | +| pptx__poi__layouts.pptx | CHECK:lost-words 5 (apache foundation friday october software); REGRESSION:recall 1.0->0.5; REGRESSION:vocab-recall 1.0->0.8718 | +| pptx__poi__missing-blip-fill.pptx | REGRESSION:ok->error | +| pptx__poi__sample_pptx_grouping_issues.pptx | REGRESSION:ok->error | +| pptx__poi__smartart-rotated-text.pptx | intended:ok->typed-error | +| pptx__poi__table_test2.pptx | REGRESSION:recall 0.8333->0.1667 | +| pptx__unstr__fake-power-point-malformed.pptx | REGRESSION:vocab-recall 1.0->0.4 | +| pptx__unstr__fake-power-point-table.pptx | REGRESSION:recall 1.0->0.0 | +| pptx__unstr__picture.pptx | intended:ok->typed-error | +| pptx__unstr__science-exploration-369p.pptx | CHECK:depleted number:126->36 motivations2007:9->0 | +| pptx__unstr__test-image-jpg-mime.pptx | intended:ok->typed-error | +| txt__gutenberg_cp1252__pg17989.txt | CHECK:depleted mme:235->17 cria:100->16 pense:48->11 clair:34->9 | +| txt__gutenberg_latin1__pg1342.txt | CHECK:depleted bingleys:54->6 bennets:40->10 gardiners:22->9 | +| txt__gutenberg_latin1__pg2600.txt | CHECK:depleted ill:244->57 wont:147->6 emperors:136->33 fathers:101->7 | +| xls__lo__formats.xls | REGRESSION:noise 0.0645->0.1212 | +| xls__lo__forum-fr-59757.xls | REGRESSION:noise 0.0556->0.2273 | +| xls__poi__45672.xls | REGRESSION:recall 1.0->0.0 | +| xls__poi__styles-3563.xls | REGRESSION:recall 0.883->0.7447 | +| xlsb__poi__62815.xlsb | REGRESSION:vocab-recall 0.625->0.375; REGRESSION:noise 0.0->0.4 | +| xlsm__poi__57181.xlsm | REGRESSION:recall 0.9735->0.1534 | +| xlsm__poi__60512.xlsm | REGRESSION:recall 0.975->0.425 | +| xlsm__poi__mv-calculator-final-2-20-2013.xlsm | REGRESSION:recall 1.0->0.3533; REGRESSION:noise 0.0006->0.0231 | +| xlsx__lo__different-column-width-excel2010.xlsx | improved:untyped->typed | +| xlsx__poi__NumberFormatApproxTests.xlsx | CHECK:depleted 2345678e142:11->0 2345678e:10->0; REGRESSION:noise 0.0->0.082 | +| xlsx__poi__clusterfuzz-testcase-minimized-POIXSSFFuzzer-4828727001088000.xlsx | improved:untyped->typed | +| xlsx__poi__clusterfuzz-testcase-minimized-XLSX2CSVFuzzer-5542865479270400.xlsx | improved:untyped->typed | +| xlsx__unstr__2023-half-year-analyses-by-segment.xlsx | REGRESSION:recall 1.0->0.6389 | +| yaml__k8s__application_nginx-app.yaml | improved:error->ok | +| yaml__k8s__application_web_web.yaml | improved:error->ok | +| yaml__k8s__application_wordpress_mysql-deployment.yaml | improved:error->ok | diff --git a/apps/sim/scripts/parser-eval/BENCHMARK.md b/apps/sim/scripts/parser-eval/BENCHMARK.md new file mode 100644 index 00000000000..dbe0bbfb444 --- /dev/null +++ b/apps/sim/scripts/parser-eval/BENCHMARK.md @@ -0,0 +1,58 @@ +# Before/after benchmark — 2026-09-09 (final) + +961 real-world files in 18 formats (`bench/manifest.json` pins every file by URL and SHA-256; `bench/build.sh` rebuilds the corpus). Baseline = `origin/staging` parsers (`bench-run.ts` → `out-before`); after = this branch at its final commit, run alone on the machine so timings are comparable. Per-file table with every flag: `BENCHMARK-raw.md`. This summary is rendered by `bench-summary.py` from `compare.json`, so it cannot drift from the comparer. + +Metrics against independent extractors (PyMuPDF, python-docx, python-pptx, openpyxl/pandas, pandoc, chardet-decoded text). `recall` = share of reference lines (sampled evenly across the whole document) found in our output; `vocab` = share of reference words present, with hyphens collapsed on both sides and digits ignored; `noise` = share of our words absent from the reference; `glued` = distinct tokens that are two reference words fused; `lines` = non-blank output lines. + +| ext | n | ok b→a | recall b→a | vocab b→a | noise b→a | glued b→a | lines b→a | ms b→a | +|---|---|---|---|---|---|---|---|---| +| csv | 52 | 52→52 | 0.373→0.373 | 0.657→0.657 | 0.088→0.088 | 0.12→0.12 | 562→562 | 12→13 | +| doc | 51 | 51→41 | 0.522→0.794 | 0.537→0.935 | 0.753→0.003 | 0.04→0.00 | 1→14 | 1→1 | +| docx | 102 | 81→84 | 0.830→0.816 | 0.936→0.962 | 0.001→0.001 | 0.04→0.05 | 164→44 | 23→18 | +| html | 60 | 60→60 | 0.709→0.711 | 0.971→0.974 | 0.146→0.148 | 15.90→7.25 | 2078→1119 | 42→50 | +| json | 31 | 30→31 | 0.989→0.985 | 0.982→0.974 | 0.002→0.002 | 1.03→1.00 | 19655→19023 | 3→4 | +| md | 38 | 38→38 | 1.000→1.000 | 0.996→0.996 | 0.000→0.000 | 0.29→0.29 | 162→162 | 0→0 | +| odp | 27 | 13→14 | — | — | — | — | 4→3 | 1→1 | +| ods | 32 | 32→32 | 0.839→0.927 | 0.976→0.976 | 0.148→0.065 | 0.00→0.00 | 26→26 | 1→1 | +| odt | 44 | 33→32 | 0.892→0.861 | 0.867→0.970 | 0.169→0.066 | 0.30→0.04 | 6→6 | 1→0 | +| pdf | 190 | 190→190 | 0.990→0.989 | 0.977→0.976 | 0.011→0.008 | 3.16→1.23 | 1→1475 | 96→97 | +| ppt | 30 | 30→0 | — | — | — | — | 1→— | 1→— | +| pptx | 72 | 66→44 | 0.979→0.903 | 0.997→0.980 | 0.000→0.001 | 0.00→0.00 | 102→144 | 7→6 | +| txt | 42 | 42→42 | 0.986→1.000 | 0.966→0.974 | 0.003→0.000 | 8.40→5.07 | 8124→8124 | 12→12 | +| xls | 43 | 43→43 | 0.785→0.901 | 0.961→0.961 | 0.070→0.075 | 0.00→0.00 | 104→104 | 6→7 | +| xlsb | 17 | 17→17 | 1.000→1.000 | 0.948→0.933 | 0.041→0.066 | 0.00→0.00 | 28→28 | 1→1 | +| xlsm | 17 | 16→16 | 0.850→0.562 | 0.926→0.926 | 0.124→0.122 | 0.00→0.00 | 61→54 | 11→11 | +| xlsx | 71 | 68→68 | 0.909→0.912 | 0.981→0.983 | 0.108→0.109 | 0.05→0.05 | 91→90 | 11→10 | +| yaml | 42 | 39→42 | 0.829→0.838 | 0.858→0.866 | 0.009→0.009 | 0.15→0.14 | 210→201 | 0→0 | + +## What moved and why + +- **pdf**: 190/190 parse; line recall and vocabulary unchanged (0.989 / 0.976). Output went from one line per document to real lines and paragraphs (mean 1,475), fused tokens fell 3.2 → 1.2 per file, running headers/footers repeat once instead of once per page, page numbers are dropped. Latency flat (96 → 97 ms mean, p99 757 → 667 ms) after the page join was made linear. +- **doc**: 51 byte-scraped `degraded` outputs (noise 0.75: ZIP names, XML, placeholders; two files returned 3% and 17% of their body) → 41 real extractions via `word-extractor` (noise 0.003, recall 0.52 → 0.79, `degraded` false) plus 10 typed errors: 5 Word 6/95 (`unsupported_type`), 3 files with no body text (textutil agrees), 2 fuzzer fixtures (`invalid_format`). +- **ppt**: 30 degraded scrapes → 30 typed `unsupported_type`; uploads and connectors refuse `.ppt` up front. +- **docx / pptx / odt / odp**: tables emit `[Table]` / `| a | b |` rows, footnotes are kept, notes-page placeholders (slide numbers, headers), ODT comments and tracked deletions are dropped, nested tables are rendered once. Line recall against python-pptx/python-docx falls where the reference emits one cell per line (pptx 0.979 → 0.903) while vocabulary rises (docx 0.936 → 0.962, odt 0.867 → 0.970). 22 image-only LibreOffice pptx fixtures that returned `[Content_Types].xml…` as degraded now raise `no_extractable_text`; 3 decks whose only text was a slide number or a tracked deletion do too. +- **spreadsheets**: cells are display text (`$4,715`, `20%`, `2013-01-12`, `30:00`, `TRUE`) instead of stored values; references hold raw values, so "noise" rises by exactly those tokens and `xlsm` line recall drops on two dashboards whose every cell is formatted. Text-only sheets are byte-identical. +- **txt / yaml / json**: Latin-1, Windows-1252 and BOM inputs decode correctly (glued 8.4 → 5.1 were accent-stripped words); three Kubernetes multi-document manifests and a commented tsconfig now parse. +- **html**: nested tables rendered once (Wikipedia navboxes were triplicated), nested list items keep their marker, ordered lists are numbered; glued 15.9 → 7.3. + +## Regression gate + +Rules: any ok→error not intended, line recall −0.02, vocabulary −0.02, noise +0.02, any new glued token, junk +0.5/1k, newly degraded; `CHECK` flags for count-aware word depletion and for ≥5 reference words present before and absent after. Final result: **56 files with a REGRESSION flag, 0 SLOWER flags**, every one traced: + +| flag | files | cause | +|---|---|---| +| line recall (docx/pptx/odt/doc) | 27 | table rows vs one-cell-per-line references; textutil references include field codes and comment text | +| spreadsheet noise / recall | 13 | display text vs the reference's raw values | +| pdf vocabulary | 13 | math papers: staging output had fused glyph runs (`bσg0`, `2x2`) counted as words; one form lost its folio | +| glued / junk | 4 | Cyrillic cells un-glued (the metric counts the new split as a change), one slide deck | +| ok→error | 3 | two decks whose only text was a slide-number field; one ODT whose body is entirely a tracked deletion | + +`CHECK:depleted` fires on 99 files: running footers on IRS/NIST publications (intended, first copy kept), Wikipedia navboxes that were triplicated before, and math-glyph junk. The check exists because an earlier build of this branch deleted repeated table column headers on multi-page tables (IRS tax table: "Married filing jointly" 76 → 20); that is fixed and the counts are back (`Single` 80 → 80, `And your filing status is` 25 → 25). + +## Metric history + +The gate was tightened twice during the work and the parser fixed in between: the first after-run reported 195 flags (pdf glued 2.4 → 14.1 from over-eager dehyphenation, two spreadsheets refused by the sniffer); collapsing hyphens on both sides and ignoring bare page numbers in the vocabulary metric brought the same output to 81, and the parser fixes to 57. The audit pass then replaced head-of-document line sampling with whole-document sampling and added the count-aware checks; this final run under that stricter comparer shows 56. + +## Ground-truth corpus + +`REPORT-before.md` → `REPORT-after.md`: pdf paragraph retention 0.06 → 0.98, heading retention 0.00 → 0.91 (headings on their own lines; `## ` markers are off by default), glued 0.14 → 0.00; docx/pptx/odt table adjacency 0.00 → 1.00; xlsx/xls/xlsb typed-cell presence 0.87 → 1.00 and noise 0.12 → 0.02; robustness 14/14 (three expectations were rewritten to the by-design outcome: magic bytes win over the extension, `.ppt` is refused). No format lost presence or order. diff --git a/apps/sim/scripts/parser-eval/FINDINGS.md b/apps/sim/scripts/parser-eval/FINDINGS.md new file mode 100644 index 00000000000..a56a17978c0 --- /dev/null +++ b/apps/sim/scripts/parser-eval/FINDINGS.md @@ -0,0 +1,35 @@ +# Findings — 2026-09-09 run + +Corpus: 107 ground-truth renders (14 docs × docx/odt/pptx/html/md/pdf, 3 two-column PDFs, 4 workbooks × xlsx/xls/xlsb/ods/csv), 30 real-world files, 14 robustness cases. Raw metrics in `REPORT-before.md` (staging) and `REPORT-after.md` (this branch). Reproduce with the scripts in this directory (see `PLAN.md`). + +Content recall is 0.99–1.00 in every prose format and PDF text matches PyMuPDF at NED 0.996–1.000 on six real documents. The problems are structure, boilerplate, and typed cells. + +| # | Finding | Where | Evidence | +|---|---|---|---| +| 1 | PDF text flattened to one line before chunking (both modes) | `pdf-parser.ts` `.replace(/\s+/g, ' ')` | paragraph retention 0.06, heading retention 0.00; IRS p17 → 246 chunks, none at a paragraph | +| 2 | Spreadsheet dates/percent/currency indexed raw; Google Sheets sync inherits it | `xlsx-parser.ts` (no `raw:false`/`cellDates`) | `2026-03-04` → `46085`, `20%` → `0.2`; ODS dates → JS local-time string | +| 3 | Running headers/footers/page numbers leak into every PDF | `pdf-parser.ts` | absence 0.00 on 17/17 renders | +| 4 | Tables exploded one cell per line in docx/pptx/odt/odp | mammoth `extractRawText`, officeparser | table adjacency 0.00; mammoth HTML computed but unused | +| 5 | Words glued at line/column/cell boundaries in PDFs | items joined without separator when `hasEOL` false | irs-p17 39 glued tokens, omnidocbench 19, 2 of 3 two-column renders | +| 6 | Legacy .doc/.ppt fallback emits ZIP names, XML, master-slide placeholders | `doc-parser.ts`, `pptx-parser.ts` fallback | 7/7 real files degraded; KB and workspace-files search honour the flag | +| 7 | Slide numbers, footer placeholders, review comments indexed as body | officeparser | `poi-notes.pptx` bare `1..11` + `testdoc`; odt comment spliced mid-sentence | +| 8 | Non-UTF-8 text silently stripped | `txt-parser.ts`/`md-parser.ts` + `sanitizeTextForUTF8` | Latin-1 "Café résumé naïve £" → "Caf rsum nave" | +| 9 | Mislabelled inputs accepted; corrupt PDF error untyped (transient → OCR) | `index.ts` extension routing; pdf.js `Invalid PDF structure.` | CSV-as-xlsx mojibake + serials; HTML-as-txt raw markup | + +Fix order: PDF line structure + spacing → PDF furniture suppression → SheetJS formatted text → DOCX via mammoth HTML → officeparser post-processing → transcode fallback for text → type the PDF structure error → magic-byte sniffing. + +## Validation pass (9 parallel investigators, 2026-09-09) + +All nine findings confirmed. Corrections to the original framing: + +| # | Correction | Precedent | +|---|---|---| +| 1 | The whitespace collapse was copied from unpdf 1.4.0 for byte-identical output in #6425; unpdf fixed it in 1.7.0 (PR #58) before #6425 landed. No consumer or test depends on single-line text. pdf.js `hasEOL` arrives on an empty item whose y is the NEXT line. | unpdf #58, pdf.js text_layer, pdfplumber y_tolerance, pdfminer line_margin | +| 2 | `raw:false` alone is not enough: Excel's General format truncates 16-digit numbers to `4.11111E+15` and dates render locale-shaped; a pre-pass rewriting `w` for `t:'d'` and General cells fixes both. The Google Sheets and Microsoft Excel connectors ALREADY request formatted text, so Drive-synced Sheets disagree with Sheets-connector Sheets today. `xlsx-preview-data.ts` (file viewer) has the same defect. | SheetJS `raw`/`cellDates` docs, MarkItDown #53 | +| 3 | 62% of IRS p17 chunks carry the running footer. A frequency rule alone misses footers whose chapter title changes; Marker's consecutive-streak rule (>=3 pages) recovers it. Requires #1 first (needs reconstructed lines + y). | Marker IgnoreTextProcessor, OmniDocBench 'abandon', pymupdf4llm margins | +| 4 | DOCX via mammoth HTML -> existing HtmlParser walker prototyped: 8/8 adjacency, footnotes recovered, zero new deps. mammoth `convertToMarkdown` drops tables (do not use). officeparser 7.8 fixes tables but pulls tesseract.js + pdfjs-dist@6 (126 MB) and drops ODT header rows. | mammoth README, MarkItDown, unstructured, Docling | +| 5 | Fusions are NOT missing-hasEOL at line ends; they are (a) Form XObject boundaries (pdf.js resets prevTransform) and (b) backwards x-move on the same baseline (pdf.js flushes without EOL). Geometry join rule prototyped: catalog 15->0 fusions, IRS 56->25. Dehyphenation must check doc-local compounds or it breaks `open-source`. | pdf.js evaluator constants, MuPDF stext-device, pdfplumber | +| 6 | Worse than reported: two of four real .doc files return 3% and 17% of the body (UCS-2 text invisible to the ASCII regex). KB and workspace search honour `degraded`; Copilot file-reader, chat upload reader, File block (`internal/file/parser.ts`) and `get content` do NOT and hand the scrape to the model. `.xls` is fine (SheetJS BIFF). `word-extractor` (pure JS, frozen 2021) gets 89-100% on the POI .doc files; no viable pure-JS .ppt extractor exists. | word-extractor, Tika, Docling/unstructured shell out to soffice | +| 7 | The leaked `1..11` + `testdoc` come from NOTES PAGES (`ppt/notesSlides`), which officeparser dumps because we pass no options; not slide-level footers. officeparser 7.8 does not fix it. ODT splice includes `text:sender-initials`; tracked-change deletions also leak. Same JSZip walker as #4 fixes both. | python-pptx placeholder types, MarkItDown, POI SlideShowExtractor, pandoc ODT reader | +| 8 | Also: UTF-8 BOM leaks into content; UTF-16 'pass' was an ASCII accident; connectors keep U+FFFD as mojibake. Bun 1.3.14 TextDecoder supports `fatal` + `windows-1252` natively. Truncated-UTF-8 downloads need a tail retry before falling back. | Tika EncodingDetector chain, unstructured encoding.py, LangChain autodetect_encoding | +| 9 | `Invalid PDF structure.` is a named `InvalidPDFException`; the classifier just never checks it. docx-as-pdf never reaches OCR (`assertOcrSourceSupported` sniffs `%PDF-`). Truncated PDFs with a header DO reach OCR, deliberately, pinned by `pdf-ocr-triage.test.ts:439,580`. `ArchiveIntegrityError` is already classified permanent. `file-type@16.5.4` (CJS) is already in the tree via officeparser. | Tika detection precedence, unstructured detect_filetype, pdf.js exception names | diff --git a/apps/sim/scripts/parser-eval/PLAN.md b/apps/sim/scripts/parser-eval/PLAN.md new file mode 100644 index 00000000000..fab692e4fc0 --- /dev/null +++ b/apps/sim/scripts/parser-eval/PLAN.md @@ -0,0 +1,52 @@ +# Knowledge-base parser quality evaluation + +## Why + +Every file a connector (Drive, OneDrive, SharePoint, Box, Dropbox, S3, SFTP, Bitbucket, Gmail/Outlook attachments) or an upload delivers as bytes goes through `apps/sim/lib/file-parsers` before chunking and embedding. If a parser emits noise (XML internals, placeholder sentences, boilerplate), drops content, destroys paragraph structure, or scrambles reading order, every downstream search result inherits it silently: the document row still reads "success". + +## What the state of the art measures + +| Benchmark | What it scores | How | +|---|---|---| +| OmniDocBench (CVPR 2025) | text, tables, formulas, reading order across 10 doc types | Normalized Edit Distance on text and reading order, TEDS on tables; headers/footers/page numbers are an "abandon" class excluded from scoring | +| olmOCR-bench (Ai2) | 1,403 PDFs, 7,010 binary unit tests | text presence, text absence (headers/footers/page numbers must NOT appear), natural reading order pairs, table cell adjacency, math | +| READoc / opendataloader-bench | PDF to structured markdown | heading detection, reading order, table structure, F1 on blocks | + +Two design ideas transfer directly: (1) score with **binary unit tests per document** (presence, absence, order, adjacency), because fuzzy whole-document similarity hides localized failures; (2) treat **boilerplate leakage as a first-class failure**, not a rounding error. + +## Framework + +### Corpus (two tiers) + +**Tier A — ground truth by construction.** Source documents are authored as Markdown with a machine-readable spec (paragraphs, headings, list items, table cells, sentinel sentences, ordering pairs). Each source is rendered by pandoc to DOCX, ODT, PPTX, HTML and (via typst) PDF, so the same known content arrives in every container our parsers handle. PDFs are additionally rendered with running headers, footers and page numbers so absence tests are meaningful. Tabular specs are written with SheetJS to XLSX, XLS, XLSB, ODS and CSV. + +**Tier B — real-world documents with no gold text.** Public PDFs (two-column papers, forms, reports), DOCX/PPTX/XLSX/DOC/PPT/XLS/ODT files from open-source test corpora, and HTML pages. Scored by agreement against independent reference extractors (PyMuPDF, pdfplumber, python-docx, python-pptx, openpyxl) plus reference-free noise heuristics. + +### Metrics per (document, format) + +| Metric | Definition | Catches | +|---|---|---| +| `ned` | 1 − Levenshtein(norm(out), norm(gt)) / max(len) | gross content loss or gain | +| `presence` | share of sentinel sentences found (partial ratio ≥ 90) | dropped paragraphs, cells, slide bodies | +| `absence` | share of boilerplate strings (running header/footer/page numbers/"Sheet:" wrappers) NOT found | leakage into the index | +| `order` | share of (a before b) pairs preserved | column/slide/cell reordering | +| `table_adjacency` | share of (left cell, right cell) pairs appearing on one output line | tables exploded one cell per line | +| `noise_ratio` | share of output word tokens absent from gt vocabulary | XML names, placeholders, scraped bytes | +| `paragraph_retention` | output paragraph breaks / gt paragraphs | whitespace collapse that starves the chunker | +| `heading_retention` | share of headings appearing on their own line | headings glued into paragraphs | +| `junk_chars` | control/replacement/private-use chars per 1k chars | encoding damage | +| `chunk_sentence_boundary` | share of TextChunker chunks ending at sentence punctuation | how the parse degrades chunking | +| `metadata` | degraded/truncated/pageCount agree with reality | wrong flags either poison the index or skip good files | +| `latency_ms` | wall time | regressions | + +Robustness cases (empty, truncated, mislabeled extension, encrypted, non-UTF8) are scored pass/fail on whether a typed `FileParserError` is raised rather than garbage returned. + +### Execution + +1. `generate-corpus.py` builds Tier A sources, specs and renders (pandoc + typst + SheetJS). +2. `fetch-real-world.sh` downloads Tier B. +3. `run-parsers.ts` (bun, inside apps/sim so `@/` resolves) runs `parseBuffer` for every file exactly as the ingestion path does (`pdfTextMode: 'complete'` for PDFs) and writes JSON outputs. +4. `reference-extract.py` runs the reference extractors on the same files. +5. `score.py` computes the metric table, aggregated by format and by parser, and lists the worst documents. + +Everything reproducible from `apps/sim/scripts/parser-eval/`. diff --git a/apps/sim/scripts/parser-eval/README.md b/apps/sim/scripts/parser-eval/README.md new file mode 100644 index 00000000000..86ea7ca5135 --- /dev/null +++ b/apps/sim/scripts/parser-eval/README.md @@ -0,0 +1,33 @@ +# Knowledge base parser evaluation + +Tooling for measuring what `apps/sim/lib/file-parsers` hands to the chunker, and for comparing two checkouts on the same corpus. See `PLAN.md` for the design (modelled on olmOCR-bench unit tests and OmniDocBench scoring), `FINDINGS.md` for the audit that motivated PR #7709, and `BENCHMARK.md` for the before/after results. + +## Requirements + +- `bun` (run every `.ts` script from `apps/sim` so `@/` and the pinned `xlsx` resolve; set `DATABASE_URL=postgres://x:y@localhost:1/none` because the module graph touches `@sim/db` at import time) +- `pandoc` 3.x and a `typst` executable on `PATH` (the PyPI `typst` package is a library; wrap it in a script named `typst` that runs `typst.compile(input, output=output)`) +- A Python 3.12 environment with `pymupdf pdfplumber python-docx python-pptx openpyxl pandas xlrd pyxlsb odfpy chardet rapidfuzz`; point `PARSER_EVAL_PYTHON` at its interpreter +- `gh` (authenticated) and `curl` for the corpus fetchers; macOS `textutil` for `.doc` references + +## Ground-truth corpus (Tier A) + +```sh +python generate-corpus.py +bun scripts/parser-eval/generate-spreadsheets.ts +./fetch-real-world.sh +DATABASE_URL=postgres://x:y@localhost:1/none bun scripts/parser-eval/run-parsers.ts +python reference-extract.py +python score.py # writes report.md and scores.json +``` + +## Large real-world corpus (Tier B) + +`bench/manifest.json` pins 961 files by URL and SHA-256. `bench/build.sh` rebuilds the corpus (`fetch` re-downloads only what is missing and `manifest` verifies hashes; sources that drifted are listed in `bench/NOTES.md`), then writes reference extractions and a per-format report. + +```sh +PARSER_EVAL_PYTHON=/path/to/python bench/build.sh +DATABASE_URL=postgres://x:y@localhost:1/none bun scripts/parser-eval/bench-run.ts /out-