fix(parsers): structure-preserving knowledge base parsers with before/after benchmark

apps/docs/openapi-v2-files-audit.json

@@ -745,7 +745,7 @@
       "get": {
         "operationId": "readFileText",
         "summary": "Read File Text",
-        "description": "Extract text without changing the file. Use Unzip File to unpack archives or Download File for original bytes. Unsupported types return `400`, compiling documents return `409`, and oversized files return `413`. `degraded: true` indicates incomplete or synthesized text, including some legacy `.doc` and `.ppt` results; `truncated: true` indicates a parser limit.\n\nOAuth scope: `api:read`.",
+        "description": "Extract text without changing the file. Use Unzip File to unpack archives or Download File for original bytes. Unsupported types return `400`, compiling documents return `409`, and oversized files return `413`. `degraded: true` indicates incomplete or synthesized text, such as the legacy `.pptx` fallback; `truncated: true` indicates a parser limit.\n\nOAuth scope: `api:read`.",
         "x-sim-operation": "files.read_content",
         "x-oauth-scope": "api:read",
         "tags": ["Files"],

apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data.test.ts

@@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
 import * as XLSX from 'xlsx'
 import {
   readXlsxPreviewData,
+  readXlsxWorkbook,
   XLSX_MAX_COLUMNS,
   XLSX_MAX_ROWS,
 } from '@/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data'
@@ -26,9 +27,11 @@ describe('readXlsxPreviewData', () => {
     const result = readXlsxPreviewData(XLSX, sheet)
     const options = toJson.mock.calls[0][1] as {
       range: { s: { r: number }; e: { r: number } }
+      raw?: boolean
     }
 
     expect(options.range.e.r - options.range.s.r).toBe(XLSX_MAX_ROWS)
+    expect(options.raw).toBe(false)
     expect(result.headers).toEqual(['header-a', 'header-b'])
     expect(result.rows).toHaveLength(XLSX_MAX_ROWS)
     expect(result.rows.slice(0, 2)).toEqual([
@@ -71,4 +74,37 @@ describe('readXlsxPreviewData', () => {
     expect(result.rowTruncated).toBe(false)
     expect(result.columnTruncated).toBe(true)
   })
+
+  /**
+   * Built through the viewer's own read path rather than by hand-setting `z`,
+   * so the assertions cover the read options as well as the conversion.
+   */
+  function typedWorkbook(): ArrayBuffer {
+    const sheet = XLSX.utils.aoa_to_sheet([['Issued', 'Rate', 'Card', 'Elapsed']])
+    sheet.A2 = { t: 'd', v: new Date(Date.UTC(2026, 2, 4)), z: 'm/d/yyyy' }
+    sheet.B2 = { t: 'n', v: 0.2, z: '0%' }
+    sheet.C2 = { t: 'n', v: 4111111111111111 }
+    sheet.D2 = { t: 'n', v: 1.25, z: '[h]:mm' }
+    sheet['!ref'] = 'A1:D2'
+    const book = XLSX.utils.book_new()
+    XLSX.utils.book_append_sheet(book, sheet, 'Ledger')
+    const bytes = XLSX.write(book, { type: 'buffer', bookType: 'xlsx' }) as Buffer
+    return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer
+  }
+
+  it('shows display text rather than stored values', () => {
+    const workbook = readXlsxWorkbook(XLSX, typedWorkbook())
+
+    const result = readXlsxPreviewData(XLSX, workbook.Sheets.Ledger)
+
+    expect(result.rows).toEqual([['2026-03-04', '20%', '4111111111111111', '30:00']])
+  })
+
+  it('reads the workbook with the display-text options', () => {
+    const read = vi.fn(XLSX.read)
+
+    readXlsxWorkbook({ read, utils: XLSX.utils }, typedWorkbook())
+
+    expect(read.mock.calls[0][1]).toMatchObject({ type: 'array', cellDates: true, cellNF: true })
+  })
 })

apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data.ts

@@ -1,10 +1,25 @@
-import type { WorkSheet } from 'xlsx'
+import type { WorkBook, WorkSheet } from 'xlsx'
+import {
+  normalizeSheetDisplayText,
+  SHEET_DISPLAY_READ_OPTIONS,
+} from '@/lib/file-parsers/sheet-display-text'
 
 export const XLSX_MAX_ROWS = 1_000
 export const XLSX_MAX_COLUMNS = 200
 
 interface XlsxModule {
-  utils: Pick<typeof import('xlsx').utils, 'decode_range' | 'sheet_to_json'>
+  read: typeof import('xlsx').read
+  utils: Pick<typeof import('xlsx').utils, 'decode_range' | 'encode_cell' | 'sheet_to_json'>
+}
+
+/**
+ * Reads a workbook for preview with the options that make its cells carry
+ * display text: without `cellDates` a date arrives as a bare serial and
+ * without `cellNF` no cell has a format, so every rendered `w` would be
+ * overwritten as a General number.
+ */
+export function readXlsxWorkbook(XLSX: XlsxModule, data: ArrayBuffer): WorkBook {
+  return XLSX.read(new Uint8Array(data), { type: 'array', ...SHEET_DISPLAY_READ_OPTIONS })
 }
 
 interface XlsxPreviewData {
@@ -18,12 +33,21 @@ export function readXlsxPreviewData(XLSX: XlsxModule, sheet: WorkSheet): XlsxPre
   const declaredRange = XLSX.utils.decode_range(sheet['!ref'] || 'A1')
   const lastPreviewRow = Math.min(declaredRange.e.r, declaredRange.s.r + XLSX_MAX_ROWS)
   const lastPreviewColumn = Math.min(declaredRange.e.c, declaredRange.s.c + XLSX_MAX_COLUMNS - 1)
+  const previewRange = {
+    s: declaredRange.s,
+    e: { r: lastPreviewRow, c: lastPreviewColumn },
+  }
+
+  /**
+   * Shown as the text a user sees in Excel: `raw: false` emits each cell's
+   * formatted text, so a sheet read through {@link readXlsxWorkbook} shows a
+   * date as ISO text and `20%` rather than a serial and `0.2`.
+   */
+  normalizeSheetDisplayText(sheet, previewRange, XLSX.utils)
   const previewRows = XLSX.utils.sheet_to_json<string[]>(sheet, {
     header: 1,
-    range: {
-      s: declaredRange.s,
-      e: { r: lastPreviewRow, c: lastPreviewColumn },
-    },
+    raw: false,
+    range: previewRange,
   })
 
   return {

apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview.tsx

@@ -10,6 +10,7 @@ import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace'
 import { useHorizontalWheelScroll } from '@/app/workspace/[workspaceId]/files/components/file-viewer/use-horizontal-wheel-scroll'
 import {
   readXlsxPreviewData,
+  readXlsxWorkbook,
   XLSX_MAX_COLUMNS,
   XLSX_MAX_ROWS,
 } from '@/app/workspace/[workspaceId]/files/components/file-viewer/xlsx-preview-data'
@@ -55,7 +56,7 @@ export const XlsxPreview = memo(function XlsxPreview({
         setRenderError(null)
         await assertOoxmlPreviewWithinLimits(data)
         const XLSX = await import('xlsx')
-        const workbook = XLSX.read(new Uint8Array(data), { type: 'array' })
+        const workbook = readXlsxWorkbook(XLSX, data)
         if (!cancelled) {
           workbookRef.current = workbook
           setSheetNames(workbook.SheetNames)

apps/sim/connectors/azure-devops/azure-devops.ts

@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { azureDevopsConnectorMeta } from '@/connectors/azure-devops/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
@@ -1182,7 +1183,7 @@ async function getFileDocument(
     return null
   }
 
-  const content = buffer.toString('utf8')
+  const content = decodeTextBuffer(buffer).text
   if (!content.trim()) return null
 
   const title = path.split('/').filter(Boolean).pop() || path

apps/sim/connectors/bitbucket/bitbucket.test.ts

@@ -774,7 +774,7 @@ describe('bitbucket getDocument', () => {
     expect(doc?.skippedReason).toMatch(/Binary/)
   })
 
-  it('surfaces non-UTF-8 source as skipped instead of indexing replacement characters', async () => {
+  it('decodes non-UTF-8 source as Windows-1252 instead of skipping or indexing replacement characters', async () => {
     mockApi([
       [
         /\/src\/[a-f0-9]+\/latin1\.txt$/,
@@ -784,8 +784,9 @@ describe('bitbucket getDocument', () => {
 
     const doc = await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'file:latin1.txt', {})
 
-    expect(doc?.skippedReason).toMatch(/Non-UTF-8/)
-    expect(doc?.content).toBe('')
+    expect(doc?.skippedReason).toBeUndefined()
+    expect(doc?.content).toContain('café')
+    expect(doc?.content).not.toContain('\uFFFD')
   })
 
   it('returns null for a file the ref no longer carries', async () => {

apps/sim/connectors/bitbucket/bitbucket.ts

@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { bitbucketConnectorMeta } from '@/connectors/bitbucket/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
@@ -81,7 +82,6 @@ const BINARY_SNIFF_BYTES = 8000
  */
 const MAX_TREE_DEPTH = 5
 const BINARY_SKIP_REASON = 'Binary file was not indexed'
-const NON_UTF8_SKIP_REASON = 'Non-UTF-8 file was not indexed'
 /**
  * Bitbucket answers a raw read of an LFS-managed file with a 301 to Atlassian's
  * media services platform. The connector deliberately surfaces the file as
@@ -1240,13 +1240,7 @@ export const bitbucketConnector: ConnectorConfig = {
         return markSkipped(stub, BINARY_SKIP_REASON)
       }
 
-      let text: string
-      try {
-        text = new TextDecoder('utf-8', { fatal: true }).decode(buffer)
-      } catch {
-        logger.info('Skipping non-UTF-8 Bitbucket file', { path })
-        return markSkipped(stub, NON_UTF8_SKIP_REASON)
-      }
+      const text = decodeTextBuffer(buffer).text
 
       const body = composeBody(stub.title, text)
       if (!body.trim()) return null

apps/sim/connectors/box/box.ts

@@ -1,6 +1,7 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import { sleep } from '@sim/utils/helpers'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { boxConnectorMeta } from '@/connectors/box/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
@@ -132,7 +133,6 @@ const REPRESENTATION_EXTENSIONS = new Set([
   'odt',
   'otp',
   'pdf',
-  'ppt',
   'pptx',
   'rtf',
   'vi',
@@ -319,7 +319,7 @@ async function fetchPlainTextContent(
   extension: string
 ): Promise<string> {
   const buffer = await downloadWithinLimit(`${BOX_API_BASE}/files/${fileId}/content`, accessToken)
-  const text = buffer.toString('utf8')
+  const { text } = decodeTextBuffer(buffer)
   return HTML_EXTENSIONS.has(extension) ? htmlToPlainText(text) : text
 }
 
@@ -347,7 +347,7 @@ async function fetchExtractedText(
         urlTemplate.replace('{+asset_path}', ''),
         accessToken
       )
-      return buffer.toString('utf8')
+      return decodeTextBuffer(buffer).text
     }
     if (state === 'error' || !infoUrl) return null
     if (attempt === REPRESENTATION_POLL_ATTEMPTS) break

apps/sim/connectors/databricks/databricks.ts

@@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { truncate } from '@sim/utils/string'
 import { validateDatabricksWorkspaceHost } from '@/lib/core/security/input-validation'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import {
   DATABRICKS_CONTENT_TYPES,
@@ -586,7 +587,7 @@ async function exportNotebook(
     return { skippedReason: sizeLimitSkipReason(CONNECTOR_MAX_FILE_BYTES) }
   }
 
-  return { content: decoded.toString('utf8') }
+  return { content: decodeTextBuffer(decoded).text }
 }
 
 /**

apps/sim/connectors/dropbox/dropbox.ts

@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { dropboxConnectorMeta } from '@/connectors/dropbox/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
@@ -147,7 +148,7 @@ async function downloadFileContent(
     throw new ConnectorFileTooLargeError(MAX_FILE_SIZE)
   }
 
-  const text = buffer.toString('utf8')
+  const { text } = decodeTextBuffer(buffer)
 
   return isHtml ? htmlToPlainText(text) : text
 }

apps/sim/connectors/github/github.ts

@@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { z } from 'zod'
 import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { type RetryOptions, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { parseGitHubRepository } from '@/lib/oauth/github-repository'
 import { githubConnectorMeta } from '@/connectors/github/meta'
@@ -296,7 +297,7 @@ async function fetchBlobContent(
     throw new ConnectorFileTooLargeError(maxBytes)
   }
   if (isBinaryBuffer(buffer)) return null
-  return buffer.toString('utf8')
+  return decodeTextBuffer(buffer).text
 }
 
 /** Resolves links within one snapshot; Contents can truncate dereferenced targets at 1 MiB. */

apps/sim/connectors/gitlab/gitlab.ts

@@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
 import { isPlainRecord } from '@sim/utils/object'
 import type { SecureFetchResponse } from '@/lib/core/security/input-validation.server'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server'
 import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { gitlabConnectorMeta } from '@/connectors/gitlab/meta'
@@ -435,7 +436,7 @@ function fileToDocument(
     return skipped(sizeLimitSkipReason(MAX_FILE_SIZE), buffer.byteLength)
   }
 
-  const content = buffer.toString('utf8')
+  const content = decodeTextBuffer(buffer).text
   const body = composeBody(title, content)
   if (!body.trim()) return null
 

apps/sim/connectors/google-drive/google-drive.ts

@@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { isPlainRecord } from '@sim/utils/object'
 import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import {
   type DrivePermission,
   driveFileAcl,
@@ -212,10 +213,10 @@ async function fetchFilePayload(
         },
       }
     }
-    return { content: bytes.toString('utf8'), mimeType: 'text/plain' }
+    return { content: decodeTextBuffer(bytes).text, mimeType: 'text/plain' }
   }
   if (file.mimeType === 'text/html') {
-    const html = (await downloadFile(accessToken, file.id, resourceKey)).toString('utf8')
+    const html = decodeTextBuffer(await downloadFile(accessToken, file.id, resourceKey)).text
     return { content: htmlToPlainText(html), mimeType: 'text/plain' }
   }
   const raw = rawFileType(file)
@@ -228,7 +229,7 @@ async function fetchFilePayload(
   }
   if (isSupportedTextFile(file.mimeType)) {
     return {
-      content: (await downloadFile(accessToken, file.id, resourceKey)).toString('utf8'),
+      content: decodeTextBuffer(await downloadFile(accessToken, file.id, resourceKey)).text,
       mimeType: 'text/plain',
     }
   }

apps/sim/connectors/s3/s3.ts

@@ -4,6 +4,7 @@ import { isLoopbackHostname } from '@sim/security/ssrf'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { truncate } from '@sim/utils/string'
 import { isHosted } from '@/lib/core/config/env-flags'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server'
 import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { s3ConnectorMeta } from '@/connectors/s3/meta'
@@ -658,7 +659,7 @@ export const s3Connector: ConnectorConfig = {
           sizeLimitSkipReason(MAX_FILE_SIZE)
         )
       }
-      const raw = body.toString('utf-8')
+      const raw = decodeTextBuffer(body).text
       const content = MARKUP_EXTENSIONS.has(getExtension(key)) ? htmlToPlainText(raw) : raw
       if (!content.trim()) return null
 

apps/sim/connectors/sftp/sftp.ts

@@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { type Attributes, type Client, type SFTPWrapper, utils as ssh2Utils } from 'ssh2'
 import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import {
   createSftpConnection,
   getFileType,
@@ -779,7 +780,7 @@ export const sftpConnector: ConnectorConfig = {
         return markSkipped(stub, 'File appears to be binary and was not indexed')
       }
 
-      const raw = buffer.toString('utf-8')
+      const raw = decodeTextBuffer(buffer).text
       const content = HTML_EXTENSIONS.has(getExtension(remotePath)) ? htmlToPlainText(raw) : raw
       if (!content.trim()) return null