Skip to content

Commit 4534f51

Browse files
waleedlatif1claude
andcommitted
style(parsers): apply biome formatting
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 parent b761aa5 commit 4534f51

3 files changed

Lines changed: 136 additions & 24 deletions

File tree

apps/sim/lib/file-parsers/pdf-lines.test.ts

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -127,9 +127,7 @@ describe('joinLines', () => {
127127
const lines = paragraph(['we ship high-', 'quality builds'], 627.4)
128128
const words = collectWords(lines)
129129

130-
expect(joinLines(lines, { words, headingMarkers: false })).toBe(
131-
'we ship high-quality builds'
132-
)
130+
expect(joinLines(lines, { words, headingMarkers: false })).toBe('we ship high-quality builds')
133131
expect(joinLines(lines, { headingMarkers: false })).toBe('we ship high-quality builds')
134132
})
135133

apps/sim/scripts/parser-eval/bench-run.ts

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,21 +26,56 @@ for (const ext of readdirSync(filesRoot).sort()) {
2626
const bytes = readFileSync(path.join(dir, file))
2727
const started = performance.now()
2828
try {
29-
const result = await parseBuffer(bytes, ext, { pdfTextMode: ext === 'pdf' ? 'complete' : undefined })
29+
const result = await parseBuffer(bytes, ext, {
30+
pdfTextMode: ext === 'pdf' ? 'complete' : undefined,
31+
})
3032
const ms = performance.now() - started
3133
let chunkCount = -1
32-
try { chunkCount = (await chunker.chunk(result.content)).length } catch {}
34+
try {
35+
chunkCount = (await chunker.chunk(result.content)).length
36+
} catch {}
3337
const { html, sampledData, messages, headings, links, ...metadata } = result.metadata ?? {}
34-
writeFileSync(path.join(OUT, `${label}.json`), JSON.stringify({ label, ext, file, bytes: bytes.length, ms, ok: true, content: result.content, metadata, chunkCount }))
38+
writeFileSync(
39+
path.join(OUT, `${label}.json`),
40+
JSON.stringify({
41+
label,
42+
ext,
43+
file,
44+
bytes: bytes.length,
45+
ms,
46+
ok: true,
47+
content: result.content,
48+
metadata,
49+
chunkCount,
50+
})
51+
)
3552
summary[ext].ok++
36-
process.stdout.write(`ok ${label} ${result.content.length}ch ${ms.toFixed(0)}ms${metadata.degraded ? ' DEGRADED' : ''}${metadata.truncated ? ' TRUNCATED' : ''}\n`)
53+
process.stdout.write(
54+
`ok ${label} ${result.content.length}ch ${ms.toFixed(0)}ms${metadata.degraded ? ' DEGRADED' : ''}${metadata.truncated ? ' TRUNCATED' : ''}\n`
55+
)
3756
} catch (error) {
3857
const ms = performance.now() - started
3958
const typed = error instanceof FileParserError
40-
writeFileSync(path.join(OUT, `${label}.json`), JSON.stringify({ label, ext, file, bytes: bytes.length, ms, ok: false, typedError: typed, errorCode: typed ? error.code : undefined, errorName: (error as Error)?.name, error: String((error as Error)?.message ?? error).slice(0, 300) }))
59+
writeFileSync(
60+
path.join(OUT, `${label}.json`),
61+
JSON.stringify({
62+
label,
63+
ext,
64+
file,
65+
bytes: bytes.length,
66+
ms,
67+
ok: false,
68+
typedError: typed,
69+
errorCode: typed ? error.code : undefined,
70+
errorName: (error as Error)?.name,
71+
error: String((error as Error)?.message ?? error).slice(0, 300),
72+
})
73+
)
4174
summary[ext].error++
4275
if (typed) summary[ext].typed++
43-
process.stdout.write(`FAIL ${label} ${typed ? `typed:${error.code}` : `UNTYPED:${(error as Error)?.name}`} ${String((error as Error)?.message).slice(0, 80)}\n`)
76+
process.stdout.write(
77+
`FAIL ${label} ${typed ? `typed:${error.code}` : `UNTYPED:${(error as Error)?.name}`} ${String((error as Error)?.message).slice(0, 80)}\n`
78+
)
4479
}
4580
}
4681
}

apps/sim/scripts/parser-eval/run-parsers.ts

Lines changed: 94 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,36 @@ const OUT = process.argv[2]
1313
const OUTPUTS = path.join(OUT, 'outputs')
1414
mkdirSync(OUTPUTS, { recursive: true })
1515

16-
interface Entry { file: string; dir: string; doc: string; format: string; tier: 'A' | 'B'; absence: string[]; variant?: string; firstSheetOnly?: boolean }
16+
interface Entry {
17+
file: string
18+
dir: string
19+
doc: string
20+
format: string
21+
tier: 'A' | 'B'
22+
absence: string[]
23+
variant?: string
24+
firstSheetOnly?: boolean
25+
}
1726

1827
const entries: Entry[] = []
1928
for (const m of ['manifest-a.json', 'manifest-sheets.json']) {
2029
const p = path.join(OUT, m)
21-
if (existsSync(p)) for (const e of JSON.parse(readFileSync(p, 'utf8'))) entries.push({ ...e, dir: 'files' })
30+
if (existsSync(p))
31+
for (const e of JSON.parse(readFileSync(p, 'utf8'))) entries.push({ ...e, dir: 'files' })
2232
}
2333
const realDir = path.join(OUT, 'real')
2434
if (existsSync(realDir)) {
2535
for (const file of readdirSync(realDir).sort()) {
2636
const ext = path.extname(file).slice(1).toLowerCase()
2737
if (!ext) continue
28-
entries.push({ file, dir: 'real', doc: file.replace(/\.[^.]+$/, ''), format: ext, tier: 'B', absence: [] })
38+
entries.push({
39+
file,
40+
dir: 'real',
41+
doc: file.replace(/\.[^.]+$/, ''),
42+
format: ext,
43+
tier: 'B',
44+
absence: [],
45+
})
2946
}
3047
}
3148

@@ -38,10 +55,29 @@ const robustness: Array<{ name: string; ext: string; bytes: Buffer }> = [
3855
{ name: 'truncated-pdf', ext: 'pdf', bytes: fixture('memo.pdf').subarray(0, 3000) },
3956
{ name: 'pdf-bytes-labelled-docx', ext: 'docx', bytes: fixture('memo.pdf') },
4057
{ name: 'docx-bytes-labelled-pdf', ext: 'pdf', bytes: fixture('memo.docx') },
41-
{ name: 'png-labelled-doc', ext: 'doc', bytes: Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), Buffer.from(Array.from({ length: 4000 }, (_, i) => (i * 7919) % 256))]) },
42-
{ name: 'random-bytes-labelled-ppt', ext: 'ppt', bytes: Buffer.from(Array.from({ length: 50000 }, (_, i) => (i * 104729 + 17) % 256)) },
43-
{ name: 'latin1-txt', ext: 'txt', bytes: Buffer.from('Caf\xe9 r\xe9sum\xe9 na\xefve \xa3 42', 'latin1') },
44-
{ name: 'utf16-txt', ext: 'txt', bytes: Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from('Hello UTF-16 world', 'utf16le')]) },
58+
{
59+
name: 'png-labelled-doc',
60+
ext: 'doc',
61+
bytes: Buffer.concat([
62+
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
63+
Buffer.from(Array.from({ length: 4000 }, (_, i) => (i * 7919) % 256)),
64+
]),
65+
},
66+
{
67+
name: 'random-bytes-labelled-ppt',
68+
ext: 'ppt',
69+
bytes: Buffer.from(Array.from({ length: 50000 }, (_, i) => (i * 104729 + 17) % 256)),
70+
},
71+
{
72+
name: 'latin1-txt',
73+
ext: 'txt',
74+
bytes: Buffer.from('Caf\xe9 r\xe9sum\xe9 na\xefve \xa3 42', 'latin1'),
75+
},
76+
{
77+
name: 'utf16-txt',
78+
ext: 'txt',
79+
bytes: Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from('Hello UTF-16 world', 'utf16le')]),
80+
},
4581
{ name: 'html-labelled-txt', ext: 'txt', bytes: fixture('memo.html') },
4682
{ name: 'docx-labelled-xlsx', ext: 'xlsx', bytes: fixture('memo.docx') },
4783
{ name: 'csv-labelled-xlsx', ext: 'xlsx', bytes: fixture('sheet-employees.csv') },
@@ -55,27 +91,70 @@ const chunker = new TextChunker({ chunkSize: 1024, chunkOverlap: 200, minCharact
5591
async function runOne(label: string, ext: string, bytes: Buffer, meta: Record<string, unknown>) {
5692
const started = performance.now()
5793
try {
58-
const result = await parseBuffer(bytes, ext, { pdfTextMode: ext === 'pdf' ? 'complete' : undefined })
94+
const result = await parseBuffer(bytes, ext, {
95+
pdfTextMode: ext === 'pdf' ? 'complete' : undefined,
96+
})
5997
const ms = performance.now() - started
6098
let chunks: string[] = []
61-
try { chunks = (await chunker.chunk(result.content)).map((c) => c.text) } catch (e) { chunks = [`CHUNK_ERROR ${String(e)}`] }
99+
try {
100+
chunks = (await chunker.chunk(result.content)).map((c) => c.text)
101+
} catch (e) {
102+
chunks = [`CHUNK_ERROR ${String(e)}`]
103+
}
62104
const { html, sampledData, messages, ...metadata } = result.metadata ?? {}
63-
const record = { label, ext, bytes: bytes.length, ms, ok: true, content: result.content, metadata: { ...metadata, messageCount: Array.isArray(messages) ? messages.length : 0 }, chunks, ...meta }
105+
const record = {
106+
label,
107+
ext,
108+
bytes: bytes.length,
109+
ms,
110+
ok: true,
111+
content: result.content,
112+
metadata: { ...metadata, messageCount: Array.isArray(messages) ? messages.length : 0 },
113+
chunks,
114+
...meta,
115+
}
64116
writeFileSync(path.join(OUTPUTS, `${label}.json`), JSON.stringify(record, null, 1))
65-
results.push({ ...record, content: undefined, chunks: undefined, contentLength: result.content.length, chunkCount: chunks.length })
66-
process.stdout.write(`ok ${label} ${result.content.length}ch ${ms.toFixed(0)}ms ${metadata.degraded ? 'DEGRADED' : ''} ${metadata.truncated ? 'TRUNCATED' : ''}\n`)
117+
results.push({
118+
...record,
119+
content: undefined,
120+
chunks: undefined,
121+
contentLength: result.content.length,
122+
chunkCount: chunks.length,
123+
})
124+
process.stdout.write(
125+
`ok ${label} ${result.content.length}ch ${ms.toFixed(0)}ms ${metadata.degraded ? 'DEGRADED' : ''} ${metadata.truncated ? 'TRUNCATED' : ''}\n`
126+
)
67127
} catch (error) {
68128
const ms = performance.now() - started
69129
const typed = error instanceof FileParserError
70-
const record = { label, ext, bytes: bytes.length, ms, ok: false, typedError: typed, errorCode: typed ? (error as FileParserError).code : undefined, error: String((error as Error)?.message ?? error), ...meta }
130+
const record = {
131+
label,
132+
ext,
133+
bytes: bytes.length,
134+
ms,
135+
ok: false,
136+
typedError: typed,
137+
errorCode: typed ? (error as FileParserError).code : undefined,
138+
error: String((error as Error)?.message ?? error),
139+
...meta,
140+
}
71141
writeFileSync(path.join(OUTPUTS, `${label}.json`), JSON.stringify(record, null, 1))
72142
results.push(record)
73-
process.stdout.write(`FAIL ${label} ${typed ? `typed:${record.errorCode}` : 'UNTYPED'} ${record.error.slice(0, 100)}\n`)
143+
process.stdout.write(
144+
`FAIL ${label} ${typed ? `typed:${record.errorCode}` : 'UNTYPED'} ${record.error.slice(0, 100)}\n`
145+
)
74146
}
75147
}
76148

77149
for (const e of entries) {
78-
await runOne(e.file, e.format, readFileSync(path.join(OUT, e.dir, e.file)), { doc: e.doc, format: e.format, tier: e.tier, absence: e.absence, variant: e.variant, firstSheetOnly: e.firstSheetOnly })
150+
await runOne(e.file, e.format, readFileSync(path.join(OUT, e.dir, e.file)), {
151+
doc: e.doc,
152+
format: e.format,
153+
tier: e.tier,
154+
absence: e.absence,
155+
variant: e.variant,
156+
firstSheetOnly: e.firstSheetOnly,
157+
})
79158
}
80159
for (const r of robustness) await runOne(`robust__${r.name}`, r.ext, r.bytes, { tier: 'R' })
81160

0 commit comments

Comments
 (0)