@@ -13,19 +13,36 @@ const OUT = process.argv[2]
1313const OUTPUTS = path . join ( OUT , 'outputs' )
1414mkdirSync ( OUTPUTS , { recursive : true } )
1515
16- interface Entry { file : string ; dir : string ; doc : string ; format : string ; tier : 'A' | 'B' ; absence : string [ ] ; variant ?: string ; firstSheetOnly ?: boolean }
16+ interface Entry {
17+ file : string
18+ dir : string
19+ doc : string
20+ format : string
21+ tier : 'A' | 'B'
22+ absence : string [ ]
23+ variant ?: string
24+ firstSheetOnly ?: boolean
25+ }
1726
1827const entries : Entry [ ] = [ ]
1928for ( const m of [ 'manifest-a.json' , 'manifest-sheets.json' ] ) {
2029 const p = path . join ( OUT , m )
21- if ( existsSync ( p ) ) for ( const e of JSON . parse ( readFileSync ( p , 'utf8' ) ) ) entries . push ( { ...e , dir : 'files' } )
30+ if ( existsSync ( p ) )
31+ for ( const e of JSON . parse ( readFileSync ( p , 'utf8' ) ) ) entries . push ( { ...e , dir : 'files' } )
2232}
2333const realDir = path . join ( OUT , 'real' )
2434if ( existsSync ( realDir ) ) {
2535 for ( const file of readdirSync ( realDir ) . sort ( ) ) {
2636 const ext = path . extname ( file ) . slice ( 1 ) . toLowerCase ( )
2737 if ( ! ext ) continue
28- entries . push ( { file, dir : 'real' , doc : file . replace ( / \. [ ^ . ] + $ / , '' ) , format : ext , tier : 'B' , absence : [ ] } )
38+ entries . push ( {
39+ file,
40+ dir : 'real' ,
41+ doc : file . replace ( / \. [ ^ . ] + $ / , '' ) ,
42+ format : ext ,
43+ tier : 'B' ,
44+ absence : [ ] ,
45+ } )
2946 }
3047}
3148
@@ -38,10 +55,29 @@ const robustness: Array<{ name: string; ext: string; bytes: Buffer }> = [
3855 { name : 'truncated-pdf' , ext : 'pdf' , bytes : fixture ( 'memo.pdf' ) . subarray ( 0 , 3000 ) } ,
3956 { name : 'pdf-bytes-labelled-docx' , ext : 'docx' , bytes : fixture ( 'memo.pdf' ) } ,
4057 { name : 'docx-bytes-labelled-pdf' , ext : 'pdf' , bytes : fixture ( 'memo.docx' ) } ,
41- { name : 'png-labelled-doc' , ext : 'doc' , bytes : Buffer . concat ( [ Buffer . from ( [ 0x89 , 0x50 , 0x4e , 0x47 , 0x0d , 0x0a , 0x1a , 0x0a ] ) , Buffer . from ( Array . from ( { length : 4000 } , ( _ , i ) => ( i * 7919 ) % 256 ) ) ] ) } ,
42- { name : 'random-bytes-labelled-ppt' , ext : 'ppt' , bytes : Buffer . from ( Array . from ( { length : 50000 } , ( _ , i ) => ( i * 104729 + 17 ) % 256 ) ) } ,
43- { name : 'latin1-txt' , ext : 'txt' , bytes : Buffer . from ( 'Caf\xe9 r\xe9sum\xe9 na\xefve \xa3 42' , 'latin1' ) } ,
44- { name : 'utf16-txt' , ext : 'txt' , bytes : Buffer . concat ( [ Buffer . from ( [ 0xff , 0xfe ] ) , Buffer . from ( 'Hello UTF-16 world' , 'utf16le' ) ] ) } ,
58+ {
59+ name : 'png-labelled-doc' ,
60+ ext : 'doc' ,
61+ bytes : Buffer . concat ( [
62+ Buffer . from ( [ 0x89 , 0x50 , 0x4e , 0x47 , 0x0d , 0x0a , 0x1a , 0x0a ] ) ,
63+ Buffer . from ( Array . from ( { length : 4000 } , ( _ , i ) => ( i * 7919 ) % 256 ) ) ,
64+ ] ) ,
65+ } ,
66+ {
67+ name : 'random-bytes-labelled-ppt' ,
68+ ext : 'ppt' ,
69+ bytes : Buffer . from ( Array . from ( { length : 50000 } , ( _ , i ) => ( i * 104729 + 17 ) % 256 ) ) ,
70+ } ,
71+ {
72+ name : 'latin1-txt' ,
73+ ext : 'txt' ,
74+ bytes : Buffer . from ( 'Caf\xe9 r\xe9sum\xe9 na\xefve \xa3 42' , 'latin1' ) ,
75+ } ,
76+ {
77+ name : 'utf16-txt' ,
78+ ext : 'txt' ,
79+ bytes : Buffer . concat ( [ Buffer . from ( [ 0xff , 0xfe ] ) , Buffer . from ( 'Hello UTF-16 world' , 'utf16le' ) ] ) ,
80+ } ,
4581 { name : 'html-labelled-txt' , ext : 'txt' , bytes : fixture ( 'memo.html' ) } ,
4682 { name : 'docx-labelled-xlsx' , ext : 'xlsx' , bytes : fixture ( 'memo.docx' ) } ,
4783 { name : 'csv-labelled-xlsx' , ext : 'xlsx' , bytes : fixture ( 'sheet-employees.csv' ) } ,
@@ -55,27 +91,70 @@ const chunker = new TextChunker({ chunkSize: 1024, chunkOverlap: 200, minCharact
5591async function runOne ( label : string , ext : string , bytes : Buffer , meta : Record < string , unknown > ) {
5692 const started = performance . now ( )
5793 try {
58- const result = await parseBuffer ( bytes , ext , { pdfTextMode : ext === 'pdf' ? 'complete' : undefined } )
94+ const result = await parseBuffer ( bytes , ext , {
95+ pdfTextMode : ext === 'pdf' ? 'complete' : undefined ,
96+ } )
5997 const ms = performance . now ( ) - started
6098 let chunks : string [ ] = [ ]
61- try { chunks = ( await chunker . chunk ( result . content ) ) . map ( ( c ) => c . text ) } catch ( e ) { chunks = [ `CHUNK_ERROR ${ String ( e ) } ` ] }
99+ try {
100+ chunks = ( await chunker . chunk ( result . content ) ) . map ( ( c ) => c . text )
101+ } catch ( e ) {
102+ chunks = [ `CHUNK_ERROR ${ String ( e ) } ` ]
103+ }
62104 const { html, sampledData, messages, ...metadata } = result . metadata ?? { }
63- const record = { label, ext, bytes : bytes . length , ms, ok : true , content : result . content , metadata : { ...metadata , messageCount : Array . isArray ( messages ) ? messages . length : 0 } , chunks, ...meta }
105+ const record = {
106+ label,
107+ ext,
108+ bytes : bytes . length ,
109+ ms,
110+ ok : true ,
111+ content : result . content ,
112+ metadata : { ...metadata , messageCount : Array . isArray ( messages ) ? messages . length : 0 } ,
113+ chunks,
114+ ...meta ,
115+ }
64116 writeFileSync ( path . join ( OUTPUTS , `${ label } .json` ) , JSON . stringify ( record , null , 1 ) )
65- results . push ( { ...record , content : undefined , chunks : undefined , contentLength : result . content . length , chunkCount : chunks . length } )
66- process . stdout . write ( `ok ${ label } ${ result . content . length } ch ${ ms . toFixed ( 0 ) } ms ${ metadata . degraded ? 'DEGRADED' : '' } ${ metadata . truncated ? 'TRUNCATED' : '' } \n` )
117+ results . push ( {
118+ ...record ,
119+ content : undefined ,
120+ chunks : undefined ,
121+ contentLength : result . content . length ,
122+ chunkCount : chunks . length ,
123+ } )
124+ process . stdout . write (
125+ `ok ${ label } ${ result . content . length } ch ${ ms . toFixed ( 0 ) } ms ${ metadata . degraded ? 'DEGRADED' : '' } ${ metadata . truncated ? 'TRUNCATED' : '' } \n`
126+ )
67127 } catch ( error ) {
68128 const ms = performance . now ( ) - started
69129 const typed = error instanceof FileParserError
70- const record = { label, ext, bytes : bytes . length , ms, ok : false , typedError : typed , errorCode : typed ? ( error as FileParserError ) . code : undefined , error : String ( ( error as Error ) ?. message ?? error ) , ...meta }
130+ const record = {
131+ label,
132+ ext,
133+ bytes : bytes . length ,
134+ ms,
135+ ok : false ,
136+ typedError : typed ,
137+ errorCode : typed ? ( error as FileParserError ) . code : undefined ,
138+ error : String ( ( error as Error ) ?. message ?? error ) ,
139+ ...meta ,
140+ }
71141 writeFileSync ( path . join ( OUTPUTS , `${ label } .json` ) , JSON . stringify ( record , null , 1 ) )
72142 results . push ( record )
73- process . stdout . write ( `FAIL ${ label } ${ typed ? `typed:${ record . errorCode } ` : 'UNTYPED' } ${ record . error . slice ( 0 , 100 ) } \n` )
143+ process . stdout . write (
144+ `FAIL ${ label } ${ typed ? `typed:${ record . errorCode } ` : 'UNTYPED' } ${ record . error . slice ( 0 , 100 ) } \n`
145+ )
74146 }
75147}
76148
77149for ( const e of entries ) {
78- await runOne ( e . file , e . format , readFileSync ( path . join ( OUT , e . dir , e . file ) ) , { doc : e . doc , format : e . format , tier : e . tier , absence : e . absence , variant : e . variant , firstSheetOnly : e . firstSheetOnly } )
150+ await runOne ( e . file , e . format , readFileSync ( path . join ( OUT , e . dir , e . file ) ) , {
151+ doc : e . doc ,
152+ format : e . format ,
153+ tier : e . tier ,
154+ absence : e . absence ,
155+ variant : e . variant ,
156+ firstSheetOnly : e . firstSheetOnly ,
157+ } )
79158}
80159for ( const r of robustness ) await runOne ( `robust__${ r . name } ` , r . ext , r . bytes , { tier : 'R' } )
81160
0 commit comments