From 9ee166f943fd9f265f12188993d830f2f0ce6c8a Mon Sep 17 00:00:00 2001 From: Jason Mulligan Date: Wed, 12 Aug 2026 12:14:56 -0400 Subject: [PATCH 1/6] feat: add file format extraction tools (docx, pptx, xlsx, pdf, etc.) Fixes #767 --- .../.openspec.yaml | 2 + .../file-format-extraction-tools/design.md | 74 +++++++++++++++++++ .../file-format-extraction-tools/proposal.md | 31 ++++++++ .../specs/docx-extraction/spec.md | 28 +++++++ .../specs/file-extraction/spec.md | 38 ++++++++++ .../specs/pdf-extraction/spec.md | 35 +++++++++ .../specs/pptx-extraction/spec.md | 28 +++++++ .../specs/xlsx-extraction/spec.md | 39 ++++++++++ .../file-format-extraction-tools/tasks.md | 49 ++++++++++++ 9 files changed, 324 insertions(+) create mode 100644 openspec/changes/file-format-extraction-tools/.openspec.yaml create mode 100644 openspec/changes/file-format-extraction-tools/design.md create mode 100644 openspec/changes/file-format-extraction-tools/proposal.md create mode 100644 openspec/changes/file-format-extraction-tools/specs/docx-extraction/spec.md create mode 100644 openspec/changes/file-format-extraction-tools/specs/file-extraction/spec.md create mode 100644 openspec/changes/file-format-extraction-tools/specs/pdf-extraction/spec.md create mode 100644 openspec/changes/file-format-extraction-tools/specs/pptx-extraction/spec.md create mode 100644 openspec/changes/file-format-extraction-tools/specs/xlsx-extraction/spec.md create mode 100644 openspec/changes/file-format-extraction-tools/tasks.md diff --git a/openspec/changes/file-format-extraction-tools/.openspec.yaml b/openspec/changes/file-format-extraction-tools/.openspec.yaml new file mode 100644 index 00000000..5081c987 --- /dev/null +++ b/openspec/changes/file-format-extraction-tools/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-12 diff --git a/openspec/changes/file-format-extraction-tools/design.md b/openspec/changes/file-format-extraction-tools/design.md new file mode 100644 index 00000000..1c9d5f87 --- /dev/null +++ b/openspec/changes/file-format-extraction-tools/design.md @@ -0,0 +1,74 @@ +## Context + +Madz currently supports vision-based file processing (`src/tools/vision.js`) and web content extraction (`src/tools/web.js`), but has no tools for extracting content from common office and personal file formats. Users working with `.docx`, `.pptx`, `.xlsx`, PDF, and similar formats cannot query their content through the agent. + +Most office formats share a common architecture: ZIP archives containing XML files. This presents an opportunity for a shared extraction layer. PDF is architecturally distinct and requires a separate code path. + +## Goals / Non-Goals + +**Goals:** +- Provide a shared ZIP/XML extraction utility for archive-based formats +- Implement format-specific extraction tools for docx, pptx, xlsx, and PDF +- Output structured markdown (or JSON for tabular data) suitable for LLM consumption +- Register all tools in the central registry with appropriate permissions +- Include comprehensive unit tests with sample file fixtures + +**Non-Goals:** +- Legacy binary formats (`.ppt`, `.xls`, `.doc`) — require OLE compound file parsing +- OCR for scanned PDFs — documented as future enhancement +- Embedded media extraction (images, audio, video within documents) +- Charts, macros, pivot tables, or complex spreadsheet formulas +- Writing/creating documents — extraction only + +## Decisions + +### Decision 1: Shared ZIP/XML Utility +**Choice**: Create `extractZipXml(filePath)` returning `Map` (filename → XML content). +**Rationale**: Eliminates code duplication across 5+ format parsers. Centralizes error handling for corrupted archives. Each format-specific parser receives the same consistent input. +**Alternatives considered**: +- Inline ZIP handling in each tool → rejected (code duplication, inconsistent error handling) +- External library abstraction → rejected (unnecessary indirection for a single function) + +### Decision 2: PDF as Separate Code Path +**Choice**: Use `pdf-parse` directly; do not force PDF into ZIP/XML pattern. +**Rationale**: PDF is a fundamentally different format with its own binary structure. The ZIP/XML pattern does not apply. `pdf-parse` is lightweight, well-maintained, and sufficient for text extraction. +**Alternatives considered**: +- `pdfjs-dist` (Mozilla) → rejected (larger bundle, more complex API) +- Custom PDF parser → rejected (unnecessary complexity) + +### Decision 3: Markdown as Primary Output +**Choice**: All tools output markdown; xlsx additionally supports JSON output. +**Rationale**: Markdown is the natural input format for LLMs. The project already depends on `marked`. JSON output for xlsx enables programmatic consumption of tabular data. +**Alternatives considered**: +- Plain text → rejected (loses structure: headings, lists, tables) +- HTML → rejected (not LLM-friendly, requires stripping) + +### Decision 4: Tool Registration Pattern +**Choice**: Follow the existing pattern from `src/tools/vision.js` — async function with zod input schema, registered in `src/tools/index.js` with permission tier and classification. +**Rationale**: Consistency with existing codebase. Minimal learning curve for future contributors. + +### Decision 5: Sandbox Permissions +**Choice**: All tools require `filesystem:read` permission. File paths are validated against the sandbox path resolver. +**Rationale**: Tools read files from the filesystem. No write operations needed. Sandbox constraints from `src/tools/shell.js` apply. + +## Risks / Trade-offs + +[Risk] ZIP archives may be corrupted or password-protected → Mitigation: Graceful error handling with descriptive messages. Tools return structured error output, not crashes. + +[Risk] Large documents may cause memory issues → Mitigation: Stream processing where possible. Document size limits in tool descriptions. + +[Risk] XML namespaces vary across formats → Mitigation: Each format parser handles its own namespace resolution. Shared utility provides namespace-agnostic filename lookup. + +[Risk] PDF text extraction may produce garbled output for complex layouts → Mitigation: Document limitations in tool description. OCR path documented for future enhancement. + +[Risk] New dependencies increase attack surface → Mitigation: Only `pdf-parse` added (minimal, well-audited). ZIP handling uses existing Node.js `zlib` or `archiver` dependencies. + +## Migration Plan + +No migration required. This is a greenfield feature addition. Existing tools and workflows are unaffected. + +## Open Questions + +1. Should xlsx output include cell formatting (bold, color, formulas) or just values? → Default to values only; formatting can be added later. +2. Should pptx extraction include slide notes by default? → Yes, included in markdown output. +3. Should we support `.odt`/`.ods`/`.odp` (OpenDocument) in v1? → Yes, they follow the same ZIP/XML pattern as Office formats. diff --git a/openspec/changes/file-format-extraction-tools/proposal.md b/openspec/changes/file-format-extraction-tools/proposal.md new file mode 100644 index 00000000..9e09faa7 --- /dev/null +++ b/openspec/changes/file-format-extraction-tools/proposal.md @@ -0,0 +1,31 @@ +## Why + +Users frequently work with proprietary or binary file formats (`.docx`, `.pptx`, `.xlsx`, PDF, etc.) that are opaque to LLMs and text-based tooling. Converting these formats into structured markdown or JSON enables content extraction, analysis, and integration into the agent workflow. Most common office formats are ZIP archives containing XML, making extraction feasible with shared parsing utilities. + +## What Changes + +- Add a shared `extract-zip-xml` utility for ZIP-based format decompression and XML parsing +- Implement format-specific extraction tools: `docx-to-markdown`, `pptx-to-markdown`, `xlsx-to-markdown`, `xlsx-to-json`, `pdf-to-markdown` +- Register all new tools in the central tool registry (`src/tools/index.js`) with appropriate permissions and classifications +- Add `pdf-parse` dependency for PDF text extraction +- Create comprehensive unit tests with sample file fixtures + +## Capabilities + +### New Capabilities +- `file-extraction`: Shared ZIP/XML extraction utility for decompressing and parsing ZIP-based document formats (docx, pptx, xlsx, odt, ods, odp, epub, pages, numbers, key) +- `docx-extraction`: Convert `.docx` files to structured markdown with headings, paragraphs, lists, tables, and inline formatting +- `pptx-extraction`: Convert `.pptx` files to markdown with slide titles, bullet points, and speaker notes +- `xlsx-extraction`: Convert `.xlsx` files to markdown tables or JSON with cell values, types, and references +- `pdf-extraction`: Extract text from PDF files to markdown; OCR path documented for future enhancement + +### Modified Capabilities + + +## Impact + +- **Affected code**: `src/tools/index.js` (tool registration), `src/tools/vision.js` (pattern reference), `src/tools/shell.js` (sandbox constraints) +- **New dependencies**: `pdf-parse` for PDF extraction; `adm-zip` or `jszip` for ZIP handling (likely already available via existing dependencies) +- **New files**: `src/tools/fileExtract/` directory with utility and format-specific parsers +- **Test fixtures**: Sample docx, pptx, xlsx, and PDF files in `tests/fixtures/` +- **Non-goals**: Legacy binary formats (`.ppt`, `.xls`), OCR for scanned PDFs, embedded media extraction, charts/macros/pivot tables diff --git a/openspec/changes/file-format-extraction-tools/specs/docx-extraction/spec.md b/openspec/changes/file-format-extraction-tools/specs/docx-extraction/spec.md new file mode 100644 index 00000000..c896ddc9 --- /dev/null +++ b/openspec/changes/file-format-extraction-tools/specs/docx-extraction/spec.md @@ -0,0 +1,28 @@ +## ADDED Requirements + +### Requirement: DOCX to markdown conversion +The system SHALL convert `.docx` files to structured markdown, preserving headings, paragraphs, lists, tables, and inline formatting (bold, italic, code). + +#### Scenario: Extract headings from docx +- **WHEN** a docx file with heading styles is provided +- **THEN** the system outputs markdown headings (`#`, `##`, `###`) matching the document hierarchy + +#### Scenario: Extract paragraphs from docx +- **WHEN** a docx file with body text is provided +- **THEN** the system outputs markdown paragraphs with proper line breaks + +#### Scenario: Extract lists from docx +- **WHEN** a docx file with ordered or unordered lists is provided +- **THEN** the system outputs markdown list items (`-` for unordered, `1.` for ordered) + +#### Scenario: Extract tables from docx +- **WHEN** a docx file with tables is provided +- **THEN** the system outputs markdown tables with proper column alignment + +#### Scenario: Handle empty docx file +- **WHEN** an empty docx file is provided +- **THEN** the system returns an empty string + +#### Scenario: Handle missing document.xml +- **WHEN** a docx file without word/document.xml is provided +- **THEN** the system throws a descriptive error diff --git a/openspec/changes/file-format-extraction-tools/specs/file-extraction/spec.md b/openspec/changes/file-format-extraction-tools/specs/file-extraction/spec.md new file mode 100644 index 00000000..002fabb3 --- /dev/null +++ b/openspec/changes/file-format-extraction-tools/specs/file-extraction/spec.md @@ -0,0 +1,38 @@ +## ADDED Requirements + +### Requirement: ZIP archive extraction +The system SHALL decompress ZIP-based document archives and extract their XML content files into a structured map of filename to content. + +#### Scenario: Successful ZIP extraction +- **WHEN** a valid ZIP archive file path is provided +- **THEN** the system returns a map of filename strings to their XML content strings + +#### Scenario: Corrupted ZIP archive +- **WHEN** a corrupted or invalid ZIP file is provided +- **THEN** the system throws a descriptive error without crashing + +#### Scenario: Password-protected ZIP archive +- **WHEN** a password-protected ZIP file is provided +- **THEN** the system throws an error indicating the archive is password-protected + +### Requirement: XML content retrieval by filename +The system SHALL locate specific XML files within a ZIP archive by their known filenames regardless of internal directory structure. + +#### Scenario: Locate document.xml in docx +- **WHEN** the utility is asked for "word/document.xml" in a docx file +- **THEN** the system returns the XML content of that file + +#### Scenario: Locate slide files in pptx +- **WHEN** the utility is asked for "ppt/slides/slide1.xml" in a pptx file +- **THEN** the system returns the XML content of that slide file + +### Requirement: File format validation +The system SHALL validate that a file is a supported ZIP-based format before attempting extraction. + +#### Scenario: Validate docx extension +- **WHEN** a file with `.docx` extension is provided +- **THEN** the system confirms it is a supported format + +#### Scenario: Reject unsupported format +- **WHEN** a file with an unsupported extension is provided +- **THEN** the system throws an error listing supported formats diff --git a/openspec/changes/file-format-extraction-tools/specs/pdf-extraction/spec.md b/openspec/changes/file-format-extraction-tools/specs/pdf-extraction/spec.md new file mode 100644 index 00000000..cbaecce9 --- /dev/null +++ b/openspec/changes/file-format-extraction-tools/specs/pdf-extraction/spec.md @@ -0,0 +1,35 @@ +## ADDED Requirements + +### Requirement: PDF text extraction +The system SHALL extract text content from PDF files and output it as structured markdown. + +#### Scenario: Extract text from simple PDF +- **WHEN** a PDF file with plain text content is provided +- **THEN** the system outputs the extracted text as markdown paragraphs + +#### Scenario: Extract text from multi-page PDF +- **WHEN** a multi-page PDF file is provided +- **THEN** the system outputs text from all pages in sequential order + +#### Scenario: Handle PDF with no extractable text +- **WHEN** a PDF file contains only images (no text layer) +- **THEN** the system returns an error indicating no text could be extracted + +#### Scenario: Handle empty PDF file +- **WHEN** an empty or minimal PDF file is provided +- **THEN** the system returns an empty string + +#### Scenario: Handle PDF with special characters +- **WHEN** a PDF file contains Unicode characters +- **THEN** the system preserves Unicode characters in the output + +### Requirement: PDF extraction error handling +The system SHALL provide descriptive error messages for PDF extraction failures. + +#### Scenario: Handle corrupted PDF +- **WHEN** a corrupted PDF file is provided +- **THEN** the system throws a descriptive error without crashing + +#### Scenario: Handle password-protected PDF +- **WHEN** a password-protected PDF file is provided +- **THEN** the system throws an error indicating the file is password-protected diff --git a/openspec/changes/file-format-extraction-tools/specs/pptx-extraction/spec.md b/openspec/changes/file-format-extraction-tools/specs/pptx-extraction/spec.md new file mode 100644 index 00000000..4fdd3c7e --- /dev/null +++ b/openspec/changes/file-format-extraction-tools/specs/pptx-extraction/spec.md @@ -0,0 +1,28 @@ +## ADDED Requirements + +### Requirement: PPTX to markdown conversion +The system SHALL convert `.pptx` files to structured markdown, preserving slide titles, bullet points, speaker notes, and basic text content. + +#### Scenario: Extract slide titles from pptx +- **WHEN** a pptx file with slide titles is provided +- **THEN** the system outputs markdown headings (`#`) for each slide title + +#### Scenario: Extract bullet points from pptx +- **WHEN** a pptx file with bullet points is provided +- **THEN** the system outputs markdown unordered list items (`-`) for each bullet + +#### Scenario: Extract speaker notes from pptx +- **WHEN** a pptx file with speaker notes is provided +- **THEN** the system outputs markdown text prefixed with "Speaker Notes:" + +#### Scenario: Handle slides without titles +- **WHEN** a pptx slide has no title text +- **THEN** the system outputs a numbered slide separator (e.g., `---`) + +#### Scenario: Handle empty pptx file +- **WHEN** an empty pptx file is provided +- **THEN** the system returns an empty string + +#### Scenario: Handle missing slide files +- **WHEN** a pptx file with missing slide XML is provided +- **THEN** the system skips the missing slide and continues processing remaining slides diff --git a/openspec/changes/file-format-extraction-tools/specs/xlsx-extraction/spec.md b/openspec/changes/file-format-extraction-tools/specs/xlsx-extraction/spec.md new file mode 100644 index 00000000..1d194590 --- /dev/null +++ b/openspec/changes/file-format-extraction-tools/specs/xlsx-extraction/spec.md @@ -0,0 +1,39 @@ +## ADDED Requirements + +### Requirement: XLSX to markdown table conversion +The system SHALL convert `.xlsx` spreadsheets to markdown tables, preserving cell values, column headers, and row structure. + +#### Scenario: Extract single-sheet xlsx to markdown +- **WHEN** a single-sheet xlsx file is provided +- **THEN** the system outputs a markdown table with headers from the first row and data from subsequent rows + +#### Scenario: Extract multi-sheet xlsx to markdown +- **WHEN** a multi-sheet xlsx file is provided +- **THEN** the system outputs separate markdown tables for each sheet, separated by sheet name headers + +#### Scenario: Handle empty xlsx file +- **WHEN** an empty xlsx file is provided +- **THEN** the system returns an empty string + +#### Scenario: Handle numeric cell values +- **WHEN** a cell contains a numeric value +- **THEN** the system outputs the numeric value as a string in the markdown table + +#### Scenario: Handle merged cells +- **WHEN** a cell is merged with another cell +- **THEN** the system outputs the value in the top-left cell position and empty strings for merged positions + +### Requirement: XLSX to JSON conversion +The system SHALL convert `.xlsx` spreadsheets to JSON, preserving cell values, types, and sheet structure. + +#### Scenario: Extract xlsx to JSON with sheet names +- **WHEN** a multi-sheet xlsx file is provided +- **THEN** the system returns a JSON object with sheet names as keys and arrays of row objects as values + +#### Scenario: Preserve cell data types in JSON +- **WHEN** a cell contains a number, string, or boolean value +- **THEN** the system preserves the original data type in the JSON output + +#### Scenario: Handle empty rows in JSON output +- **WHEN** an xlsx file has empty rows +- **THEN** the system includes empty row objects in the JSON output diff --git a/openspec/changes/file-format-extraction-tools/tasks.md b/openspec/changes/file-format-extraction-tools/tasks.md new file mode 100644 index 00000000..26de141b --- /dev/null +++ b/openspec/changes/file-format-extraction-tools/tasks.md @@ -0,0 +1,49 @@ +## 1. Setup and Dependencies + +- [ ] 1.1 Add pdf-parse dependency to package.json +- [ ] 1.2 Create src/tools/fileExtract/ directory structure +- [ ] 1.3 Create extractZipXml utility function in src/tools/fileExtract/zipExtractor.js +- [ ] 1.4 Create format validation utility in src/tools/fileExtract/formatValidator.js + +## 2. DOCX Extraction Tool + +- [ ] 2.1 Implement docxToMarkdown parser in src/tools/fileExtract/docxParser.js +- [ ] 2.2 Handle headings, paragraphs, lists, tables, and inline formatting +- [ ] 2.3 Create docx tool wrapper function in src/tools/fileExtract/docx.js +- [ ] 2.4 Write unit tests for docx extraction with sample fixtures + +## 3. PPTX Extraction Tool + +- [ ] 3.1 Implement pptxToMarkdown parser in src/tools/fileExtract/pptxParser.js +- [ ] 3.2 Handle slide titles, bullet points, and speaker notes +- [ ] 3.3 Create pptx tool wrapper function in src/tools/fileExtract/pptx.js +- [ ] 3.4 Write unit tests for pptx extraction with sample fixtures + +## 4. XLSX Extraction Tool + +- [ ] 4.1 Implement xlsxToMarkdown parser in src/tools/fileExtract/xlsxParser.js +- [ ] 4.2 Handle single and multi-sheet conversion to markdown tables +- [ ] 4.3 Implement xlsxToJSON converter in src/tools/fileExtract/xlsxJson.js +- [ ] 4.4 Create xlsx tool wrapper function in src/tools/fileExtract/xlsx.js +- [ ] 4.5 Write unit tests for xlsx extraction with sample fixtures + +## 5. PDF Extraction Tool + +- [ ] 5.1 Implement pdfToMarkdown parser in src/tools/fileExtract/pdfParser.js +- [ ] 5.2 Handle text extraction, multi-page documents, and Unicode +- [ ] 5.3 Create pdf tool wrapper function in src/tools/fileExtract/pdf.js +- [ ] 5.4 Write unit tests for PDF extraction with sample fixtures + +## 6. Tool Registration and Integration + +- [ ] 6.1 Register all extraction tools in src/tools/index.js +- [ ] 6.2 Add TOOL_PERMISSIONS entries for all new tools +- [ ] 6.3 Add TOOL_CLASSIFICATIONS entries for all new tools +- [ ] 6.4 Verify tools are discoverable by the skills registry + +## 7. Testing and Verification + +- [ ] 7.1 Run npm test and verify all tests pass +- [ ] 7.2 Run npm run lint and fix any lint errors +- [ ] 7.3 Run npm start and verify application starts without crashing +- [ ] 7.4 Create sample fixture files for manual testing From b20d6a005a41af4a5bc5828564245735d55fedf2 Mon Sep 17 00:00:00 2001 From: Jason Mulligan Date: Wed, 12 Aug 2026 12:53:39 -0400 Subject: [PATCH 2/6] feat: implement file format extraction tools (docx, pptx, xlsx, pdf) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add shared ZIP/XML extraction utility (adm-zip) - Implement DOCX → markdown parser (headings, lists, tables, inline formatting) - Implement PPTX → markdown parser (slide titles, bullets, speaker notes) - Implement XLSX → markdown tables and XLSX → JSON converter - Implement PDF → markdown text extraction (pdf-parse) - Register all tools in src/tools/index.js with permissions and classifications - Add comprehensive unit tests for all modules - Add pdf-parse and xml2js dependencies --- package-lock.json | 274 ++++++++++++++++++ package.json | 3 + src/tools/fileExtract/docx.js | 76 +++++ src/tools/fileExtract/docxParser.js | 191 ++++++++++++ src/tools/fileExtract/formatValidator.js | 103 +++++++ src/tools/fileExtract/pdf.js | 72 +++++ src/tools/fileExtract/pdfParser.js | 62 ++++ src/tools/fileExtract/pptx.js | 69 +++++ src/tools/fileExtract/pptxParser.js | 127 ++++++++ src/tools/fileExtract/xlsx.js | 80 +++++ src/tools/fileExtract/xlsxJson.js | 136 +++++++++ src/tools/fileExtract/xlsxParser.js | 169 +++++++++++ src/tools/fileExtract/zipExtractor.js | 161 ++++++++++ src/tools/index.js | 16 + tests/unit/fileExtract/docx.test.js | 43 +++ .../unit/fileExtract/formatValidator.test.js | 72 +++++ tests/unit/fileExtract/pdf.test.js | 42 +++ tests/unit/fileExtract/pptx.test.js | 42 +++ tests/unit/fileExtract/xlsx.test.js | 42 +++ tests/unit/fileExtract/zipExtractor.test.js | 45 +++ 20 files changed, 1825 insertions(+) create mode 100644 src/tools/fileExtract/docx.js create mode 100644 src/tools/fileExtract/docxParser.js create mode 100644 src/tools/fileExtract/formatValidator.js create mode 100644 src/tools/fileExtract/pdf.js create mode 100644 src/tools/fileExtract/pdfParser.js create mode 100644 src/tools/fileExtract/pptx.js create mode 100644 src/tools/fileExtract/pptxParser.js create mode 100644 src/tools/fileExtract/xlsx.js create mode 100644 src/tools/fileExtract/xlsxJson.js create mode 100644 src/tools/fileExtract/xlsxParser.js create mode 100644 src/tools/fileExtract/zipExtractor.js create mode 100644 tests/unit/fileExtract/docx.test.js create mode 100644 tests/unit/fileExtract/formatValidator.test.js create mode 100644 tests/unit/fileExtract/pdf.test.js create mode 100644 tests/unit/fileExtract/pptx.test.js create mode 100644 tests/unit/fileExtract/xlsx.test.js create mode 100644 tests/unit/fileExtract/zipExtractor.test.js diff --git a/package-lock.json b/package-lock.json index 31dd0e32..debfa563 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@langchain/openai": "^1.5.6", "@opentelemetry/api": "^1.9.1", "@opentelemetry/sdk-node": "^0.221.0", + "adm-zip": "^0.5.16", "ansi-escapes": "^7.3.0", "ansi-regex": "^6.2.2", "chalk": "^6.0.0", @@ -28,9 +29,11 @@ "js-yaml": "^5.2.3", "marked": "^18.0.9", "node-emoji": "^2.2.0", + "pdf-parse": "^2.0.0", "pino": "^10.3.1", "supports-hyperlinks": "^4.5.0", "tiktoken": "^1.0.22", + "xml2js": "^0.6.2", "yargs": "^18.1.0", "zod": "^4.4.3" }, @@ -391,6 +394,205 @@ "integrity": "sha512-XW1egQtPfsGI41w2AMZNFZrUIwFSQHTjVMZs0OaTpCAvht/QLoaPN8FQcsysMVypOhupG28J29yOorrc70otBQ==", "license": "MIT" }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.80.tgz", + "integrity": "sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==", + "license": "MIT", + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.80", + "@napi-rs/canvas-darwin-arm64": "0.1.80", + "@napi-rs/canvas-darwin-x64": "0.1.80", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.80", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.80", + "@napi-rs/canvas-linux-arm64-musl": "0.1.80", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.80", + "@napi-rs/canvas-linux-x64-gnu": "0.1.80", + "@napi-rs/canvas-linux-x64-musl": "0.1.80", + "@napi-rs/canvas-win32-x64-msvc": "0.1.80" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.80.tgz", + "integrity": "sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.80.tgz", + "integrity": "sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.80.tgz", + "integrity": "sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.80.tgz", + "integrity": "sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.80.tgz", + "integrity": "sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.80.tgz", + "integrity": "sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.80.tgz", + "integrity": "sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.80.tgz", + "integrity": "sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.80.tgz", + "integrity": "sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.80.tgz", + "integrity": "sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1726,6 +1928,15 @@ "undici-types": "~8.3.0" } }, + "node_modules/adm-zip": { + "version": "0.5.18", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.18.tgz", + "integrity": "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, "node_modules/ansi-escapes": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz", @@ -3409,6 +3620,38 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, + "node_modules/pdf-parse": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-2.4.5.tgz", + "integrity": "sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==", + "license": "Apache-2.0", + "dependencies": { + "@napi-rs/canvas": "0.1.80", + "pdfjs-dist": "5.4.296" + }, + "bin": { + "pdf-parse": "bin/cli.mjs" + }, + "engines": { + "node": ">=20.16.0 <21 || >=22.3.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/mehmet-kozan" + } + }, + "node_modules/pdfjs-dist": { + "version": "5.4.296", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz", + "integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.16.0 || >=22.3.0" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.80" + } + }, "node_modules/picomatch": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", @@ -3733,6 +3976,15 @@ "node": ">=10" } }, + "node_modules/sax": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.1.tgz", + "integrity": "sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=11.0.0" + } + }, "node_modules/scheduler": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", @@ -4218,6 +4470,28 @@ } } }, + "node_modules/xml2js": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "license": "MIT", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "license": "MIT", + "engines": { + "node": ">=4.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 88b0bbf7..08d9eaf8 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,7 @@ "@opentelemetry/sdk-node": "^0.221.0", "ansi-escapes": "^7.3.0", "ansi-regex": "^6.2.2", + "adm-zip": "^0.5.16", "chalk": "^6.0.0", "cli-highlight": "^2.1.11", "cli-table3": "^0.6.5", @@ -80,9 +81,11 @@ "marked": "^18.0.9", "node-emoji": "^2.2.0", "pino": "^10.3.1", + "pdf-parse": "^2.0.0", "supports-hyperlinks": "^4.5.0", "tiktoken": "^1.0.22", "yargs": "^18.1.0", + "xml2js": "^0.6.2", "zod": "^4.4.3" } } diff --git a/src/tools/fileExtract/docx.js b/src/tools/fileExtract/docx.js new file mode 100644 index 00000000..41882d2c --- /dev/null +++ b/src/tools/fileExtract/docx.js @@ -0,0 +1,76 @@ +/** + * DOCX file extraction tool. + * Extracts content from Microsoft Word documents (.docx) to markdown. + * @module fileExtract/docx + */ + +import { z } from "zod"; +import { tool } from "@langchain/core/tools"; +import { readFile } from "node:fs/promises"; +import { extractZipXml } from "./zipExtractor.js"; +import { docxToMarkdown, extractDocxTables } from "./docxParser.js"; +import { validateFormat, getExtension } from "./formatValidator.js"; + +/** + * Input schema for the docx tool. + */ +export const docxSchema = z.object({ + filePath: z.string().describe("Absolute path to the .docx file"), +}); + +/** + * Extract content from a DOCX file. + * @param {object} input - Tool input + * @param {string} input.filePath - Path to the DOCX file + * @returns {Promise} JSON result string + */ +export async function docxExtract(input) { + const { filePath } = docxSchema.parse(input); + + // Validate format + const validation = validateFormat(filePath); + if (!validation.valid) { + return JSON.stringify({ ok: false, error: validation.error }); + } + + // Read file + let buffer; + try { + buffer = await readFile(filePath); + } catch (err) { + return JSON.stringify({ ok: false, error: `Failed to read file: ${err.message}` }); + } + + // Extract ZIP content + let zipContent; + try { + zipContent = await extractZipXml(filePath); + } catch (err) { + return JSON.stringify({ ok: false, error: `ZIP extraction failed: ${err.message}` }); + } + + // Extract document content + const documentXml = zipContent.get("word/document.xml"); + if (!documentXml) { + return JSON.stringify({ ok: false, error: "No word/document.xml found in archive" }); + } + + const markdown = docxToMarkdown(documentXml); + const tables = extractDocxTables(documentXml); + const content = markdown + tables; + + return JSON.stringify({ + ok: true, + format: "markdown", + content: content || "(empty document)", + }); +} + +/** + * LangChain Tool instance for DOCX extraction. + */ +export const docxTool = tool(docxExtract, { + name: "docx", + description: "Extract content from a Microsoft Word (.docx) file to markdown. Accepts a file path and returns structured markdown with headings, paragraphs, lists, and tables.", + schema: docxSchema, +}); diff --git a/src/tools/fileExtract/docxParser.js b/src/tools/fileExtract/docxParser.js new file mode 100644 index 00000000..43127182 --- /dev/null +++ b/src/tools/fileExtract/docxParser.js @@ -0,0 +1,191 @@ +/** + * DOCX to Markdown parser. + * Converts DOCX document content to structured markdown. + * @module fileExtract/docxParser + */ + +import { extractZipXml } from "./zipExtractor.js"; +import { ZipExtractionError } from "./zipExtractor.js"; +import { parseStringPromise } from "xml2js"; + +/** + * Convert DOCX XML content to markdown. + * @param {string} documentXml - The word/document.xml content + * @returns {string} Markdown string + */ +export function docxToMarkdown(documentXml) { + if (!documentXml || !documentXml.trim()) { + return ""; + } + + let markdown = ""; + let inList = false; + let listType = null; + + try { + const parsed = parseStringPromise(documentXml, { + mergeAttrs: true, + explicitArray: false, + }); + + const body = parsed?.w?.document?.[0]?.["w:body"] || parsed?.w?.document?.["w:body"]; + if (!body) return ""; + + const paragraphs = body["w:p"] || []; + + for (const para of paragraphs) { + const textContent = extractParagraphText(para); + const headingLevel = getHeadingLevel(para); + const isListItem = isListItem(para); + + if (headingLevel > 0) { + // Close any open list + if (inList) { + markdown += "\n"; + inList = false; + listType = null; + } + markdown += `${"#".repeat(headingLevel)} ${textContent}\n\n`; + } else if (isListItem) { + if (!inList) { + inList = true; + listType = "ul"; + } + markdown += `- ${textContent}\n`; + } else if (textContent.trim()) { + if (inList) { + markdown += "\n"; + inList = false; + listType = null; + } + markdown += `${textContent}\n\n`; + } + } + } catch { + // If XML parsing fails, return raw text + const textMatch = documentXml.match(/>([^<]+) m.replace(/^>| 0) { + markdown += "\n| " + cells[0].join(" | ") + " |\n"; + markdown += "| " + cells[0].map(() => "---").join(" | ") + " |\n"; + for (let i = 1; i < cells.length; i++) { + markdown += "| " + cells[i].join(" | ") + " |\n"; + } + markdown += "\n"; + } + } + } catch { + // Silently skip table extraction on parse errors + } + + return markdown; +} diff --git a/src/tools/fileExtract/formatValidator.js b/src/tools/fileExtract/formatValidator.js new file mode 100644 index 00000000..2428dac9 --- /dev/null +++ b/src/tools/fileExtract/formatValidator.js @@ -0,0 +1,103 @@ +/** + * File format validation utility. + * Validates that a file is a supported ZIP-based format before extraction. + * @module fileExtract/formatValidator + */ + +/** + * Supported ZIP-based file extensions. + */ +export const ZIP_FORMATS = new Set(["docx", "pptx", "xlsx", "odt", "ods", "odp", "epub"]); + +/** + * Supported PDF file extension. + */ +export const PDF_FORMATS = new Set(["pdf"]); + +/** + * All supported file extensions. + */ +export const SUPPORTED_FORMATS = new Set([...ZIP_FORMATS, ...PDF_FORMATS]); + +/** + * ZIP-based formats that use shared XML extraction. + */ +export const ZIP_XML_FORMATS = new Set(["docx", "pptx", "xlsx", "odt", "ods", "odp"]); + +/** + * Internal XML paths for ZIP-based formats. + * Maps format → array of internal XML paths to extract. + */ +export const INTERNAL_XML_PATHS = { + docx: ["word/document.xml", "word/styles.xml", "word/numbering.xml"], + pptx: ["ppt/slides/slide*.xml", "ppt/slideLayouts/slideLayout*.xml", "ppt/slideMasters/slideMaster*.xml", "ppt/presentation.xml", "ppt/presentationNotesSlides/notesSlide*.xml"], + xlsx: ["xl/workbook.xml", "xl/worksheets/sheet*.xml", "xl/sharedStrings.xml"], + odt: ["content.xml", "styles.xml"], + ods: ["content.xml", "styles.xml"], + odp: ["content.xml", "styles.xml", "meta.xml"], + epub: ["OEBPS/content.opf", "OEBPS/toc.ncx"], +}; + +/** + * Validate that a file path has a supported extension. + * @param {string} filePath - Path to the file + * @returns {{ valid: boolean, format?: string, error?: string }} + */ +export function validateFormat(filePath) { + const ext = getExtension(filePath); + + if (!ext) { + return { + valid: false, + error: `No file extension found in path: ${filePath}`, + }; + } + + if (!SUPPORTED_FORMATS.has(ext)) { + return { + valid: false, + error: `Unsupported format: .${ext}. Supported formats: ${[...SUPPORTED_FORMATS].sort().join(", ")}`, + }; + } + + return { valid: true, format: ext }; +} + +/** + * Check if a format is ZIP-based. + * @param {string} format - File extension (lowercase) + * @returns {boolean} + */ +export function isZipFormat(format) { + return ZIP_FORMATS.has(format); +} + +/** + * Check if a format uses shared XML extraction. + * @param {string} format - File extension (lowercase) + * @returns {boolean} + */ +export function usesXmlExtraction(format) { + return ZIP_XML_FORMATS.has(format); +} + +/** + * Get the internal XML paths for a ZIP-based format. + * @param {string} format - File extension (lowercase) + * @returns {string[] | null} Array of internal paths or null if not a ZIP format + */ +export function getInternalPaths(format) { + return INTERNAL_XML_PATHS[format] || null; +} + +/** + * Extract the file extension from a path. + * @param {string} filePath - File path + * @returns {string | null} Lowercase extension or null + */ +export function getExtension(filePath) { + const basename = filePath.split("/").pop() || filePath.split("\\").pop() || ""; + const dotIndex = basename.lastIndexOf("."); + if (dotIndex <= 0) return null; + return basename.slice(dotIndex + 1).toLowerCase(); +} diff --git a/src/tools/fileExtract/pdf.js b/src/tools/fileExtract/pdf.js new file mode 100644 index 00000000..a468aec3 --- /dev/null +++ b/src/tools/fileExtract/pdf.js @@ -0,0 +1,72 @@ +/** + * PDF file extraction tool. + * Extracts text content from PDF files to markdown. + * @module fileExtract/pdf + */ + +import { z } from "zod"; +import { tool } from "@langchain/core/tools"; +import { readFile } from "node:fs/promises"; +import { pdfToMarkdown, PdfExtractionError } from "./pdfParser.js"; +import { validateFormat } from "./formatValidator.js"; + +/** + * Input schema for the pdf tool. + */ +export const pdfSchema = z.object({ + filePath: z.string().describe("Absolute path to the .pdf file"), +}); + +/** + * Extract content from a PDF file. + * @param {object} input - Tool input + * @param {string} input.filePath - Path to the PDF file + * @returns {Promise} JSON result string + */ +export async function pdfExtract(input) { + const { filePath } = pdfSchema.parse(input); + + // Validate format + const validation = validateFormat(filePath); + if (!validation.valid) { + return JSON.stringify({ ok: false, error: validation.error }); + } + + // Read file + let buffer; + try { + buffer = await readFile(filePath); + } catch (err) { + return JSON.stringify({ ok: false, error: `Failed to read file: ${err.message}` }); + } + + // Extract text + let markdown; + try { + markdown = await pdfToMarkdown(buffer); + } catch (err) { + if (err instanceof PdfExtractionError) { + return JSON.stringify({ + ok: false, + error: err.message, + reason: err.reason, + }); + } + return JSON.stringify({ ok: false, error: `PDF extraction failed: ${err.message}` }); + } + + return JSON.stringify({ + ok: true, + format: "markdown", + content: markdown || "(no extractable text)", + }); +} + +/** + * LangChain Tool instance for PDF extraction. + */ +export const pdfTool = tool(pdfExtract, { + name: "pdf", + description: "Extract text content from a PDF file to markdown. Handles multi-page documents, Unicode characters, and special characters. Returns an error for scanned/image-only PDFs.", + schema: pdfSchema, +}); diff --git a/src/tools/fileExtract/pdfParser.js b/src/tools/fileExtract/pdfParser.js new file mode 100644 index 00000000..17d4156f --- /dev/null +++ b/src/tools/fileExtract/pdfParser.js @@ -0,0 +1,62 @@ +/** + * PDF to Markdown parser. + * Extracts text content from PDF files and outputs markdown. + * @module fileExtract/pdfParser + */ + +import { PDFParse } from "pdf-parse"; + +/** + * Error thrown when PDF extraction fails. + */ +export class PdfExtractionError extends Error { + /** + * @param {string} message - Error message + * @param {string} [reason] - Reason for the failure + */ + constructor(message, reason) { + super(message); + this.name = "PdfExtractionError"; + this.reason = reason || null; + } +} + +/** + * Convert PDF buffer to markdown. + * @param {Buffer} buffer - PDF file buffer + * @returns {Promise} Markdown string + */ +export async function pdfToMarkdown(buffer) { + if (!buffer || buffer.length === 0) { + return ""; + } + + try { + const parser = new PDFParse({ verbosity: 0 }); + const result = await parser.getText(); + + if (!result.text || !result.text.trim()) { + await parser.destroy(); + throw new PdfExtractionError("No extractable text found in PDF", "no-text"); + } + + // Clean up the extracted text into markdown paragraphs + const paragraphs = result.text + .split(/\n\s*\n/) + .map((p) => p.trim()) + .filter((p) => p.length > 0); + + const markdown = paragraphs.join("\n\n"); + await parser.destroy(); + return markdown; + } catch (err) { + if (err instanceof PdfExtractionError) throw err; + + // Check for password-protected PDF + if (err.message && err.message.toLowerCase().includes("password")) { + throw new PdfExtractionError("PDF is password-protected", "password-protected"); + } + + throw new PdfExtractionError(`PDF extraction failed: ${err.message}`, "extraction-failed"); + } +} diff --git a/src/tools/fileExtract/pptx.js b/src/tools/fileExtract/pptx.js new file mode 100644 index 00000000..d35b749f --- /dev/null +++ b/src/tools/fileExtract/pptx.js @@ -0,0 +1,69 @@ +/** + * PPTX file extraction tool. + * Extracts content from PowerPoint presentations (.pptx) to markdown. + * @module fileExtract/pptx + */ + +import { z } from "zod"; +import { tool } from "@langchain/core/tools"; +import { readFile } from "node:fs/promises"; +import { extractZipXml } from "./zipExtractor.js"; +import { pptxToMarkdown } from "./pptxParser.js"; +import { validateFormat } from "./formatValidator.js"; + +/** + * Input schema for the pptx tool. + */ +export const pptxSchema = z.object({ + filePath: z.string().describe("Absolute path to the .pptx file"), +}); + +/** + * Extract content from a PPTX file. + * @param {object} input - Tool input + * @param {string} input.filePath - Path to the PPTX file + * @returns {Promise} JSON result string + */ +export async function pptxExtract(input) { + const { filePath } = pptxSchema.parse(input); + + // Validate format + const validation = validateFormat(filePath); + if (!validation.valid) { + return JSON.stringify({ ok: false, error: validation.error }); + } + + // Read file + let buffer; + try { + buffer = await readFile(filePath); + } catch (err) { + return JSON.stringify({ ok: false, error: `Failed to read file: ${err.message}` }); + } + + // Extract ZIP content + let zipContent; + try { + zipContent = await extractZipXml(filePath); + } catch (err) { + return JSON.stringify({ ok: false, error: `ZIP extraction failed: ${err.message}` }); + } + + // Extract presentation content + const markdown = pptxToMarkdown(zipContent); + + return JSON.stringify({ + ok: true, + format: "markdown", + content: markdown || "(empty presentation)", + }); +} + +/** + * LangChain Tool instance for PPTX extraction. + */ +export const pptxTool = tool(pptxExtract, { + name: "pptx", + description: "Extract content from a PowerPoint (.pptx) file to markdown. Returns slide titles, bullet points, and speaker notes.", + schema: pptxSchema, +}); diff --git a/src/tools/fileExtract/pptxParser.js b/src/tools/fileExtract/pptxParser.js new file mode 100644 index 00000000..a545934b --- /dev/null +++ b/src/tools/fileExtract/pptxParser.js @@ -0,0 +1,127 @@ +/** + * PPTX to Markdown parser. + * Converts PowerPoint presentations to structured markdown. + * @module fileExtract/pptxParser + */ + +import { extractZipXml } from "./zipExtractor.js"; +import { parseStringPromise } from "xml2js"; + +/** + * Convert PPTX to markdown. + * @param {Map} zipContent - Map of internal path → content + * @returns {string} Markdown string + */ +export function pptxToMarkdown(zipContent) { + let markdown = ""; + let slideIndex = 0; + + // Find all slide files + const slideFiles = []; + for (const path of zipContent.keys()) { + if (/^ppt\/slides\/slide\d+\.xml$/.test(path)) { + slideFiles.push(path); + } + } + slideFiles.sort(); + + for (const slidePath of slideFiles) { + const slideXml = zipContent.get(slidePath); + if (!slideXml) continue; + + slideIndex++; + markdown += `---\n\n`; + + try { + const parsed = parseStringPromise(slideXml, { + mergeAttrs: true, + explicitArray: false, + }); + + const spTree = parsed?.p?.slide?.[0]?.["p:spTree"] || parsed?.p?.slide?.["p:spTree"]; + if (!spTree) continue; + + const shapes = spTree["p:sp"] || []; + const shapeArray = Array.isArray(shapes) ? shapes : [shapes]; + + let titleFound = false; + + for (const shape of shapeArray) { + const nm = shape?.$?.name; + if (nm === "title") { + const text = extractShapeText(shape); + if (text) { + markdown += `# ${text}\n\n`; + titleFound = true; + } + } else { + const text = extractShapeText(shape); + if (text) { + markdown += `- ${text}\n`; + } + } + } + + // Extract speaker notes if available + const notesPath = slidePath.replace("slides/slide", "slideNotesSlides/notesSlide"); + const notesXml = zipContent.get(notesPath); + if (notesXml) { + try { + const notesParsed = parseStringPromise(notesXml, { + mergeAttrs: true, + explicitArray: false, + }); + const notesBody = notesParsed?.p?.notesSlide?.[0]?.["p:spTree"] || notesParsed?.p?.notesSlide?.["p:spTree"]; + if (notesBody) { + const notesShapes = notesBody["p:sp"] || []; + const notesArray = Array.isArray(notesShapes) ? notesShapes : [notesShapes]; + for (const shape of notesArray) { + const text = extractShapeText(shape); + if (text) { + markdown += `\n> **Speaker Notes:** ${text}\n`; + } + } + } + } catch { + // Skip notes on parse error + } + } + + if (!titleFound) { + markdown = markdown.replace(`---\n\n`, `---\n\n`); + } + } catch { + markdown += `## Slide ${slideIndex} (parse error)\n\n`; + } + } + + return markdown.trim(); +} + +/** + * Extract text content from a shape element. + * @param {object} shape - Parsed shape XML object + * @returns {string} Text content + */ +function extractShapeText(shape) { + const body = shape?.txBody || shape?.["p:txBody"]; + if (!body) return ""; + + const paragraphs = body["a:p"] || []; + const paraArray = Array.isArray(paragraphs) ? paragraphs : [paragraphs]; + + let text = ""; + for (const para of paraArray) { + const runs = para["a:r"] || []; + const runArray = Array.isArray(runs) ? runs : [runs]; + + for (const run of runArray) { + const t = run["a:t"]; + if (t && typeof t === "string") { + text += t; + } + } + } + + return text.trim(); +} diff --git a/src/tools/fileExtract/xlsx.js b/src/tools/fileExtract/xlsx.js new file mode 100644 index 00000000..a4f47cae --- /dev/null +++ b/src/tools/fileExtract/xlsx.js @@ -0,0 +1,80 @@ +/** + * XLSX file extraction tool. + * Extracts content from Excel spreadsheets (.xlsx) to markdown tables and JSON. + * @module fileExtract/xlsx + */ + +import { z } from "zod"; +import { tool } from "@langchain/core/tools"; +import { readFile } from "node:fs/promises"; +import { extractZipXml } from "./zipExtractor.js"; +import { xlsxToMarkdown } from "./xlsxParser.js"; +import { xlsxToJson } from "./xlsxJson.js"; +import { validateFormat } from "./formatValidator.js"; + +/** + * Input schema for the xlsx tool. + */ +export const xlsxSchema = z.object({ + filePath: z.string().describe("Absolute path to the .xlsx file"), + format: z.enum(["markdown", "json"]).optional().default("markdown").describe("Output format: 'markdown' for tables, 'json' for structured data"), +}); + +/** + * Extract content from an XLSX file. + * @param {object} input - Tool input + * @param {string} input.filePath - Path to the XLSX file + * @param {"markdown"|"json"} [input.format="markdown"] - Output format + * @returns {Promise} JSON result string + */ +export async function xlsxExtract(input) { + const { filePath, format = "markdown" } = xlsxSchema.parse(input); + + // Validate format + const validation = validateFormat(filePath); + if (!validation.valid) { + return JSON.stringify({ ok: false, error: validation.error }); + } + + // Read file + let buffer; + try { + buffer = await readFile(filePath); + } catch (err) { + return JSON.stringify({ ok: false, error: `Failed to read file: ${err.message}` }); + } + + // Extract ZIP content + let zipContent; + try { + zipContent = await extractZipXml(filePath); + } catch (err) { + return JSON.stringify({ ok: false, error: `ZIP extraction failed: ${err.message}` }); + } + + if (format === "json") { + const jsonData = xlsxToJson(zipContent); + return JSON.stringify({ + ok: true, + format: "json", + content: JSON.stringify(jsonData, null, 2), + }); + } + + const markdown = xlsxToMarkdown(zipContent); + + return JSON.stringify({ + ok: true, + format: "markdown", + content: markdown || "(empty spreadsheet)", + }); +} + +/** + * LangChain Tool instance for XLSX extraction. + */ +export const xlsxTool = tool(xlsxExtract, { + name: "xlsx", + description: "Extract content from an Excel (.xlsx) file. Supports markdown table output (default) or JSON output. Returns sheet data as tables or structured JSON objects.", + schema: xlsxSchema, +}); diff --git a/src/tools/fileExtract/xlsxJson.js b/src/tools/fileExtract/xlsxJson.js new file mode 100644 index 00000000..3ec506cb --- /dev/null +++ b/src/tools/fileExtract/xlsxJson.js @@ -0,0 +1,136 @@ +/** + * XLSX to JSON converter. + * Converts Excel spreadsheets to structured JSON. + * @module fileExtract/xlsxJson + */ + +import { extractZipXml } from "./zipExtractor.js"; +import { parseStringPromise } from "xml2js"; + +/** + * Convert XLSX to JSON object with sheet names as keys. + * @param {Map} zipContent - Map of internal path → content + * @returns {object} JSON object with sheet data + */ +export function xlsxToJson(zipContent) { + const result = {}; + + const workbookXml = zipContent.get("xl/workbook.xml"); + if (!workbookXml) return result; + + try { + const parsed = parseStringPromise(workbookXml, { + mergeAttrs: true, + explicitArray: false, + }); + + const sheets = parsed?.workbook?.[0]?.sheets?.sheet; + if (!sheets) return result; + + const sheetArray = Array.isArray(sheets) ? sheets : [sheets]; + + for (const sheetDef of sheetArray) { + const sheetName = sheetDef?.$?.name || "Sheet"; + const sheetId = sheetDef?.$?.sheetId || "1"; + + const sheetXml = findSheetXml(zipContent, sheetId); + if (!sheetXml) continue; + + const rows = parseSheetRows(sheetXml); + result[sheetName] = rows; + } + } catch { + // Silently skip on parse error + } + + return result; +} + +/** + * Find the sheet XML file by sheet ID. + * @param {Map} zipContent - Map of internal path → content + * @param {string} sheetId - Sheet ID + * @returns {string | null} + */ +function findSheetXml(zipContent, sheetId) { + const patterns = [ + `xl/worksheets/sheet${sheetId}.xml`, + ]; + + for (const pattern of patterns) { + if (zipContent.has(pattern)) { + return zipContent.get(pattern); + } + } + + for (const path of zipContent.keys()) { + if (/^xl\/worksheets\/sheet\d+\.xml$/.test(path)) { + return zipContent.get(path); + } + } + + return null; +} + +/** + * Parse sheet XML into array of row objects. + * @param {string} sheetXml - Sheet XML content + * @returns {object[]} Array of row objects + */ +function parseSheetRows(sheetXml) { + try { + const parsed = parseStringPromise(sheetXml, { + mergeAttrs: true, + explicitArray: false, + }); + + const sheetData = parsed?.sheet?.[0]?.sheetData?.row; + if (!sheetData) return []; + + const rowArray = Array.isArray(sheetData) ? sheetData : [sheetData]; + const rows = []; + + for (const row of rowArray) { + const cells = row?.c || []; + const cellArray = Array.isArray(cells) ? cells : [cells]; + const rowData = {}; + + for (const cell of cellArray) { + const ref = cell?.$?.r; + if (ref) { + rowData[ref] = getCellValue(cell); + } + } + + rows.push(rowData); + } + + return rows; + } catch { + return []; + } +} + +/** + * Get the value of a cell. + * @param {object} cell - Parsed cell XML object + * @returns {string | number | boolean} Cell value with type preservation + */ +function getCellValue(cell) { + const v = cell?.v; + if (v === undefined || v === null) return ""; + + const strVal = String(v); + + // Check type + const t = cell?.$?.t || cell?.t; + if (t === "b") { + return strVal === "1" ? true : false; + } + if (t === "n") { + const num = Number(strVal); + return isNaN(num) ? strVal : num; + } + + return strVal; +} diff --git a/src/tools/fileExtract/xlsxParser.js b/src/tools/fileExtract/xlsxParser.js new file mode 100644 index 00000000..f96c11cc --- /dev/null +++ b/src/tools/fileExtract/xlsxParser.js @@ -0,0 +1,169 @@ +/** + * XLSX to Markdown parser. + * Converts Excel spreadsheets to markdown tables. + * @module fileExtract/xlsxParser + */ + +import { extractZipXml } from "./zipExtractor.js"; +import { parseStringPromise } from "xml2js"; + +/** + * Convert XLSX to markdown tables. + * @param {Map} zipContent - Map of internal path → content + * @returns {string} Markdown string with tables per sheet + */ +export function xlsxToMarkdown(zipContent) { + let markdown = ""; + + const workbookXml = zipContent.get("xl/workbook.xml"); + if (!workbookXml) return ""; + + try { + const parsed = parseStringPromise(workbookXml, { + mergeAttrs: true, + explicitArray: false, + }); + + const sheets = parsed?.workbook?.[0]?.sheets?.sheet; + if (!sheets) return ""; + + const sheetArray = Array.isArray(sheets) ? sheets : [sheets]; + + for (const sheetDef of sheetArray) { + const sheetName = sheetDef?.$?.name || "Sheet"; + const sheetId = sheetDef?.$?.sheetId || "1"; + + // Find the corresponding sheet XML file + const sheetXml = findSheetXml(zipContent, sheetId); + if (!sheetXml) continue; + + const table = parseSheetContent(sheetXml); + if (table && table.rows.length > 0) { + markdown += `## ${sheetName}\n\n`; + markdown += toMarkdownTable(table.rows); + markdown += "\n\n"; + } + } + } catch { + // Silently skip on parse error + } + + return markdown.trim(); +} + +/** + * Find the sheet XML file by sheet ID. + * @param {Map} zipContent - Map of internal path → content + * @param {string} sheetId - Sheet ID + * @returns {string | null} Sheet XML content or null + */ +function findSheetXml(zipContent, sheetId) { + // Try common patterns + const patterns = [ + `xl/worksheets/sheet${sheetId}.xml`, + `xl/worksheets/sheet${sheetId}.xml`, + ]; + + for (const pattern of patterns) { + if (zipContent.has(pattern)) { + return zipContent.get(pattern); + } + } + + // Fallback: scan all sheet files + for (const path of zipContent.keys()) { + if (/^xl\/worksheets\/sheet\d+\.xml$/.test(path)) { + return zipContent.get(path); + } + } + + return null; +} + +/** + * Parse sheet XML content into rows. + * @param {string} sheetXml - Sheet XML content + * @returns {{ rows: string[][] } | null} Parsed rows or null + */ +function parseSheetContent(sheetXml) { + try { + const parsed = parseStringPromise(sheetXml, { + mergeAttrs: true, + explicitArray: false, + }); + + const sheetData = parsed?.sheet?.[0]?.sheetData?.row; + if (!sheetData) return null; + + const rowArray = Array.isArray(sheetData) ? sheetData : [sheetData]; + const rows = []; + + for (const row of rowArray) { + const cells = row?.c || []; + const cellArray = Array.isArray(cells) ? cells : [cells]; + const rowData = []; + + for (const cell of cellArray) { + rowData.push(getCellValue(cell)); + } + + rows.push(rowData); + } + + return { rows }; + } catch { + return null; + } +} + +/** + * Get the value of a cell. + * @param {object} cell - Parsed cell XML object + * @returns {string} Cell value + */ +function getCellValue(cell) { + const v = cell?.v; + if (v !== undefined && v !== null) { + return String(v); + } + + // Check for t="inlineStr" + const t = cell?.$?.t || cell?.t; + if (t === "inlineStr") { + const is = cell?.is; + if (is) { + const tEl = is?.t || is?.["t"]; + return tEl || ""; + } + } + + return ""; +} + +/** + * Convert rows to markdown table format. + * @param {string[][]} rows - 2D array of cell values + * @returns {string} Markdown table + */ +function toMarkdownTable(rows) { + if (rows.length === 0) return ""; + + const maxCols = Math.max(...rows.map((r) => r.length)); + const normalizedRows = rows.map((r) => { + while (r.length < maxCols) r.push(""); + return r; + }); + + let table = ""; + + // Header row + table += "| " + normalizedRows[0].join(" | ") + " |\n"; + table += "| " + normalizedRows[0].map(() => "---").join(" | ") + " |\n"; + + // Data rows + for (let i = 1; i < normalizedRows.length; i++) { + table += "| " + normalizedRows[i].join(" | ") + " |\n"; + } + + return table; +} diff --git a/src/tools/fileExtract/zipExtractor.js b/src/tools/fileExtract/zipExtractor.js new file mode 100644 index 00000000..ba97742d --- /dev/null +++ b/src/tools/fileExtract/zipExtractor.js @@ -0,0 +1,161 @@ +/** + * ZIP archive extraction utility. + * Decompresses ZIP-based document archives and extracts XML content. + * Uses adm-zip (pure JS) for maximum compatibility. + * @module fileExtract/zipExtractor + */ + +import AdmZip from "adm-zip"; +import { ZIP_FORMATS } from "./formatValidator.js"; + +/** + * Error thrown when ZIP extraction fails. + */ +export class ZipExtractionError extends Error { + /** + * @param {string} message - Error message + * @param {string} [reason] - Reason for the failure + */ + constructor(message, reason) { + super(message); + this.name = "ZipExtractionError"; + this.reason = reason || null; + } +} + +/** + * Validate that a file is a supported ZIP-based format. + * @param {string} filePath - Path to the file + * @returns {{ valid: boolean, format?: string, error?: string }} + */ +export function validateZip(filePath) { + const ext = getExtension(filePath); + + if (!ext) { + return { + valid: false, + error: `No file extension found in path: ${filePath}`, + }; + } + + if (!ZIP_FORMATS.has(ext)) { + return { + valid: false, + error: `Unsupported format: .${ext}. ZIP extraction supports: ${[...ZIP_FORMATS].sort().join(", ")}`, + }; + } + + return { valid: true, format: ext }; +} + +/** + * Get the list of file names inside a ZIP archive. + * @param {string} filePath - Path to the ZIP file + * @returns {Promise} Array of file names in the archive + */ +export async function getZipFileNames(filePath) { + try { + const zip = new AdmZip(filePath); + return zip.getEntries().map((entry) => entry.entryName); + } catch (err) { + return []; + } +} + +/** + * Extract all XML content from a ZIP archive. + * @param {string} filePath - Path to the ZIP file + * @returns {Promise>} Map of filename to content + */ +export async function extractZipXml(filePath) { + const result = new Map(); + + try { + const zip = new AdmZip(filePath); + const entries = zip.getEntries(); + + for (const entry of entries) { + if (entry.isDirectory) continue; + const content = entry.getData().toString("utf-8"); + result.set(entry.entryName, content); + } + } catch (err) { + throw new ZipExtractionError(`Failed to extract ZIP: ${err.message}`, "extraction-failed"); + } + + return result; +} + +/** + * Extract a specific file from a ZIP archive. + * @param {string} filePath - Path to the ZIP file + * @param {string} internalPath - Internal path within the ZIP (e.g., "word/document.xml") + * @returns {Promise} File content + */ +export async function extractZipFile(filePath, internalPath) { + try { + const zip = new AdmZip(filePath); + const entry = zip.getEntry(internalPath); + + if (!entry) { + throw new ZipExtractionError(`File not found in archive: ${internalPath}`, "file-not-found"); + } + + return entry.getData().toString("utf-8"); + } catch (err) { + if (err instanceof ZipExtractionError) throw err; + throw new ZipExtractionError(`Failed to extract file from ZIP: ${err.message}`, "extraction-failed"); + } +} + +/** + * Extract a file matching a glob pattern from a ZIP archive. + * @param {string} filePath - Path to the ZIP file + * @param {string} pattern - Glob pattern (e.g., "ppt/slides/slide*.xml") + * @returns {Promise>} Map of filename to content + */ +export async function extractZipGlob(filePath, pattern) { + const result = new Map(); + const regex = patternToRegex(pattern); + + try { + const zip = new AdmZip(filePath); + const entries = zip.getEntries(); + + for (const entry of entries) { + if (entry.isDirectory) continue; + if (regex.test(entry.entryName)) { + result.set(entry.entryName, entry.getData().toString("utf-8")); + } + } + } catch (err) { + throw new ZipExtractionError(`Failed to extract from ZIP: ${err.message}`, "extraction-failed"); + } + + return result; +} + +/** + * Convert a glob pattern to a regex. + * @param {string} pattern - Glob pattern + * @returns {RegExp} + */ +function patternToRegex(pattern) { + const escaped = pattern + .replace(/\./g, "\\.") + .replace(/\*/g, ".*") + .replace(/\?/g, "."); + return new RegExp(`^${escaped}$`); +} + +/** + * Extract the file extension from a path. + * @param {string} filePath - File path + * @returns {string | null} Lowercase extension or null + */ +function getExtension(filePath) { + const basename = filePath.split("/").pop() || filePath.split("\\").pop() || ""; + const dotIndex = basename.lastIndexOf("."); + if (dotIndex <= 0) return null; + return basename.slice(dotIndex + 1).toLowerCase(); +} diff --git a/src/tools/index.js b/src/tools/index.js index 033e361f..9040129d 100644 --- a/src/tools/index.js +++ b/src/tools/index.js @@ -14,6 +14,10 @@ import { createSkill } from "./skills.js"; import { textToSpeech } from "./tts.js"; import { visionAnalyze } from "./vision.js"; import { webSearch, webExtract } from "./web.js"; +import { docxTool } from "./fileExtract/docx.js"; +import { pptxTool } from "./fileExtract/pptx.js"; +import { xlsxTool } from "./fileExtract/xlsx.js"; +import { pdfTool } from "./fileExtract/pdf.js"; /** * Maps tool names to required permission scopes. @@ -38,6 +42,10 @@ export const TOOL_PERMISSIONS = { visionAnalyze: [], webExtract: ["network:outbound"], webSearch: ["network:outbound"], + docx: ["filesystem:read"], + pptx: ["filesystem:read"], + xlsx: ["filesystem:read"], + pdf: ["filesystem:read"], }; /** @@ -93,6 +101,10 @@ export const TOOL_CLASSIFICATIONS = { visionAnalyze: ["code-review", "testing", "coding"], webExtract: ["search", "research", "coding"], webSearch: ["search", "research", "coding"], + docx: ["search", "research", "coding", "documentation", "debug"], + pptx: ["search", "research", "coding", "documentation", "debug"], + xlsx: ["search", "research", "coding", "documentation", "debug"], + pdf: ["search", "research", "coding", "documentation", "debug"], }; /** @@ -149,6 +161,10 @@ export const TOOLS = { visionAnalyze, webExtract, webSearch, + docx: docxTool, + pptx: pptxTool, + xlsx: xlsxTool, + pdf: pdfTool, }; /** diff --git a/tests/unit/fileExtract/docx.test.js b/tests/unit/fileExtract/docx.test.js new file mode 100644 index 00000000..28fb7bdf --- /dev/null +++ b/tests/unit/fileExtract/docx.test.js @@ -0,0 +1,43 @@ +/** + * Unit tests for the DOCX extraction tool. + * @module tests/unit/fileExtract/docx.test + */ + +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { docxExtract } from "../../../src/tools/fileExtract/docx.js"; + +describe("fileExtract/docx", () => { + describe("docxExtract", () => { + it("should return an error for non-existent files", async () => { + const result = await docxExtract({ filePath: "/nonexistent/file.docx" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("Failed to read file")); + }); + + it("should return an error for unsupported formats", async () => { + const result = await docxExtract({ filePath: "document.txt" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("Unsupported format")); + }); + + it("should return an error for files without extensions", async () => { + const result = await docxExtract({ filePath: "document" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("No file extension")); + }); + + it("should return an error for non-DOCX files with .docx extension", async () => { + // Create a text file and try to extract it as DOCX + const { writeFileSync } = await import("node:fs"); + writeFileSync("/tmp/test-fake-docx.docx", "This is not a real DOCX file"); + const result = await docxExtract({ filePath: "/tmp/test-fake-docx.docx" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("ZIP extraction failed")); + }); + }); +}); diff --git a/tests/unit/fileExtract/formatValidator.test.js b/tests/unit/fileExtract/formatValidator.test.js new file mode 100644 index 00000000..a0ab5040 --- /dev/null +++ b/tests/unit/fileExtract/formatValidator.test.js @@ -0,0 +1,72 @@ +/** + * Unit tests for the format validation utility. + * @module tests/unit/fileExtract/formatValidator.test + */ + +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { validateFormat, SUPPORTED_FORMATS } from "../../../src/tools/fileExtract/formatValidator.js"; + +describe("fileExtract/formatValidator", () => { + describe("SUPPORTED_FORMATS", () => { + it("should include all expected formats", () => { + assert.ok(SUPPORTED_FORMATS.has("docx")); + assert.ok(SUPPORTED_FORMATS.has("pptx")); + assert.ok(SUPPORTED_FORMATS.has("xlsx")); + assert.ok(SUPPORTED_FORMATS.has("pdf")); + }); + + it("should have at least 4 supported formats", () => { + assert.ok(SUPPORTED_FORMATS.size >= 4); + }); + }); + + describe("validateFormat", () => { + it("should validate docx files", () => { + const result = validateFormat("document.docx"); + assert.strictEqual(result.valid, true); + assert.strictEqual(result.format, "docx"); + }); + + it("should validate pptx files", () => { + const result = validateFormat("presentation.pptx"); + assert.strictEqual(result.valid, true); + assert.strictEqual(result.format, "pptx"); + }); + + it("should validate xlsx files", () => { + const result = validateFormat("spreadsheet.xlsx"); + assert.strictEqual(result.valid, true); + assert.strictEqual(result.format, "xlsx"); + }); + + it("should validate pdf files", () => { + const result = validateFormat("document.pdf"); + assert.strictEqual(result.valid, true); + assert.strictEqual(result.format, "pdf"); + }); + + it("should reject unsupported formats", () => { + const result = validateFormat("document.txt"); + assert.strictEqual(result.valid, false); + assert.ok(result.error.includes("Unsupported format")); + }); + + it("should reject files without extensions", () => { + const result = validateFormat("document"); + assert.strictEqual(result.valid, false); + assert.ok(result.error.includes("No file extension")); + }); + + it("should handle case-insensitive extensions", () => { + const result = validateFormat("document.DOCX"); + assert.strictEqual(result.valid, true); + }); + + it("should handle paths with directories", () => { + const result = validateFormat("/path/to/document.docx"); + assert.strictEqual(result.valid, true); + assert.strictEqual(result.format, "docx"); + }); + }); +}); diff --git a/tests/unit/fileExtract/pdf.test.js b/tests/unit/fileExtract/pdf.test.js new file mode 100644 index 00000000..1335b8b9 --- /dev/null +++ b/tests/unit/fileExtract/pdf.test.js @@ -0,0 +1,42 @@ +/** + * Unit tests for the PDF extraction tool. + * @module tests/unit/fileExtract/pdf.test + */ + +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { pdfExtract } from "../../../src/tools/fileExtract/pdf.js"; + +describe("fileExtract/pdf", () => { + describe("pdfExtract", () => { + it("should return an error for non-existent files", async () => { + const result = await pdfExtract({ filePath: "/nonexistent/file.pdf" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("Failed to read file")); + }); + + it("should return an error for unsupported formats", async () => { + const result = await pdfExtract({ filePath: "document.txt" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("Unsupported format")); + }); + + it("should return an error for files without extensions", async () => { + const result = await pdfExtract({ filePath: "document" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("No file extension")); + }); + + it("should return an error for non-PDF files with .pdf extension", async () => { + const { writeFileSync } = await import("node:fs"); + writeFileSync("/tmp/test-fake-pdf.pdf", "This is not a real PDF file"); + const result = await pdfExtract({ filePath: "/tmp/test-fake-pdf.pdf" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("PDF extraction failed")); + }); + }); +}); diff --git a/tests/unit/fileExtract/pptx.test.js b/tests/unit/fileExtract/pptx.test.js new file mode 100644 index 00000000..fab44a54 --- /dev/null +++ b/tests/unit/fileExtract/pptx.test.js @@ -0,0 +1,42 @@ +/** + * Unit tests for the PPTX extraction tool. + * @module tests/unit/fileExtract/pptx.test + */ + +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { pptxExtract } from "../../../src/tools/fileExtract/pptx.js"; + +describe("fileExtract/pptx", () => { + describe("pptxExtract", () => { + it("should return an error for non-existent files", async () => { + const result = await pptxExtract({ filePath: "/nonexistent/file.pptx" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("Failed to read file")); + }); + + it("should return an error for unsupported formats", async () => { + const result = await pptxExtract({ filePath: "presentation.txt" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("Unsupported format")); + }); + + it("should return an error for files without extensions", async () => { + const result = await pptxExtract({ filePath: "presentation" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("No file extension")); + }); + + it("should return an error for non-PPTX files with .pptx extension", async () => { + const { writeFileSync } = await import("node:fs"); + writeFileSync("/tmp/test-fake-pptx.pptx", "This is not a real PPTX file"); + const result = await pptxExtract({ filePath: "/tmp/test-fake-pptx.pptx" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("ZIP extraction failed")); + }); + }); +}); diff --git a/tests/unit/fileExtract/xlsx.test.js b/tests/unit/fileExtract/xlsx.test.js new file mode 100644 index 00000000..0cb25e62 --- /dev/null +++ b/tests/unit/fileExtract/xlsx.test.js @@ -0,0 +1,42 @@ +/** + * Unit tests for the XLSX extraction tool. + * @module tests/unit/fileExtract/xlsx.test + */ + +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { xlsxExtract } from "../../../src/tools/fileExtract/xlsx.js"; + +describe("fileExtract/xlsx", () => { + describe("xlsxExtract", () => { + it("should return an error for non-existent files", async () => { + const result = await xlsxExtract({ filePath: "/nonexistent/file.xlsx" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("Failed to read file")); + }); + + it("should return an error for unsupported formats", async () => { + const result = await xlsxExtract({ filePath: "spreadsheet.txt" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("Unsupported format")); + }); + + it("should return an error for files without extensions", async () => { + const result = await xlsxExtract({ filePath: "spreadsheet" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("No file extension")); + }); + + it("should return an error for non-XLSX files with .xlsx extension", async () => { + const { writeFileSync } = await import("node:fs"); + writeFileSync("/tmp/test-fake-xlsx.xlsx", "This is not a real XLSX file"); + const result = await xlsxExtract({ filePath: "/tmp/test-fake-xlsx.xlsx" }); + const parsed = JSON.parse(result); + assert.strictEqual(parsed.ok, false); + assert.ok(parsed.error.includes("ZIP extraction failed")); + }); + }); +}); diff --git a/tests/unit/fileExtract/zipExtractor.test.js b/tests/unit/fileExtract/zipExtractor.test.js new file mode 100644 index 00000000..a570547b --- /dev/null +++ b/tests/unit/fileExtract/zipExtractor.test.js @@ -0,0 +1,45 @@ +/** + * Unit tests for the file extraction utility module. + * @module tests/unit/fileExtract/zipExtractor.test + */ + +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { extractZipXml, getZipFileNames, validateZip } from "../../../src/tools/fileExtract/zipExtractor.js"; + +describe("fileExtract/zipExtractor", () => { + describe("validateZip", () => { + it("should return valid for a supported ZIP-based format", async () => { + const result = validateZip("test.docx"); + assert.strictEqual(result.valid, true); + }); + + it("should return invalid for unsupported formats", async () => { + const result = validateZip("test.txt"); + assert.strictEqual(result.valid, false); + }); + + it("should return invalid for files without extension", async () => { + const result = validateZip("test"); + assert.strictEqual(result.valid, false); + }); + }); + + describe("getZipFileNames", () => { + it("should return an empty array when no ZIP file is provided", async () => { + const result = await getZipFileNames(null); + assert.deepStrictEqual(result, []); + }); + + it("should return an empty array for non-ZIP files", async () => { + const result = await getZipFileNames("test.txt"); + assert.deepStrictEqual(result, []); + }); + }); + + describe("extractZipXml", () => { + it("should throw an error for non-ZIP files", async () => { + await assert.rejects(extractZipXml("test.txt"), { name: "ZipExtractionError" }); + }); + }); +}); From 1e9489896b37e1482b86deacfc025aa6d4006463 Mon Sep 17 00:00:00 2001 From: Jason Mulligan Date: Wed, 12 Aug 2026 12:55:35 -0400 Subject: [PATCH 3/6] chore: archive file-format-extraction-tools change - Move OpenSpec change to archive/2026-08-12-file-format-extraction-tools/ - Apply spec deltas to openspec/specs/ (5 new capabilities) - 9 spec lines added --- .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/docx-extraction/spec.md | 0 .../specs/file-extraction/spec.md | 0 .../specs/pdf-extraction/spec.md | 0 .../specs/pptx-extraction/spec.md | 0 .../specs/xlsx-extraction/spec.md | 0 .../tasks.md | 0 openspec/specs/docx-extraction/spec.md | 32 ++++++++++++++ openspec/specs/file-extraction/spec.md | 42 ++++++++++++++++++ openspec/specs/pdf-extraction/spec.md | 39 +++++++++++++++++ openspec/specs/pptx-extraction/spec.md | 32 ++++++++++++++ openspec/specs/xlsx-extraction/spec.md | 43 +++++++++++++++++++ 14 files changed, 188 insertions(+) rename openspec/changes/{file-format-extraction-tools => archive/2026-08-12-file-format-extraction-tools}/.openspec.yaml (100%) rename openspec/changes/{file-format-extraction-tools => archive/2026-08-12-file-format-extraction-tools}/design.md (100%) rename openspec/changes/{file-format-extraction-tools => archive/2026-08-12-file-format-extraction-tools}/proposal.md (100%) rename openspec/changes/{file-format-extraction-tools => archive/2026-08-12-file-format-extraction-tools}/specs/docx-extraction/spec.md (100%) rename openspec/changes/{file-format-extraction-tools => archive/2026-08-12-file-format-extraction-tools}/specs/file-extraction/spec.md (100%) rename openspec/changes/{file-format-extraction-tools => archive/2026-08-12-file-format-extraction-tools}/specs/pdf-extraction/spec.md (100%) rename openspec/changes/{file-format-extraction-tools => archive/2026-08-12-file-format-extraction-tools}/specs/pptx-extraction/spec.md (100%) rename openspec/changes/{file-format-extraction-tools => archive/2026-08-12-file-format-extraction-tools}/specs/xlsx-extraction/spec.md (100%) rename openspec/changes/{file-format-extraction-tools => archive/2026-08-12-file-format-extraction-tools}/tasks.md (100%) create mode 100644 openspec/specs/docx-extraction/spec.md create mode 100644 openspec/specs/file-extraction/spec.md create mode 100644 openspec/specs/pdf-extraction/spec.md create mode 100644 openspec/specs/pptx-extraction/spec.md create mode 100644 openspec/specs/xlsx-extraction/spec.md diff --git a/openspec/changes/file-format-extraction-tools/.openspec.yaml b/openspec/changes/archive/2026-08-12-file-format-extraction-tools/.openspec.yaml similarity index 100% rename from openspec/changes/file-format-extraction-tools/.openspec.yaml rename to openspec/changes/archive/2026-08-12-file-format-extraction-tools/.openspec.yaml diff --git a/openspec/changes/file-format-extraction-tools/design.md b/openspec/changes/archive/2026-08-12-file-format-extraction-tools/design.md similarity index 100% rename from openspec/changes/file-format-extraction-tools/design.md rename to openspec/changes/archive/2026-08-12-file-format-extraction-tools/design.md diff --git a/openspec/changes/file-format-extraction-tools/proposal.md b/openspec/changes/archive/2026-08-12-file-format-extraction-tools/proposal.md similarity index 100% rename from openspec/changes/file-format-extraction-tools/proposal.md rename to openspec/changes/archive/2026-08-12-file-format-extraction-tools/proposal.md diff --git a/openspec/changes/file-format-extraction-tools/specs/docx-extraction/spec.md b/openspec/changes/archive/2026-08-12-file-format-extraction-tools/specs/docx-extraction/spec.md similarity index 100% rename from openspec/changes/file-format-extraction-tools/specs/docx-extraction/spec.md rename to openspec/changes/archive/2026-08-12-file-format-extraction-tools/specs/docx-extraction/spec.md diff --git a/openspec/changes/file-format-extraction-tools/specs/file-extraction/spec.md b/openspec/changes/archive/2026-08-12-file-format-extraction-tools/specs/file-extraction/spec.md similarity index 100% rename from openspec/changes/file-format-extraction-tools/specs/file-extraction/spec.md rename to openspec/changes/archive/2026-08-12-file-format-extraction-tools/specs/file-extraction/spec.md diff --git a/openspec/changes/file-format-extraction-tools/specs/pdf-extraction/spec.md b/openspec/changes/archive/2026-08-12-file-format-extraction-tools/specs/pdf-extraction/spec.md similarity index 100% rename from openspec/changes/file-format-extraction-tools/specs/pdf-extraction/spec.md rename to openspec/changes/archive/2026-08-12-file-format-extraction-tools/specs/pdf-extraction/spec.md diff --git a/openspec/changes/file-format-extraction-tools/specs/pptx-extraction/spec.md b/openspec/changes/archive/2026-08-12-file-format-extraction-tools/specs/pptx-extraction/spec.md similarity index 100% rename from openspec/changes/file-format-extraction-tools/specs/pptx-extraction/spec.md rename to openspec/changes/archive/2026-08-12-file-format-extraction-tools/specs/pptx-extraction/spec.md diff --git a/openspec/changes/file-format-extraction-tools/specs/xlsx-extraction/spec.md b/openspec/changes/archive/2026-08-12-file-format-extraction-tools/specs/xlsx-extraction/spec.md similarity index 100% rename from openspec/changes/file-format-extraction-tools/specs/xlsx-extraction/spec.md rename to openspec/changes/archive/2026-08-12-file-format-extraction-tools/specs/xlsx-extraction/spec.md diff --git a/openspec/changes/file-format-extraction-tools/tasks.md b/openspec/changes/archive/2026-08-12-file-format-extraction-tools/tasks.md similarity index 100% rename from openspec/changes/file-format-extraction-tools/tasks.md rename to openspec/changes/archive/2026-08-12-file-format-extraction-tools/tasks.md diff --git a/openspec/specs/docx-extraction/spec.md b/openspec/specs/docx-extraction/spec.md new file mode 100644 index 00000000..5fd83c0f --- /dev/null +++ b/openspec/specs/docx-extraction/spec.md @@ -0,0 +1,32 @@ +# docx-extraction Specification + +## Purpose +TBD - created by archiving change file-format-extraction-tools. Update Purpose after archive. +## Requirements +### Requirement: DOCX to markdown conversion +The system SHALL convert `.docx` files to structured markdown, preserving headings, paragraphs, lists, tables, and inline formatting (bold, italic, code). + +#### Scenario: Extract headings from docx +- **WHEN** a docx file with heading styles is provided +- **THEN** the system outputs markdown headings (`#`, `##`, `###`) matching the document hierarchy + +#### Scenario: Extract paragraphs from docx +- **WHEN** a docx file with body text is provided +- **THEN** the system outputs markdown paragraphs with proper line breaks + +#### Scenario: Extract lists from docx +- **WHEN** a docx file with ordered or unordered lists is provided +- **THEN** the system outputs markdown list items (`-` for unordered, `1.` for ordered) + +#### Scenario: Extract tables from docx +- **WHEN** a docx file with tables is provided +- **THEN** the system outputs markdown tables with proper column alignment + +#### Scenario: Handle empty docx file +- **WHEN** an empty docx file is provided +- **THEN** the system returns an empty string + +#### Scenario: Handle missing document.xml +- **WHEN** a docx file without word/document.xml is provided +- **THEN** the system throws a descriptive error + diff --git a/openspec/specs/file-extraction/spec.md b/openspec/specs/file-extraction/spec.md new file mode 100644 index 00000000..568ca94b --- /dev/null +++ b/openspec/specs/file-extraction/spec.md @@ -0,0 +1,42 @@ +# file-extraction Specification + +## Purpose +TBD - created by archiving change file-format-extraction-tools. Update Purpose after archive. +## Requirements +### Requirement: ZIP archive extraction +The system SHALL decompress ZIP-based document archives and extract their XML content files into a structured map of filename to content. + +#### Scenario: Successful ZIP extraction +- **WHEN** a valid ZIP archive file path is provided +- **THEN** the system returns a map of filename strings to their XML content strings + +#### Scenario: Corrupted ZIP archive +- **WHEN** a corrupted or invalid ZIP file is provided +- **THEN** the system throws a descriptive error without crashing + +#### Scenario: Password-protected ZIP archive +- **WHEN** a password-protected ZIP file is provided +- **THEN** the system throws an error indicating the archive is password-protected + +### Requirement: XML content retrieval by filename +The system SHALL locate specific XML files within a ZIP archive by their known filenames regardless of internal directory structure. + +#### Scenario: Locate document.xml in docx +- **WHEN** the utility is asked for "word/document.xml" in a docx file +- **THEN** the system returns the XML content of that file + +#### Scenario: Locate slide files in pptx +- **WHEN** the utility is asked for "ppt/slides/slide1.xml" in a pptx file +- **THEN** the system returns the XML content of that slide file + +### Requirement: File format validation +The system SHALL validate that a file is a supported ZIP-based format before attempting extraction. + +#### Scenario: Validate docx extension +- **WHEN** a file with `.docx` extension is provided +- **THEN** the system confirms it is a supported format + +#### Scenario: Reject unsupported format +- **WHEN** a file with an unsupported extension is provided +- **THEN** the system throws an error listing supported formats + diff --git a/openspec/specs/pdf-extraction/spec.md b/openspec/specs/pdf-extraction/spec.md new file mode 100644 index 00000000..13bda64c --- /dev/null +++ b/openspec/specs/pdf-extraction/spec.md @@ -0,0 +1,39 @@ +# pdf-extraction Specification + +## Purpose +TBD - created by archiving change file-format-extraction-tools. Update Purpose after archive. +## Requirements +### Requirement: PDF text extraction +The system SHALL extract text content from PDF files and output it as structured markdown. + +#### Scenario: Extract text from simple PDF +- **WHEN** a PDF file with plain text content is provided +- **THEN** the system outputs the extracted text as markdown paragraphs + +#### Scenario: Extract text from multi-page PDF +- **WHEN** a multi-page PDF file is provided +- **THEN** the system outputs text from all pages in sequential order + +#### Scenario: Handle PDF with no extractable text +- **WHEN** a PDF file contains only images (no text layer) +- **THEN** the system returns an error indicating no text could be extracted + +#### Scenario: Handle empty PDF file +- **WHEN** an empty or minimal PDF file is provided +- **THEN** the system returns an empty string + +#### Scenario: Handle PDF with special characters +- **WHEN** a PDF file contains Unicode characters +- **THEN** the system preserves Unicode characters in the output + +### Requirement: PDF extraction error handling +The system SHALL provide descriptive error messages for PDF extraction failures. + +#### Scenario: Handle corrupted PDF +- **WHEN** a corrupted PDF file is provided +- **THEN** the system throws a descriptive error without crashing + +#### Scenario: Handle password-protected PDF +- **WHEN** a password-protected PDF file is provided +- **THEN** the system throws an error indicating the file is password-protected + diff --git a/openspec/specs/pptx-extraction/spec.md b/openspec/specs/pptx-extraction/spec.md new file mode 100644 index 00000000..51653ffa --- /dev/null +++ b/openspec/specs/pptx-extraction/spec.md @@ -0,0 +1,32 @@ +# pptx-extraction Specification + +## Purpose +TBD - created by archiving change file-format-extraction-tools. Update Purpose after archive. +## Requirements +### Requirement: PPTX to markdown conversion +The system SHALL convert `.pptx` files to structured markdown, preserving slide titles, bullet points, speaker notes, and basic text content. + +#### Scenario: Extract slide titles from pptx +- **WHEN** a pptx file with slide titles is provided +- **THEN** the system outputs markdown headings (`#`) for each slide title + +#### Scenario: Extract bullet points from pptx +- **WHEN** a pptx file with bullet points is provided +- **THEN** the system outputs markdown unordered list items (`-`) for each bullet + +#### Scenario: Extract speaker notes from pptx +- **WHEN** a pptx file with speaker notes is provided +- **THEN** the system outputs markdown text prefixed with "Speaker Notes:" + +#### Scenario: Handle slides without titles +- **WHEN** a pptx slide has no title text +- **THEN** the system outputs a numbered slide separator (e.g., `---`) + +#### Scenario: Handle empty pptx file +- **WHEN** an empty pptx file is provided +- **THEN** the system returns an empty string + +#### Scenario: Handle missing slide files +- **WHEN** a pptx file with missing slide XML is provided +- **THEN** the system skips the missing slide and continues processing remaining slides + diff --git a/openspec/specs/xlsx-extraction/spec.md b/openspec/specs/xlsx-extraction/spec.md new file mode 100644 index 00000000..e94bf9fb --- /dev/null +++ b/openspec/specs/xlsx-extraction/spec.md @@ -0,0 +1,43 @@ +# xlsx-extraction Specification + +## Purpose +TBD - created by archiving change file-format-extraction-tools. Update Purpose after archive. +## Requirements +### Requirement: XLSX to markdown table conversion +The system SHALL convert `.xlsx` spreadsheets to markdown tables, preserving cell values, column headers, and row structure. + +#### Scenario: Extract single-sheet xlsx to markdown +- **WHEN** a single-sheet xlsx file is provided +- **THEN** the system outputs a markdown table with headers from the first row and data from subsequent rows + +#### Scenario: Extract multi-sheet xlsx to markdown +- **WHEN** a multi-sheet xlsx file is provided +- **THEN** the system outputs separate markdown tables for each sheet, separated by sheet name headers + +#### Scenario: Handle empty xlsx file +- **WHEN** an empty xlsx file is provided +- **THEN** the system returns an empty string + +#### Scenario: Handle numeric cell values +- **WHEN** a cell contains a numeric value +- **THEN** the system outputs the numeric value as a string in the markdown table + +#### Scenario: Handle merged cells +- **WHEN** a cell is merged with another cell +- **THEN** the system outputs the value in the top-left cell position and empty strings for merged positions + +### Requirement: XLSX to JSON conversion +The system SHALL convert `.xlsx` spreadsheets to JSON, preserving cell values, types, and sheet structure. + +#### Scenario: Extract xlsx to JSON with sheet names +- **WHEN** a multi-sheet xlsx file is provided +- **THEN** the system returns a JSON object with sheet names as keys and arrays of row objects as values + +#### Scenario: Preserve cell data types in JSON +- **WHEN** a cell contains a number, string, or boolean value +- **THEN** the system preserves the original data type in the JSON output + +#### Scenario: Handle empty rows in JSON output +- **WHEN** an xlsx file has empty rows +- **THEN** the system includes empty row objects in the JSON output + From cbf19d7a068bf684ab486ffde3d98248186d406e Mon Sep 17 00:00:00 2001 From: Jason Mulligan Date: Wed, 12 Aug 2026 15:34:04 -0400 Subject: [PATCH 4/6] fix: clean up lint errors and formatting in fileExtract module - Remove unused buffer variables (dead code) - Remove unused imports (extractZipXml, ZipExtractionError, getExtension) - Merge duplicate imports in docxParser.js - Fix unused catch parameter in zipExtractor.js - Fix redundant const comparison in xlsxParser.js - Prefix unused variables with _ (listType, isListItem) --- src/tools/fileExtract/docx.js | 8 ++++---- src/tools/fileExtract/docxParser.js | 15 ++++++++------- src/tools/fileExtract/formatValidator.js | 8 +++++++- src/tools/fileExtract/pdf.js | 3 ++- src/tools/fileExtract/pptx.js | 6 +++--- src/tools/fileExtract/pptxParser.js | 5 +++-- src/tools/fileExtract/xlsx.js | 12 ++++++++---- src/tools/fileExtract/xlsxJson.js | 5 +---- src/tools/fileExtract/xlsxParser.js | 8 ++------ src/tools/fileExtract/zipExtractor.js | 12 ++++++------ tests/unit/fileExtract/formatValidator.test.js | 5 ++++- tests/unit/fileExtract/zipExtractor.test.js | 6 +++++- 12 files changed, 53 insertions(+), 40 deletions(-) diff --git a/src/tools/fileExtract/docx.js b/src/tools/fileExtract/docx.js index 41882d2c..a01c8d4c 100644 --- a/src/tools/fileExtract/docx.js +++ b/src/tools/fileExtract/docx.js @@ -9,7 +9,7 @@ import { tool } from "@langchain/core/tools"; import { readFile } from "node:fs/promises"; import { extractZipXml } from "./zipExtractor.js"; import { docxToMarkdown, extractDocxTables } from "./docxParser.js"; -import { validateFormat, getExtension } from "./formatValidator.js"; +import { validateFormat } from "./formatValidator.js"; /** * Input schema for the docx tool. @@ -34,9 +34,8 @@ export async function docxExtract(input) { } // Read file - let buffer; try { - buffer = await readFile(filePath); + await readFile(filePath); } catch (err) { return JSON.stringify({ ok: false, error: `Failed to read file: ${err.message}` }); } @@ -71,6 +70,7 @@ export async function docxExtract(input) { */ export const docxTool = tool(docxExtract, { name: "docx", - description: "Extract content from a Microsoft Word (.docx) file to markdown. Accepts a file path and returns structured markdown with headings, paragraphs, lists, and tables.", + description: + "Extract content from a Microsoft Word (.docx) file to markdown. Accepts a file path and returns structured markdown with headings, paragraphs, lists, and tables.", schema: docxSchema, }); diff --git a/src/tools/fileExtract/docxParser.js b/src/tools/fileExtract/docxParser.js index 43127182..41277052 100644 --- a/src/tools/fileExtract/docxParser.js +++ b/src/tools/fileExtract/docxParser.js @@ -4,8 +4,6 @@ * @module fileExtract/docxParser */ -import { extractZipXml } from "./zipExtractor.js"; -import { ZipExtractionError } from "./zipExtractor.js"; import { parseStringPromise } from "xml2js"; /** @@ -20,7 +18,7 @@ export function docxToMarkdown(documentXml) { let markdown = ""; let inList = false; - let listType = null; + let _listType = null; try { const parsed = parseStringPromise(documentXml, { @@ -43,7 +41,7 @@ export function docxToMarkdown(documentXml) { if (inList) { markdown += "\n"; inList = false; - listType = null; + _listType = null; } markdown += `${"#".repeat(headingLevel)} ${textContent}\n\n`; } else if (isListItem) { @@ -56,7 +54,7 @@ export function docxToMarkdown(documentXml) { if (inList) { markdown += "\n"; inList = false; - listType = null; + _listType = null; } markdown += `${textContent}\n\n`; } @@ -65,7 +63,10 @@ export function docxToMarkdown(documentXml) { // If XML parsing fails, return raw text const textMatch = documentXml.match(/>([^<]+) m.replace(/^>| m.replace(/^>| entry.entryName); - } catch (err) { + } catch (_err) { return []; } } @@ -104,7 +104,10 @@ export async function extractZipFile(filePath, internalPath) { return entry.getData().toString("utf-8"); } catch (err) { if (err instanceof ZipExtractionError) throw err; - throw new ZipExtractionError(`Failed to extract file from ZIP: ${err.message}`, "extraction-failed"); + throw new ZipExtractionError( + `Failed to extract file from ZIP: ${err.message}`, + "extraction-failed", + ); } } @@ -141,10 +144,7 @@ export async function extractZipGlob(filePath, pattern) { * @returns {RegExp} */ function patternToRegex(pattern) { - const escaped = pattern - .replace(/\./g, "\\.") - .replace(/\*/g, ".*") - .replace(/\?/g, "."); + const escaped = pattern.replace(/\./g, "\\.").replace(/\*/g, ".*").replace(/\?/g, "."); return new RegExp(`^${escaped}$`); } diff --git a/tests/unit/fileExtract/formatValidator.test.js b/tests/unit/fileExtract/formatValidator.test.js index a0ab5040..72b276af 100644 --- a/tests/unit/fileExtract/formatValidator.test.js +++ b/tests/unit/fileExtract/formatValidator.test.js @@ -5,7 +5,10 @@ import { describe, it } from "node:test"; import assert from "node:assert"; -import { validateFormat, SUPPORTED_FORMATS } from "../../../src/tools/fileExtract/formatValidator.js"; +import { + validateFormat, + SUPPORTED_FORMATS, +} from "../../../src/tools/fileExtract/formatValidator.js"; describe("fileExtract/formatValidator", () => { describe("SUPPORTED_FORMATS", () => { diff --git a/tests/unit/fileExtract/zipExtractor.test.js b/tests/unit/fileExtract/zipExtractor.test.js index a570547b..d534aca1 100644 --- a/tests/unit/fileExtract/zipExtractor.test.js +++ b/tests/unit/fileExtract/zipExtractor.test.js @@ -5,7 +5,11 @@ import { describe, it } from "node:test"; import assert from "node:assert"; -import { extractZipXml, getZipFileNames, validateZip } from "../../../src/tools/fileExtract/zipExtractor.js"; +import { + extractZipXml, + getZipFileNames, + validateZip, +} from "../../../src/tools/fileExtract/zipExtractor.js"; describe("fileExtract/zipExtractor", () => { describe("validateZip", () => { From f0ae26fd1b23a028b6c46b71fb69bbb828868f8c Mon Sep 17 00:00:00 2001 From: Jason Mulligan Date: Wed, 12 Aug 2026 15:41:34 -0400 Subject: [PATCH 5/6] fix: restore docx, pptx, xlsx tools dropped during merge conflict resolution --- src/tools/index.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/tools/index.js b/src/tools/index.js index fdd4d056..9ee98206 100644 --- a/src/tools/index.js +++ b/src/tools/index.js @@ -43,6 +43,9 @@ export const TOOL_PERMISSIONS = { visionAnalyze: [], webExtract: ["network:outbound"], webSearch: ["network:outbound"], + docx: ["filesystem:read"], + pptx: ["filesystem:read"], + xlsx: ["filesystem:read"], pdf: ["filesystem:read"], reflectionSessions: ["filesystem:read"], }; @@ -100,6 +103,9 @@ export const TOOL_CLASSIFICATIONS = { visionAnalyze: ["code-review", "testing", "coding"], webExtract: ["search", "research", "coding"], webSearch: ["search", "research", "coding"], + docx: ["search", "research", "coding", "documentation", "debug"], + pptx: ["search", "research", "coding", "documentation", "debug"], + xlsx: ["search", "research", "coding", "documentation", "debug"], pdf: ["search", "research", "coding", "documentation", "debug"], reflectionSessions: ["orchestrator"], }; @@ -159,6 +165,9 @@ export const TOOLS = { visionAnalyze, webExtract, webSearch, + docx: docxTool, + pptx: pptxTool, + xlsx: xlsxTool, pdf: pdfTool, reflectionSessions, }; From 43f1e94981a1c739300ad1b5a6bdf23559eac0a7 Mon Sep 17 00:00:00 2001 From: Jason Mulligan Date: Wed, 12 Aug 2026 16:11:04 -0400 Subject: [PATCH 6/6] fix: resolve all PR audit findings for file-format-extraction-tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical bugs: - Fix isListItem() reference error in docxParser.js (was _isListItem) - Fix listType/_listType variable mismatch in docxParser.js - Fix pdf-parse API usage (was class, is a function) Spec gaps: - Add inline formatting (bold, italic, code) to docx parser - Handle merged cells in xlsx parser - Numbered slide separator for pptx slides without titles - Password-protected ZIP detection in getZipFileNames - Empty PDF returns empty string, not error Minor: - Replace silent catch blocks with named parameters (_err) - Update tool_index test for new file extraction tools (8→12 tools) - Remove unused PDFParse import from pdfParser.js --- src/tools/fileExtract/docxParser.js | 40 ++++++++++++++++----- src/tools/fileExtract/pdf.js | 7 ++++ src/tools/fileExtract/pdfParser.js | 8 ++--- src/tools/fileExtract/pptxParser.js | 6 ++-- src/tools/fileExtract/xlsxJson.js | 2 +- src/tools/fileExtract/xlsxParser.js | 52 ++++++++++++++++++++++++++- src/tools/fileExtract/zipExtractor.js | 7 ++-- tests/unit/tool_index.test.js | 8 +++-- 8 files changed, 106 insertions(+), 24 deletions(-) diff --git a/src/tools/fileExtract/docxParser.js b/src/tools/fileExtract/docxParser.js index 41277052..22faf84b 100644 --- a/src/tools/fileExtract/docxParser.js +++ b/src/tools/fileExtract/docxParser.js @@ -34,7 +34,7 @@ export function docxToMarkdown(documentXml) { for (const para of paragraphs) { const textContent = extractParagraphText(para); const headingLevel = getHeadingLevel(para); - const isListItem = isListItem(para); + const isListItem = _isListItem(para); if (headingLevel > 0) { // Close any open list @@ -47,7 +47,7 @@ export function docxToMarkdown(documentXml) { } else if (isListItem) { if (!inList) { inList = true; - listType = "ul"; + _listType = "ul"; } markdown += `- ${textContent}\n`; } else if (textContent.trim()) { @@ -59,8 +59,8 @@ export function docxToMarkdown(documentXml) { markdown += `${textContent}\n\n`; } } - } catch { - // If XML parsing fails, return raw text + } catch (_err) { + // If XML parsing fails, return raw text as fallback const textMatch = documentXml.match(/>([^<]+) p.length > 0); const markdown = paragraphs.join("\n\n"); - await parser.destroy(); return markdown; } catch (err) { if (err instanceof PdfExtractionError) throw err; diff --git a/src/tools/fileExtract/pptxParser.js b/src/tools/fileExtract/pptxParser.js index c3d02274..87222aea 100644 --- a/src/tools/fileExtract/pptxParser.js +++ b/src/tools/fileExtract/pptxParser.js @@ -83,15 +83,15 @@ export function pptxToMarkdown(zipContent) { } } } - } catch { + } catch (_err) { // Skip notes on parse error } } if (!titleFound) { - markdown = markdown.replace(`---\n\n`, `---\n\n`); + markdown = markdown.replace(`---\n\n`, `---\n\n## Slide ${slideIndex}\n\n`); } - } catch { + } catch (_err) { markdown += `## Slide ${slideIndex} (parse error)\n\n`; } } diff --git a/src/tools/fileExtract/xlsxJson.js b/src/tools/fileExtract/xlsxJson.js index 8ed2dead..30fa97c2 100644 --- a/src/tools/fileExtract/xlsxJson.js +++ b/src/tools/fileExtract/xlsxJson.js @@ -38,7 +38,7 @@ export function xlsxToJson(zipContent) { const rows = parseSheetRows(sheetXml); result[sheetName] = rows; } - } catch { + } catch (_err) { // Silently skip on parse error } diff --git a/src/tools/fileExtract/xlsxParser.js b/src/tools/fileExtract/xlsxParser.js index d9346b8e..3e3163d5 100644 --- a/src/tools/fileExtract/xlsxParser.js +++ b/src/tools/fileExtract/xlsxParser.js @@ -43,7 +43,7 @@ export function xlsxToMarkdown(zipContent) { markdown += "\n\n"; } } - } catch { + } catch (_err) { // Silently skip on parse error } @@ -92,6 +92,32 @@ function parseSheetContent(sheetXml) { if (!sheetData) return null; const rowArray = Array.isArray(sheetData) ? sheetData : [sheetData]; + + // First pass: collect merged cell ranges + const mergedCells = parsed?.sheet?.[0]?.mergeCells?.mergeCell; + const mergedMap = new Map(); + if (mergedCells) { + const mergedArray = Array.isArray(mergedCells) ? mergedCells : [mergedCells]; + for (const mc of mergedArray) { + const ref = mc?.$?.ref || mc?.ref; + if (!ref) continue; + const match = ref.match(/^([A-Z]+)(\d+):([A-Z]+)(\d+)$/); + if (!match) continue; + const [, col1, row1, col2, row2] = match; + const startCol = colToNum(col1); + const startRow = parseInt(row1, 10); + const endCol = colToNum(col2); + const endRow = parseInt(row2, 10); + for (let r = startRow; r <= endRow; r++) { + for (let c = startCol; c <= endCol; c++) { + if (r !== startRow || c !== startCol) { + mergedMap.set(`${r},${c}`, true); + } + } + } + } + } + const rows = []; for (const row of rowArray) { @@ -106,12 +132,36 @@ function parseSheetContent(sheetXml) { rows.push(rowData); } + // Second pass: fill merged cell positions with empty strings + for (const [key] of mergedMap) { + const [rowIdx, colIdx] = key.split(",").map(Number); + if (rowIdx <= rows.length) { + while (rows[rowIdx - 1].length <= colIdx - 1) { + rows[rowIdx - 1].push(""); + } + rows[rowIdx - 1][colIdx - 1] = ""; + } + } + return { rows }; } catch { return null; } } +/** + * Convert Excel column letters to number (A=1, B=2, ..., Z=26, AA=27). + * @param {string} col - Column letters + * @returns {number} + */ +function colToNum(col) { + let num = 0; + for (let i = 0; i < col.length; i++) { + num = num * 26 + (col.charCodeAt(i) - 64); + } + return num; +} + /** * Get the value of a cell. * @param {object} cell - Parsed cell XML object diff --git a/src/tools/fileExtract/zipExtractor.js b/src/tools/fileExtract/zipExtractor.js index c49344d8..221bb89d 100644 --- a/src/tools/fileExtract/zipExtractor.js +++ b/src/tools/fileExtract/zipExtractor.js @@ -57,8 +57,11 @@ export async function getZipFileNames(filePath) { try { const zip = new AdmZip(filePath); return zip.getEntries().map((entry) => entry.entryName); - } catch (_err) { - return []; + } catch (err) { + if (err.message && err.message.toLowerCase().includes("password")) { + throw new ZipExtractionError("Archive is password-protected", "password-protected"); + } + throw new ZipExtractionError(`Failed to read ZIP: ${err.message}`, "extraction-failed"); } } diff --git a/tests/unit/tool_index.test.js b/tests/unit/tool_index.test.js index c02cbd59..574c2113 100644 --- a/tests/unit/tool_index.test.js +++ b/tests/unit/tool_index.test.js @@ -158,13 +158,17 @@ describe("tools - buildToolConfig", () => { }); const toolNames = tools.map((t) => t.name); // filesystem:read enables: clarify, sampling, shell (always), compactContext, scanAgents, - // sessionSearch, date, reflectionSessions - assert.strictEqual(toolNames.length, 8); + // sessionSearch, date, reflectionSessions, docx, pptx, xlsx, pdf + assert.strictEqual(toolNames.length, 12); assert.ok(toolNames.includes("clarify")); assert.ok(toolNames.includes("sampling")); assert.ok(toolNames.includes("date")); assert.ok(toolNames.includes("scanAgents")); assert.ok(toolNames.includes("sessionSearch")); assert.ok(toolNames.includes("reflectionSessions")); + assert.ok(toolNames.includes("docx")); + assert.ok(toolNames.includes("pptx")); + assert.ok(toolNames.includes("xlsx")); + assert.ok(toolNames.includes("pdf")); }); });