diff --git a/AGENTS.md b/AGENTS.md index 862e624f..7b60fee4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,21 @@ scripts/Run-Benchmark_issues.ps1 -Filter "sa8000" # focused issue run scripts/Run-Benchmark_issues.ps1 -All # full issue benchmark ``` +### Cross-Language Visual Benchmarks + +Use the following form for .NET, Rust, Java, Go, Python, and Node: + +```powershell +scripts/Run--VisualBenchmark.ps1 -Suite -Format -MaxCases +``` + +Rust may also use `scripts/Run-Rust-Benchmark.ps1`; it forwards to the same +shared runner. Microsoft 365 is always the primary scored reference and +LibreOffice is the required auxiliary reference. `-Engine` does not switch the +primary reference. The default minimum score is `0.95`; use `-SkipCandidate` +only when the corresponding candidate PDFs already exist under +`artifacts/-benchmark///candidates`. + ## Classic Benchmark Refresh Workflow When updating all XLSX classic examples, canonical benchmark reports, or stale GitHub README benchmark images, use the `refresh-classic-benchmarks` skill (`/refresh-classic-benchmarks`). diff --git a/minipdf-rs/README.md b/minipdf-rs/README.md index 1ea87382..d352025d 100644 --- a/minipdf-rs/README.md +++ b/minipdf-rs/README.md @@ -174,7 +174,9 @@ Markdown and JSON reports, side-by-side images, and heatmaps under Microsoft 365 is the primary reference used for text, visual, page-count, and overall scores. LibreOffice is generated on every run as an auxiliary reference -and is included in the visual report without affecting those scores. +and is included in the visual report without affecting those scores. Both +references are required, the default minimum score is `0.95`, and +`-SkipCandidate` reuses an existing Rust candidate PDF. The matrix is generated at `artifacts/rust-benchmark/benchmark_matrix.md` and links to the fixture coverage and comparison reports from each run. diff --git a/minipdf-rs/crates/minipdf-cli/src/main.rs b/minipdf-rs/crates/minipdf-cli/src/main.rs index 43b502c0..79441d5f 100644 --- a/minipdf-rs/crates/minipdf-cli/src/main.rs +++ b/minipdf-rs/crates/minipdf-cli/src/main.rs @@ -200,7 +200,7 @@ fn register_office_cloud_fonts() -> minipdf::Result<()> { .map(|entry| entry.path().join("CloudFonts")) .filter(|path| path.is_dir()) { - for family in ["Grandview", "Grandview Display"] { + for family in ["Grandview", "Grandview Display", "STKaiti"] { let directory = cloud_root.join(family); let Ok(entries) = fs::read_dir(directory) else { continue; diff --git a/minipdf-rs/crates/minipdf/src/pdf.rs b/minipdf-rs/crates/minipdf/src/pdf.rs index fe0ee1e1..7c004c8e 100644 --- a/minipdf-rs/crates/minipdf/src/pdf.rs +++ b/minipdf-rs/crates/minipdf/src/pdf.rs @@ -688,6 +688,7 @@ fn split_font_runs( let font_index = if ch.is_whitespace() || ch.is_ascii_punctuation() || ch == '\u{fe0f}' { runs.last() .and_then(|run| run.font_index) + .filter(|index| font_supports(&fonts[*index], ch)) .or_else(|| select_font(fonts, ch, bold, italic, preferred_font)) } else { select_font(fonts, ch, bold, italic, preferred_font) @@ -763,6 +764,9 @@ fn font_preference( if name.starts_with(&preferred) { return 1; } + if preferred == "stkaiti" && name.starts_with("simkai") { + return 2; + } } let codepoint = ch as u32; let preferred = if matches!(codepoint, 0x0530..=0x058f) { diff --git a/minipdf-rs/crates/minipdf/src/xlsx.rs b/minipdf-rs/crates/minipdf/src/xlsx.rs index 75400318..27658609 100644 --- a/minipdf-rs/crates/minipdf/src/xlsx.rs +++ b/minipdf-rs/crates/minipdf/src/xlsx.rs @@ -30,6 +30,10 @@ const O365_PRINTER_FALLBACK_HORIZONTAL_SCALE: f32 = 1.0046; const O365_PRINTER_FALLBACK_LEFT_OFFSET: f32 = 0.48; const O365_PRINTER_FALLBACK_BORDER_SCALE: f32 = 1.8; const CENTERED_VML_HORIZONTAL_SCALE: f32 = 0.9754; +const CENTERED_VML_VERTICAL_SCALE: f32 = 1.005; +const CENTERED_VML_IMAGE_HORIZONTAL_SCALE: f32 = 1.04; +const CENTERED_VML_IMAGE_VERTICAL_SCALE: f32 = 1.017; +const CENTERED_VML_IMAGE_VERTICAL_OFFSET: f32 = 4.8; const SVG_FALLBACK_HORIZONTAL_SCALE: f32 = 0.972; const GROUP_DRAWING_TOP_OFFSET: f32 = 0.96; const ROW_HEIGHT: f32 = 15.0; @@ -313,6 +317,7 @@ struct SheetImage { data: SheetImageData, pixel_width: u16, pixel_height: u16, + legacy_vml: bool, col: usize, row: usize, col_offset: f32, @@ -975,6 +980,7 @@ fn read_sheet_images( data: SheetImageData::Jpeg(data), pixel_width, pixel_height, + legacy_vml: false, col: child_number(from, "col").unwrap_or(0), row: child_number(from, "row").unwrap_or(0), col_offset: child_number(from, "colOff").unwrap_or(0) as f32 / 12_700.0, @@ -1131,6 +1137,7 @@ fn read_legacy_drawing_images( data, pixel_width, pixel_height, + legacy_vml: true, col, row, col_offset, @@ -1382,6 +1389,7 @@ fn read_two_cell_shape( data: SheetImageData::Rgba(canvas.into_raw()), pixel_width, pixel_height, + legacy_vml: false, col: child_number(from, "col").unwrap_or(0), row: child_number(from, "row").unwrap_or(0), col_offset: child_number(from, "colOff").unwrap_or(0) as f32 / 12_700.0, @@ -1510,6 +1518,7 @@ fn read_two_cell_picture( data: SheetImageData::Rgba(rgba.into_raw()), pixel_width, pixel_height, + legacy_vml: false, col: child_number(from, "col").unwrap_or(0), row: child_number(from, "row").unwrap_or(0), col_offset: child_number(from, "colOff").unwrap_or(0) as f32 / 12_700.0, @@ -1692,6 +1701,7 @@ fn read_group_image( data: SheetImageData::Rgba(canvas.into_raw()), pixel_width, pixel_height, + legacy_vml: false, col: child_number(from, "col").unwrap_or(0), row: child_number(from, "row").unwrap_or(0), col_offset: child_number(from, "colOff").unwrap_or(0) as f32 / 12_700.0, @@ -2798,16 +2808,23 @@ fn excel_pdf_font_size(font_name: Option<&str>, size: f32) -> f32 { } fn xlsx_preferred_font(font_name: Option<&str>) -> Option<&'static str> { - match font_name.unwrap_or_default().to_ascii_lowercase().as_str() { + let normalized_name = font_name + .unwrap_or_default() + .chars() + .filter(|character| !character.is_whitespace()) + .flat_map(char::to_lowercase) + .collect::(); + match normalized_name.as_str() { "arial" => Some("arial"), "corbel" => Some("corbel"), - "franklin gothic medium" => Some("framd"), + "franklingothicmedium" => Some("framd"), "garamond" => Some("gara"), "grandview" => Some("grandview"), - "grandview display" => Some("grandviewdisplay"), - "kaiti" | "stkaiti" | "华文楷体" | "楷体" => Some("simkai"), - "palatino linotype" => Some("bookos"), - "tw cen mt" => Some("tcm_____"), + "grandviewdisplay" => Some("grandviewdisplay"), + "stkaiti" | "华文楷体" => Some("stkaiti"), + "kaiti" | "楷体" => Some("simkai"), + "palatinolinotype" => Some("bookos"), + "twcenmt" => Some("tcm_____"), "verdana" => Some("verdana"), _ => None, } @@ -3946,7 +3963,14 @@ fn render_sheet( } else { 1.0 }, - ) * if sheet.page_setup.o365_printer_fallback { + ) * if sheet.page_setup.horizontal_centered + && sheet.page_setup.vertical_centered + && sheet.page_setup.legacy_vml_drawing + { + CENTERED_VML_VERTICAL_SCALE + } else { + 1.0 + } * if sheet.page_setup.o365_printer_fallback { O365_PRINTER_FALLBACK_VERTICAL_SCALE } else { 1.0 @@ -4222,9 +4246,26 @@ fn render_sheet_columns( for (image, image_id) in sheet.images.iter().zip(image_ids).filter(|(image, _)| { image.foreground && image.col >= column_start && image.col < column_end }) { + let centered_legacy_vml = image.legacy_vml + && sheet.page_setup.horizontal_centered + && sheet.page_setup.vertical_centered; + let image_anchor_horizontal_scale = if centered_legacy_vml { + horizontal_geometry_scale / CENTERED_VML_HORIZONTAL_SCALE + } else { + horizontal_geometry_scale + }; + let image_horizontal_scale = if centered_legacy_vml { + CENTERED_VML_IMAGE_HORIZONTAL_SCALE + } else { + 1.0 + }; + let image_width = image.width * image_anchor_horizontal_scale * image_horizontal_scale; let x = content_left + column_widths[column_start..image.col].iter().sum::() - + image.col_offset * horizontal_geometry_scale; + * image_anchor_horizontal_scale + / horizontal_geometry_scale + + image.col_offset * image_anchor_horizontal_scale + - image.width * image_anchor_horizontal_scale * (image_horizontal_scale - 1.0) / 2.0; let rows_above = (0..image.row) .map(|row_index| { sheet @@ -4238,12 +4279,24 @@ fn render_sheet_columns( let top = page_size.height - margin_top - rows_above * row_scale - image.row_offset * row_scale + GROUP_DRAWING_TOP_OFFSET; + let image_vertical_scale = if centered_legacy_vml { + CENTERED_VML_IMAGE_VERTICAL_SCALE + } else { + 1.0 + }; + let image_height = image.height * row_scale * image_vertical_scale; + let image_vertical_offset = if centered_legacy_vml { + CENTERED_VML_IMAGE_VERTICAL_OFFSET + + image.height * row_scale * (image_vertical_scale - 1.0) / 2.0 + } else { + 0.0 + }; page.add_image( *image_id, x, - top - image.height * row_scale, - image.width * horizontal_geometry_scale, - image.height * row_scale, + top - image.height * row_scale - image_vertical_offset, + image_width, + image_height, ); } @@ -4616,6 +4669,8 @@ fn render_xlsx_row( let mut text_style = cell.style; let wrap_padding = if is_kaiti_slash_date(&cell.text, text_style) { 0.0 + } else if text_style.preferred_font == Some("stkaiti") { + 3.0 } else { 6.0 }; @@ -5859,9 +5914,21 @@ mod tests { #[test] fn maps_kaiti_family_names_to_installed_font() { - for name in ["华文楷体", "STKaiti", "KaiTi", "楷体"] { - assert_eq!(super::xlsx_preferred_font(Some(name)), Some("simkai")); - } + assert_eq!( + super::xlsx_preferred_font(Some("华文楷体")), + Some("stkaiti") + ); + assert_eq!( + super::xlsx_preferred_font(Some("华文 楷体")), + Some("stkaiti") + ); + assert_eq!( + super::xlsx_preferred_font(Some("华 文楷体")), + Some("stkaiti") + ); + assert_eq!(super::xlsx_preferred_font(Some("STKaiti")), Some("stkaiti")); + assert_eq!(super::xlsx_preferred_font(Some("KaiTi")), Some("simkai")); + assert_eq!(super::xlsx_preferred_font(Some("楷体")), Some("simkai")); } #[test] @@ -6036,6 +6103,7 @@ mod tests { data: SheetImageData::Jpeg(Vec::new()), pixel_width: 1, pixel_height: 1, + legacy_vml: false, col: 0, row: 3, col_offset: 0.0, diff --git a/scripts/Invoke-LanguageVisualBenchmark.ps1 b/scripts/Invoke-LanguageVisualBenchmark.ps1 index 5bfdbec4..174f7a03 100644 --- a/scripts/Invoke-LanguageVisualBenchmark.ps1 +++ b/scripts/Invoke-LanguageVisualBenchmark.ps1 @@ -1,39 +1,47 @@ <# .SYNOPSIS - Runs one MiniPdf implementation against the shared XLSX, DOCX, and PPTX corpus. + Runs one MiniPdf implementation against the repository's visual benchmark fixtures. .DESCRIPTION - Every language reads tests/MiniPdf.Benchmark/shared-office-corpus.json and writes - isolated candidates and reports below artifacts/benchmark/. LibreOffice - reference PDFs are shared by content hash across all language runs. + Uses Microsoft 365 as the primary scored reference and LibreOffice as the + required auxiliary reference. Outputs are isolated below + artifacts/-benchmark//. .EXAMPLE - .\scripts\Run-Java-VisualBenchmark.ps1 -Format all -MaxCasesPerFormat 1 - .\scripts\Run-Python-VisualBenchmark.ps1 -Format pptx -Filter "Asian Pacific" + .\scripts\Run-Java-VisualBenchmark.ps1 -Suite classic -Format xlsx -MaxCases 1 + .\scripts\Run-Python-VisualBenchmark.ps1 -Suite issue -Format pptx -Filter "Asian Pacific" #> param( [Parameter(Mandatory = $true)] [ValidateSet("dotnet", "rust", "java", "go", "python", "node")] [string]$Language, - [ValidateSet("all", "xlsx", "docx", "pptx")] - [string]$Format = "all", + [ValidateSet("classic", "issue")] + [string]$Suite = "classic", + [ValidateSet("xlsx", "docx", "pptx")] + [string]$Format = "xlsx", + [ValidateSet("o365", "office", "libre")] + [string]$Engine = "o365", [string]$Filter, - [int]$MaxCasesPerFormat = 0, + [int]$MaxCases = 0, [int]$MaxComparePages = 0, - [double]$MinimumScore = 0.0, - [string]$CorpusManifest = "tests/MiniPdf.Benchmark/shared-office-corpus.json", + [double]$MinimumScore = 0.95, + [string]$SourceDir, + [string]$CandidateDir, + [string]$ReferenceDir, + [string]$AuxiliaryReferenceDir, + [string]$ReportDir, [string]$ArtifactRoot, + [switch]$SkipCandidate, [switch]$SkipBuild, [switch]$SkipReference, - [switch]$SkipCompare, [switch]$ForceReference ) $ErrorActionPreference = "Stop" $RepoRoot = Split-Path -Parent $PSScriptRoot -if ($MaxCasesPerFormat -lt 0) { throw "MaxCasesPerFormat cannot be negative." } +if ($MaxCases -lt 0) { throw "MaxCases cannot be negative." } if ($MaxComparePages -lt 0) { throw "MaxComparePages cannot be negative." } if ($MinimumScore -lt 0.0 -or $MinimumScore -gt 1.0) { throw "MinimumScore must be between 0.0 and 1.0." @@ -132,93 +140,108 @@ function Test-Pdf([string]$Path) { return $Bytes.Length -ge 5 -and [System.Text.Encoding]::ASCII.GetString($Bytes, 0, 5) -eq "%PDF-" } -function Get-LibreOfficePath { - $Command = Get-Command soffice -ErrorAction SilentlyContinue - if ($Command) { - $ConsoleLauncher = [System.IO.Path]::ChangeExtension($Command.Source, ".com") - if ($IsWindows -and (Test-Path -LiteralPath $ConsoleLauncher)) { return $ConsoleLauncher } - return $Command.Source +$Defaults = @{ + "classic:xlsx" = @{ + Source = "tests/MiniPdf.Scripts/output" + LibreReference = "tests/MiniPdf.Benchmark/reference_pdfs" + OfficeReference = "tests/MiniPdf.Benchmark/office_pdfs" + LibreReferenceScript = "tests/MiniPdf.Benchmark/generate_reference_pdfs.py" + OfficeReferenceScript = "tests/MiniPdf.Benchmark/generate_office_pdfs.py" + SourceArgument = "--xlsx-dir" + OfficeLabel = "Microsoft 365 Excel Reference" } - $Candidates = @( - (Join-Path $env:ProgramFiles "LibreOffice/program/soffice.com"), - (Join-Path ${env:ProgramFiles(x86)} "LibreOffice/program/soffice.com"), - (Join-Path $env:ProgramFiles "LibreOffice/program/soffice.exe"), - (Join-Path ${env:ProgramFiles(x86)} "LibreOffice/program/soffice.exe") - ) - foreach ($Candidate in $Candidates) { - if ($Candidate -and (Test-Path -LiteralPath $Candidate)) { return $Candidate } + "classic:docx" = @{ + Source = "tests/MiniPdf.Scripts/output_docx" + LibreReference = "tests/MiniPdf.Benchmark/reference_pdfs_docx" + OfficeReference = "tests/MiniPdf.Benchmark/office_pdfs_docx" + LibreReferenceScript = "tests/MiniPdf.Benchmark/generate_reference_pdfs_docx.py" + OfficeReferenceScript = "tests/MiniPdf.Benchmark/generate_office_pdfs_docx.py" + SourceArgument = "--docx-dir" + OfficeLabel = "Microsoft 365 Word Reference" } - throw "LibreOffice soffice was not found. Install LibreOffice or use -SkipReference." -} - -$CorpusManifest = Resolve-RepoPath $CorpusManifest -if (-not (Test-Path -LiteralPath $CorpusManifest)) { - throw "Shared corpus manifest not found: $CorpusManifest" -} -$Corpus = Get-Content -LiteralPath $CorpusManifest -Raw | ConvertFrom-Json -$SelectedCases = [System.Collections.Generic.List[object]]::new() -$CaseSources = @{} - -foreach ($Source in $Corpus.sources) { - if ($Format -ne "all" -and $Source.format -ne $Format) { continue } - $SourceRoot = Resolve-RepoPath $Source.root - if (-not (Test-Path -LiteralPath $SourceRoot -PathType Container)) { - throw "Corpus source directory not found: $SourceRoot" + "issue:xlsx" = @{ + Source = "tests/Issue_Files/xlsx" + LibreReference = "tests/Issue_Files/reference_xlsx" + OfficeReference = "tests/Issue_Files/office_xlsx" + LibreReferenceScript = "tests/MiniPdf.Benchmark/generate_reference_pdfs.py" + OfficeReferenceScript = "tests/MiniPdf.Benchmark/generate_office_pdfs.py" + SourceArgument = "--xlsx-dir" + OfficeLabel = "Microsoft 365 Excel Reference" } - $Files = @(Get-ChildItem -LiteralPath $SourceRoot -File -Filter $Source.pattern | Where-Object { - -not $Filter -or $_.Name -like "*$Filter*" - } | Sort-Object Name) - if ($MaxCasesPerFormat -gt 0) { - $Files = @($Files | Select-Object -First $MaxCasesPerFormat) + "issue:docx" = @{ + Source = "tests/Issue_Files/docx" + LibreReference = "tests/Issue_Files/reference_docx" + OfficeReference = "tests/Issue_Files/office_docx" + LibreReferenceScript = "tests/MiniPdf.Benchmark/generate_reference_pdfs_docx.py" + OfficeReferenceScript = "tests/MiniPdf.Benchmark/generate_office_pdfs_docx.py" + SourceArgument = "--docx-dir" + OfficeLabel = "Microsoft 365 Word Reference" } - foreach ($File in $Files) { - $Hash = (Get-FileHash -LiteralPath $File.FullName -Algorithm SHA256).Hash.ToLowerInvariant() - $SafeStem = ($File.BaseName -replace '[^A-Za-z0-9._-]', '_').Trim('_') - if (-not $SafeStem) { $SafeStem = "fixture" } - $CaseId = "$($Source.format)--$SafeStem--$($Hash.Substring(0, 24))" - if ($CaseSources.ContainsKey($CaseId)) { throw "Duplicate benchmark case id: $CaseId" } - $RelativePath = [System.IO.Path]::GetRelativePath($RepoRoot, $File.FullName).Replace("\", "/") - $Case = [pscustomobject]@{ - name = $CaseId - case_id = $CaseId - display_name = $File.BaseName - suite = "shared-office" - format = $Source.format - source_path = $RelativePath - source_sha256 = $Hash - conversion_status = "pending" - conversion_exit_code = $null - candidate_exists = $false - reference_exists = $false - } - $SelectedCases.Add($Case) - $CaseSources[$CaseId] = $File.FullName + "issue:pptx" = @{ + Source = "tests/Issue_Files/pptx" + LibreReference = "tests/Issue_Files/reference_pptx" + OfficeReference = "tests/Issue_Files/office_pptx" + LibreReferenceScript = "tests/MiniPdf.Benchmark/generate_reference_pdfs_pptx.py" + OfficeReferenceScript = "tests/MiniPdf.Benchmark/generate_office_pdfs_pptx.py" + SourceArgument = "--pptx-dir" + OfficeLabel = "Microsoft 365 PowerPoint Reference" } } -if ($SelectedCases.Count -eq 0) { - throw "No shared corpus cases matched format=$Format filter='$Filter'." +$Config = $Defaults["$Suite`:$Format"] +if (-not $Config) { + throw "No $Language benchmark fixtures are configured for suite=$Suite format=$Format." +} +if ($Engine -eq "libre") { + Write-Warning "-Engine libre is retained for compatibility. Microsoft 365 remains the primary scored reference; LibreOffice is auxiliary." } -$ArtifactRoot = Resolve-RepoPath $(if ($ArtifactRoot) { $ArtifactRoot } else { "artifacts/benchmark" }) -$SharedRoot = Join-Path $ArtifactRoot "shared" -$LanguageRoot = Join-Path $ArtifactRoot $Language -$CandidateDir = Join-Path $LanguageRoot "candidates" -$ReportDir = Join-Path $LanguageRoot "report" -$ReferenceDir = Join-Path $SharedRoot "libreoffice-reference" -$ResolvedManifest = Join-Path $LanguageRoot "resolved-manifest.json" -$CoverageManifest = Join-Path $LanguageRoot "benchmark-coverage.json" -$ReferenceWorkDir = Join-Path $SharedRoot "reference-work" +$SourceDir = Resolve-RepoPath $(if ($SourceDir) { $SourceDir } else { $Config.Source }) +$ReferenceDir = Resolve-RepoPath $(if ($ReferenceDir) { $ReferenceDir } else { $Config.OfficeReference }) +$AuxiliaryReferenceDir = Resolve-RepoPath $(if ($AuxiliaryReferenceDir) { $AuxiliaryReferenceDir } else { $Config.LibreReference }) +$ArtifactRoot = Resolve-RepoPath $(if ($ArtifactRoot) { $ArtifactRoot } else { "artifacts/$Language-benchmark/$Suite/$Format" }) +$CandidateDir = Resolve-RepoPath $(if ($CandidateDir) { $CandidateDir } else { Join-Path $ArtifactRoot "candidates" }) +$ReportDir = Resolve-RepoPath $(if ($ReportDir) { $ReportDir } else { Join-Path $ArtifactRoot "report" }) +$ComparisonManifest = Join-Path $ReportDir "comparison_manifest.json" +$CoverageManifest = Join-Path $ReportDir "benchmark_coverage.json" -New-Item -ItemType Directory -Force -Path $CandidateDir, $ReportDir, $ReferenceDir, $ReferenceWorkDir | Out-Null -Write-Json ([pscustomobject]@{ - corpus = [System.IO.Path]::GetRelativePath($RepoRoot, $CorpusManifest).Replace("\", "/") - corpus_version = $Corpus.version - cases = $SelectedCases -}) $ResolvedManifest +$SourceFiles = @(Get-ChildItem -LiteralPath $SourceDir -File -Filter "*.$Format" | Where-Object { + -not $Filter -or $_.BaseName -like "*$Filter*" +} | Sort-Object Name) +if ($MaxCases -gt 0) { + $SourceFiles = @($SourceFiles | Select-Object -First $MaxCases) +} +if ($SourceFiles.Count -eq 0) { + throw "No .$Format files matched '$Filter' in $SourceDir" +} + +if (Test-Path -LiteralPath $ReportDir) { + Remove-Item -LiteralPath $ReportDir -Recurse -Force +} +New-Item -ItemType Directory -Force -Path $CandidateDir, $ReferenceDir, $AuxiliaryReferenceDir, $ReportDir | Out-Null + +$SelectedCases = @($SourceFiles | ForEach-Object { + [pscustomobject]@{ + name = $_.BaseName + case_id = $_.BaseName + suite = $Suite + format = $Format + source_path = [System.IO.Path]::GetRelativePath($RepoRoot, $_.FullName).Replace("\", "/") + conversion_status = "pending" + conversion_exit_code = $null + candidate_exists = $false + reference_exists = $false + auxiliary_reference_exists = $false + } +}) +$CaseSources = @{} +for ($Index = 0; $Index -lt $SelectedCases.Count; $Index++) { + $CaseSources[$SelectedCases[$Index].case_id] = $SourceFiles[$Index].FullName +} +Write-Json ([pscustomobject]@{ cases = $SelectedCases }) $ComparisonManifest $Tools = @{} -if (-not $SkipBuild) { +if (-not $SkipCandidate -and -not $SkipBuild) { switch ($Language) { "dotnet" { $Tools.dotnet = Find-Command "dotnet" @@ -239,7 +262,7 @@ if (-not $SkipBuild) { } "go" { $Tools.go = Find-Go - $GoOutput = Join-Path $LanguageRoot $(if ($IsWindows) { "minipdf-go.exe" } else { "minipdf-go" }) + $GoOutput = Join-Path $ArtifactRoot $(if ($IsWindows) { "minipdf-go.exe" } else { "minipdf-go" }) Push-Location (Join-Path $RepoRoot "minipdf-go") try { & $Tools.go build -o $GoOutput ./cmd/minipdf } finally { Pop-Location } Assert-CommandSucceeded "Go CLI build" @@ -260,148 +283,147 @@ if (-not $SkipBuild) { } } -switch ($Language) { - "dotnet" { - if (-not $Tools.dotnet) { $Tools.dotnet = Find-Command "dotnet" } - $Tools.cli = Get-ChildItem (Join-Path $RepoRoot "src/MiniPdf.Cli/bin/Release") -Recurse -Filter "MiniPdf.Cli.dll" | - Where-Object FullName -Match 'net9\.0' | Select-Object -First 1 -ExpandProperty FullName - } - "rust" { - $RustName = if ($IsWindows) { "minipdf.exe" } else { "minipdf" } - $Tools.cli = Join-Path $RepoRoot "minipdf-rs/target/release/$RustName" - } - "java" { - if (-not $Tools.java) { $Tools.java = Find-JavaExecutable } - $Tools.cli = Get-ChildItem (Join-Path $RepoRoot "minipdf-java/minipdf-cli/target") -Filter "minipdf-cli-*.jar" | - Where-Object { $_.Name -notmatch '(sources|javadoc|original)' } | - Sort-Object LastWriteTime -Descending | Select-Object -First 1 -ExpandProperty FullName - } - "go" { - $Tools.cli = Join-Path $LanguageRoot $(if ($IsWindows) { "minipdf-go.exe" } else { "minipdf-go" }) - } - "python" { - if (-not $Tools.python) { - $Tools.python = Find-Python +if (-not $SkipCandidate) { + switch ($Language) { + "dotnet" { + if (-not $Tools.dotnet) { $Tools.dotnet = Find-Command "dotnet" } + $Tools.cli = Get-ChildItem (Join-Path $RepoRoot "src/MiniPdf.Cli/bin/Release") -Recurse -Filter "MiniPdf.Cli.dll" | + Where-Object FullName -Match 'net9\.0' | Select-Object -First 1 -ExpandProperty FullName } - $env:PYTHONPATH = Join-Path $RepoRoot "minipdf-python/src" + "rust" { + $RustName = if ($IsWindows) { "minipdf.exe" } else { "minipdf" } + $Tools.cli = Join-Path $RepoRoot "minipdf-rs/target/release/$RustName" + } + "java" { + if (-not $Tools.java) { $Tools.java = Find-JavaExecutable } + $Tools.cli = Get-ChildItem (Join-Path $RepoRoot "minipdf-java/minipdf-cli/target") -Filter "minipdf-cli-*.jar" | + Where-Object { $_.Name -notmatch '(sources|javadoc|original)' } | + Sort-Object LastWriteTime -Descending | Select-Object -First 1 -ExpandProperty FullName + } + "go" { + $Tools.cli = Join-Path $ArtifactRoot $(if ($IsWindows) { "minipdf-go.exe" } else { "minipdf-go" }) + } + "python" { + if (-not $Tools.python) { $Tools.python = Find-Python } + $env:PYTHONPATH = Join-Path $RepoRoot "minipdf-python/src" + } + "node" { $Tools.node = Find-Command "node" } } - "node" { $Tools.node = Find-Command "node" } -} -if ($Language -notin @("python", "node") -and (-not $Tools.cli -or -not (Test-Path -LiteralPath $Tools.cli))) { - throw "$Language CLI artifact was not found. Run without -SkipBuild first." -} + if ($Language -notin @("python", "node") -and (-not $Tools.cli -or -not (Test-Path -LiteralPath $Tools.cli))) { + throw "$Language CLI artifact was not found. Run without -SkipBuild first." + } -foreach ($Case in $SelectedCases) { - $InputPath = $CaseSources[$Case.case_id] - $OutputPath = Join-Path $CandidateDir ($Case.name + ".pdf") - if (Test-Path -LiteralPath $OutputPath) { Remove-Item -LiteralPath $OutputPath -Force } - switch ($Language) { - "dotnet" { & $Tools.dotnet $Tools.cli $InputPath -o $OutputPath } - "rust" { & $Tools.cli $InputPath -o $OutputPath } - "java" { & $Tools.java -jar $Tools.cli $InputPath -o $OutputPath } - "go" { & $Tools.cli $InputPath -o $OutputPath } - "python" { & $Tools.python -m minipdf $InputPath -o $OutputPath } - "node" { - & $Tools.node -e "require(process.argv[1]).convertToPdf(process.argv[2], process.argv[3])" ` - (Join-Path $RepoRoot "minipdf-node") $InputPath $OutputPath + foreach ($Case in $SelectedCases) { + $InputPath = $CaseSources[$Case.case_id] + $OutputPath = Join-Path $CandidateDir ($Case.name + ".pdf") + if (Test-Path -LiteralPath $OutputPath) { Remove-Item -LiteralPath $OutputPath -Force } + switch ($Language) { + "dotnet" { & $Tools.dotnet $Tools.cli $InputPath -o $OutputPath } + "rust" { & $Tools.cli $InputPath -o $OutputPath } + "java" { & $Tools.java -jar $Tools.cli $InputPath -o $OutputPath } + "go" { & $Tools.cli $InputPath -o $OutputPath } + "python" { & $Tools.python -m minipdf $InputPath -o $OutputPath } + "node" { + & $Tools.node -e "require(process.argv[1]).convertToPdf(process.argv[2], process.argv[3])" ` + (Join-Path $RepoRoot "minipdf-node") $InputPath $OutputPath + } } + $Case.conversion_exit_code = $LASTEXITCODE + $Case.candidate_exists = Test-Pdf $OutputPath + $Case.conversion_status = if ($LASTEXITCODE -eq 0 -and $Case.candidate_exists) { "passed" } else { "failed" } + } +} else { + foreach ($Case in $SelectedCases) { + $OutputPath = Join-Path $CandidateDir ($Case.name + ".pdf") + $Case.candidate_exists = Test-Pdf $OutputPath + $Case.conversion_status = if ($Case.candidate_exists) { "passed" } else { "failed" } } - $Case.conversion_exit_code = $LASTEXITCODE - $Case.candidate_exists = Test-Pdf $OutputPath - $Case.conversion_status = if ($LASTEXITCODE -eq 0 -and $Case.candidate_exists) { "passed" } else { "failed" } } if (-not $SkipReference) { - $Soffice = Get-LibreOfficePath - foreach ($Case in $SelectedCases) { - $ReferencePath = Join-Path $ReferenceDir ($Case.name + ".pdf") - if ($ForceReference -or -not (Test-Pdf $ReferencePath)) { - Get-ChildItem -LiteralPath $ReferenceWorkDir -File -Filter "*.pdf" | Remove-Item -Force - $ProfileDir = Join-Path $ReferenceWorkDir ("profile-" + [System.Guid]::NewGuid().ToString("N")) - New-Item -ItemType Directory -Force -Path $ProfileDir | Out-Null - $ProfileUri = ([System.Uri]$ProfileDir).AbsoluteUri - try { - & $Soffice --headless --norestore "-env:UserInstallation=$ProfileUri" ` - --convert-to pdf --outdir $ReferenceWorkDir $CaseSources[$Case.case_id] - Assert-CommandSucceeded "LibreOffice conversion for $($Case.source_path)" - } finally { - Remove-Item -LiteralPath $ProfileDir -Recurse -Force -ErrorAction SilentlyContinue - } - $GeneratedPath = Join-Path $ReferenceWorkDir ([System.IO.Path]::GetFileNameWithoutExtension($CaseSources[$Case.case_id]) + ".pdf") - if (-not (Test-Pdf $GeneratedPath)) { throw "LibreOffice did not produce a valid PDF for $($Case.source_path)." } - Move-Item -LiteralPath $GeneratedPath -Destination $ReferencePath -Force + $Python = Find-Python + $OfficeReferenceScript = Resolve-RepoPath $Config.OfficeReferenceScript + $LibreReferenceScript = Resolve-RepoPath $Config.LibreReferenceScript + $ReferenceFilters = if ($MaxCases -gt 0) { @($SourceFiles.BaseName) } else { @($Filter) } + foreach ($ReferenceFilter in $ReferenceFilters) { + $Providers = @( + [pscustomobject]@{ Script = $OfficeReferenceScript; Directory = $ReferenceDir; Label = $Config.OfficeLabel }, + [pscustomobject]@{ Script = $LibreReferenceScript; Directory = $AuxiliaryReferenceDir; Label = "LibreOffice" } + ) + foreach ($Provider in $Providers) { + $ReferenceArgs = @($Provider.Script, $Config.SourceArgument, $SourceDir, "--pdf-dir", $Provider.Directory) + if ($ReferenceFilter) { $ReferenceArgs += @("--filter", $ReferenceFilter) } + if ($ForceReference) { $ReferenceArgs += "--force" } + & $Python -X utf8 @ReferenceArgs + Assert-CommandSucceeded "$($Provider.Label) generation" } } } foreach ($Case in $SelectedCases) { $Case.reference_exists = Test-Pdf (Join-Path $ReferenceDir ($Case.name + ".pdf")) + $Case.auxiliary_reference_exists = Test-Pdf (Join-Path $AuxiliaryReferenceDir ($Case.name + ".pdf")) } $PassedConversions = @($SelectedCases | Where-Object conversion_status -eq "passed").Count $MissingReferences = @($SelectedCases | Where-Object reference_exists -eq $false).Count +$MissingAuxiliaryReferences = @($SelectedCases | Where-Object auxiliary_reference_exists -eq $false).Count $Coverage = [pscustomobject]@{ language = $Language - corpus_manifest = [System.IO.Path]::GetRelativePath($RepoRoot, $CorpusManifest).Replace("\", "/") - resolved_manifest = [System.IO.Path]::GetRelativePath($RepoRoot, $ResolvedManifest).Replace("\", "/") - fixture_scope = "shared-git-tracked-office-corpus" + suite = $Suite + format = $Format + reference_engine = "o365" + reference_label = $Config.OfficeLabel + auxiliary_reference_engine = "libreoffice" + auxiliary_reference_label = "LibreOffice (auxiliary)" + fixture_scope = "shared-on-disk-fixtures" + executes_dotnet_xunit = $false + max_compare_pages = $MaxComparePages selected_cases = $SelectedCases.Count passed_conversions = $PassedConversions failed_conversions = $SelectedCases.Count - $PassedConversions missing_references = $MissingReferences + missing_auxiliary_references = $MissingAuxiliaryReferences comparison_completed = $false comparison_results = 0 average_score = $null cases = $SelectedCases } Write-Json $Coverage $CoverageManifest -Write-Json ([pscustomobject]@{ - corpus = [System.IO.Path]::GetRelativePath($RepoRoot, $CorpusManifest).Replace("\", "/") - corpus_version = $Corpus.version - cases = $SelectedCases -}) $ResolvedManifest -if (-not $SkipCompare) { - if ($MissingReferences -gt 0) { - throw "$MissingReferences shared reference PDFs are missing. Run without -SkipReference." - } - $Python = if (Test-Path (Join-Path $RepoRoot ".venv/Scripts/python.exe")) { - Join-Path $RepoRoot ".venv/Scripts/python.exe" - } else { Find-Command "python" } - $CompareArgs = @( - (Join-Path $RepoRoot "tests/MiniPdf.Benchmark/compare_pdfs.py"), - "--minipdf-dir", $CandidateDir, - "--reference-dir", $ReferenceDir, - "--report-dir", $ReportDir, - "--manifest", $ResolvedManifest, - "--report-scope", "$Language-shared-office", - "--candidate-label", "$Language MiniPdf", - "--reference-label", "LibreOffice", - "--composite-images", - "--heatmaps" - ) - if ($MaxComparePages -gt 0) { $CompareArgs += @("--max-pages", $MaxComparePages) } - & $Python -X utf8 @CompareArgs - Assert-CommandSucceeded "$Language visual comparison" - $Results = @(Get-Content (Join-Path $ReportDir "comparison_report.json") -Raw | ConvertFrom-Json) - $Scores = @($Results | Where-Object { $null -ne $_.overall_score }) - $Coverage.comparison_completed = $true - $Coverage.comparison_results = ($Results | Measure-Object).Count - $Coverage.average_score = if ($Scores.Count -gt 0) { - ($Scores | Measure-Object -Property overall_score -Average).Average - } else { $null } - Write-Json $Coverage $CoverageManifest - $BelowThreshold = @($Results | Where-Object { $null -eq $_.overall_score -or $_.overall_score -lt $MinimumScore }) - if ($BelowThreshold.Count -gt 0) { - throw "$($BelowThreshold.Count) cases scored below MinimumScore=$MinimumScore." - } -} +$Python = Find-Python +$CompareArgs = @( + (Join-Path $RepoRoot "tests/MiniPdf.Benchmark/compare_pdfs.py"), + "--minipdf-dir", $CandidateDir, + "--reference-dir", $ReferenceDir, + "--auxiliary-dir", $AuxiliaryReferenceDir, + "--report-dir", $ReportDir, + "--manifest", $ComparisonManifest, + "--report-scope", "$Language-$Suite-$Format", + "--candidate-label", "$Language MiniPdf", + "--reference-label", $Config.OfficeLabel, + "--auxiliary-label", "LibreOffice", + "--composite-images", + "--heatmaps" +) +if ($MaxComparePages -gt 0) { $CompareArgs += @("--max-pages", $MaxComparePages) } +& $Python -X utf8 @CompareArgs +Assert-CommandSucceeded "$Language visual comparison" +$Results = @(Get-Content (Join-Path $ReportDir "comparison_report.json") -Raw | ConvertFrom-Json) +$Scores = @($Results | Where-Object { $null -ne $_.overall_score }) +$Coverage.comparison_completed = $true +$Coverage.comparison_results = ($Results | Measure-Object).Count +$Coverage.average_score = if ($Scores.Count -gt 0) { + ($Scores | Measure-Object -Property overall_score -Average).Average +} else { $null } +Write-Json $Coverage $CoverageManifest +$BelowThreshold = @($Results | Where-Object { $null -eq $_.overall_score -or $_.overall_score -lt $MinimumScore }) -Write-Host "$Language benchmark: selected=$($SelectedCases.Count), converted=$PassedConversions, missing references=$MissingReferences" +Write-Host "$Language benchmark: suite=$Suite format=$Format selected=$($SelectedCases.Count), converted=$PassedConversions, missing O365 references=$MissingReferences, missing LibreOffice references=$MissingAuxiliaryReferences" Write-Host "Coverage: $CoverageManifest" -if (-not $SkipCompare) { Write-Host "Report: $(Join-Path $ReportDir 'comparison_report.md')" } +Write-Host "Report: $(Join-Path $ReportDir 'comparison_report.md')" -if ($PassedConversions -ne $SelectedCases.Count) { - throw "$Language candidate conversion failed for $($SelectedCases.Count - $PassedConversions) cases." -} \ No newline at end of file +if ($PassedConversions -ne $SelectedCases.Count -or $MissingReferences -gt 0 -or $MissingAuxiliaryReferences -gt 0 -or $BelowThreshold.Count -gt 0) { + throw "$Language benchmark failed: conversion failures=$($SelectedCases.Count - $PassedConversions), missing O365 references=$MissingReferences, missing LibreOffice references=$MissingAuxiliaryReferences, below MinimumScore=$($BelowThreshold.Count)." +} diff --git a/scripts/Run-Rust-Benchmark-Matrix.ps1 b/scripts/Run-Rust-Benchmark-Matrix.ps1 index 477fc990..c7fe6d12 100644 --- a/scripts/Run-Rust-Benchmark-Matrix.ps1 +++ b/scripts/Run-Rust-Benchmark-Matrix.ps1 @@ -9,7 +9,7 @@ param( [int]$MaxComparePages = 1, - [double]$MinimumScore = 0, + [double]$MinimumScore = 0.95, [ValidateSet("o365", "office", "libre")] [string]$Engine = "o365", [switch]$SkipReference, diff --git a/scripts/Run-Rust-Benchmark.ps1 b/scripts/Run-Rust-Benchmark.ps1 index 5032e5aa..bdd5e07c 100644 --- a/scripts/Run-Rust-Benchmark.ps1 +++ b/scripts/Run-Rust-Benchmark.ps1 @@ -1,279 +1,2 @@ -<# -.SYNOPSIS - Compare Rust MiniPdf against the repository's shared visual fixtures and references. - -.DESCRIPTION - Reuses the same on-disk classic/issue fixtures and PDF comparison pipeline as - the .NET benchmarks. Microsoft 365 is always the primary scored reference, - while LibreOffice is generated and displayed as an auxiliary reference. It - does not execute C# xUnit tests. - -.EXAMPLE - .\scripts\Run-Rust-Benchmark.ps1 -Suite classic -Format xlsx - .\scripts\Run-Rust-Benchmark.ps1 -Suite classic -Format xlsx -Filter "classic180" -ForceReference - .\scripts\Run-Rust-Benchmark.ps1 -Suite classic -Format docx -Filter "classic01" - .\scripts\Run-Rust-Benchmark.ps1 -Suite issue -Format xlsx - .\scripts\Run-Rust-Benchmark.ps1 -Suite issue -Format docx -Filter "SA8000" - .\scripts\Run-Rust-Benchmark.ps1 -Suite issue -Format pptx -Filter "Asian Pacific" -#> - -param( - [ValidateSet("classic", "issue")] - [string]$Suite = "classic", - [ValidateSet("xlsx", "docx", "pptx")] - [string]$Format = "xlsx", - [ValidateSet("o365", "office", "libre")] - [string]$Engine = "o365", - [string]$Filter, - [int]$MaxCases = 0, - [int]$MaxComparePages = 0, - [string]$SourceDir, - [string]$CandidateDir, - [string]$ReferenceDir, - [string]$AuxiliaryReferenceDir, - [string]$ReportDir, - [double]$MinimumScore = 0.95, - [switch]$SkipCandidate, - [switch]$ForceReference, - [switch]$SkipReference -) - -$ErrorActionPreference = "Stop" -$RepoRoot = Split-Path -Parent $PSScriptRoot - -if ($MaxCases -lt 0) { - throw "MaxCases must be zero (all cases) or a positive number." -} -if ($MaxComparePages -lt 0) { - throw "MaxComparePages must be zero (all pages) or a positive number." -} - -function Resolve-RepoPath([string]$PathValue) { - if ([System.IO.Path]::IsPathRooted($PathValue)) { return $PathValue } - return Join-Path $RepoRoot $PathValue -} - -$Defaults = @{ - "classic:xlsx" = @{ - Source = "tests/MiniPdf.Scripts/output" - LibreReference = "tests/MiniPdf.Benchmark/reference_pdfs" - OfficeReference = "tests/MiniPdf.Benchmark/office_pdfs" - LibreReferenceScript = "tests/MiniPdf.Benchmark/generate_reference_pdfs.py" - OfficeReferenceScript = "tests/MiniPdf.Benchmark/generate_office_pdfs.py" - SourceArgument = "--xlsx-dir" - OfficeLabel = "Microsoft 365 Excel Reference" - } - "classic:docx" = @{ - Source = "tests/MiniPdf.Scripts/output_docx" - LibreReference = "tests/MiniPdf.Benchmark/reference_pdfs_docx" - OfficeReference = "tests/MiniPdf.Benchmark/office_pdfs_docx" - LibreReferenceScript = "tests/MiniPdf.Benchmark/generate_reference_pdfs_docx.py" - OfficeReferenceScript = "tests/MiniPdf.Benchmark/generate_office_pdfs_docx.py" - SourceArgument = "--docx-dir" - OfficeLabel = "Microsoft 365 Word Reference" - } - "issue:xlsx" = @{ - Source = "tests/Issue_Files/xlsx" - LibreReference = "tests/Issue_Files/reference_xlsx" - OfficeReference = "tests/Issue_Files/office_xlsx" - LibreReferenceScript = "tests/MiniPdf.Benchmark/generate_reference_pdfs.py" - OfficeReferenceScript = "tests/MiniPdf.Benchmark/generate_office_pdfs.py" - SourceArgument = "--xlsx-dir" - OfficeLabel = "Microsoft 365 Excel Reference" - } - "issue:docx" = @{ - Source = "tests/Issue_Files/docx" - LibreReference = "tests/Issue_Files/reference_docx" - OfficeReference = "tests/Issue_Files/office_docx" - LibreReferenceScript = "tests/MiniPdf.Benchmark/generate_reference_pdfs_docx.py" - OfficeReferenceScript = "tests/MiniPdf.Benchmark/generate_office_pdfs_docx.py" - SourceArgument = "--docx-dir" - OfficeLabel = "Microsoft 365 Word Reference" - } - "issue:pptx" = @{ - Source = "tests/Issue_Files/pptx" - LibreReference = "tests/Issue_Files/reference_pptx" - OfficeReference = "tests/Issue_Files/office_pptx" - LibreReferenceScript = "tests/MiniPdf.Benchmark/generate_reference_pdfs_pptx.py" - OfficeReferenceScript = "tests/MiniPdf.Benchmark/generate_office_pdfs_pptx.py" - SourceArgument = "--pptx-dir" - OfficeLabel = "Microsoft 365 PowerPoint Reference" - } -} - -$Config = $Defaults["$Suite`:$Format"] -if (-not $Config) { - throw "No Rust benchmark fixtures are configured for suite=$Suite format=$Format." -} -$SourceDir = Resolve-RepoPath $(if ($SourceDir) { $SourceDir } else { $Config.Source }) -$ReferenceDir = Resolve-RepoPath $(if ($ReferenceDir) { $ReferenceDir } else { $Config.OfficeReference }) -$AuxiliaryReferenceDir = Resolve-RepoPath $(if ($AuxiliaryReferenceDir) { $AuxiliaryReferenceDir } else { $Config.LibreReference }) -$ReferenceLabel = $Config.OfficeLabel -$AuxiliaryReferenceLabel = "LibreOffice" -if ($Engine -eq "libre") { - Write-Warning "-Engine libre is retained for compatibility. Microsoft 365 remains the primary scored reference; LibreOffice is auxiliary." -} -$IsFocusedRun = -not [string]::IsNullOrWhiteSpace($Filter) -or $MaxCases -gt 0 -$DefaultArtifactRoot = "artifacts/rust-benchmark/$Suite/$Format" -if ($IsFocusedRun) { - $FilterLabel = if ([string]::IsNullOrWhiteSpace($Filter)) { "all" } else { $Filter -replace '[^A-Za-z0-9._-]', '_' } - $CaseLabel = if ($MaxCases -gt 0) { "max-$MaxCases" } else { "all" } - $DefaultArtifactRoot = "artifacts/rust-benchmark/focused/$Suite/$Format/$FilterLabel-$CaseLabel" -} -$CandidateDir = Resolve-RepoPath $(if ($CandidateDir) { $CandidateDir } else { "$DefaultArtifactRoot/candidates" }) -$ReportDir = Resolve-RepoPath $(if ($ReportDir) { $ReportDir } else { "$DefaultArtifactRoot/report" }) - -$Cargo = Join-Path $env:USERPROFILE ".cargo/bin/cargo.exe" -$Python = Join-Path $RepoRoot ".venv/Scripts/python.exe" -if (-not (Test-Path $Cargo)) { $Cargo = (Get-Command cargo -ErrorAction Stop).Source } -if (-not (Test-Path $Python)) { $Python = (Get-Command python -ErrorAction Stop).Source } - -$CargoManifest = Join-Path $RepoRoot "minipdf-rs/Cargo.toml" -$OfficeReferenceScript = Resolve-RepoPath $Config.OfficeReferenceScript -$LibreReferenceScript = Resolve-RepoPath $Config.LibreReferenceScript -$CompareScript = Join-Path $RepoRoot "tests/MiniPdf.Benchmark/compare_pdfs.py" -$ComparisonManifest = Join-Path $ReportDir "comparison_manifest.json" -$CoverageManifest = Join-Path $ReportDir "benchmark_coverage.json" - -$SourceFiles = @(Get-ChildItem $SourceDir -File -Filter "*.$Format" | Where-Object { - -not $Filter -or $_.BaseName -like "*$Filter*" -} | Sort-Object Name) -if ($MaxCases -gt 0) { - $SourceFiles = @($SourceFiles | Select-Object -First $MaxCases) -} -if ($SourceFiles.Count -eq 0) { - throw "No .$Format files matched '$Filter' in $SourceDir" -} - -if (Test-Path -LiteralPath $ReportDir) { - Remove-Item -LiteralPath $ReportDir -Recurse -Force -} -New-Item -ItemType Directory -Force -Path $CandidateDir, $ReferenceDir, $AuxiliaryReferenceDir, $ReportDir | Out-Null - -$Cases = @($SourceFiles | ForEach-Object { - [pscustomobject]@{ - name = $_.BaseName - case_id = $_.BaseName - suite = $Suite - format = $Format - source_path = [System.IO.Path]::GetRelativePath($RepoRoot, $_.FullName).Replace("\", "/") - conversion_status = "pending" - conversion_exit_code = $null - candidate_exists = $false - reference_exists = $false - auxiliary_reference_exists = $false - } -}) - -[pscustomobject]@{ cases = $Cases } | ConvertTo-Json -Depth 5 | Set-Content $ComparisonManifest -Encoding UTF8 - -Write-Host "Rust benchmark matrix: suite=$Suite format=$Format primary=o365 auxiliary=libreoffice selected=$($Cases.Count)" -Write-Host "Shared fixtures only; C# xUnit assertions are not executed by this command." - -$Cli = Join-Path $RepoRoot "minipdf-rs/target/release/minipdf.exe" -if (-not $SkipCandidate) { - & $Cargo build --release --manifest-path $CargoManifest -p minipdf-cli - if ($LASTEXITCODE -ne 0) { throw "Rust CLI build failed." } - - for ($Index = 0; $Index -lt $SourceFiles.Count; $Index++) { - $SourceFile = $SourceFiles[$Index] - $OutputFile = Join-Path $CandidateDir ($SourceFile.BaseName + ".pdf") - if (Test-Path $OutputFile) { Remove-Item $OutputFile -Force } - - & $Cli $SourceFile.FullName -o $OutputFile - $ExitCode = $LASTEXITCODE - $Cases[$Index].conversion_exit_code = $ExitCode - $Cases[$Index].candidate_exists = Test-Path $OutputFile - $Cases[$Index].conversion_status = if ($ExitCode -eq 0 -and $Cases[$Index].candidate_exists) { "passed" } else { "failed" } - } -} else { - for ($Index = 0; $Index -lt $SourceFiles.Count; $Index++) { - $OutputFile = Join-Path $CandidateDir ($SourceFiles[$Index].BaseName + ".pdf") - $Cases[$Index].candidate_exists = Test-Path $OutputFile - $Cases[$Index].conversion_status = if ($Cases[$Index].candidate_exists) { "passed" } else { "failed" } - } -} - -if (-not $SkipReference) { - $ReferenceFilters = if ($MaxCases -gt 0) { @($SourceFiles.BaseName) } else { @($Filter) } - foreach ($ReferenceFilter in $ReferenceFilters) { - $Providers = @( - [pscustomobject]@{ Script = $OfficeReferenceScript; Directory = $ReferenceDir; Label = $ReferenceLabel }, - [pscustomobject]@{ Script = $LibreReferenceScript; Directory = $AuxiliaryReferenceDir; Label = $AuxiliaryReferenceLabel } - ) - foreach ($Provider in $Providers) { - $ReferenceArgs = @($Provider.Script, $Config.SourceArgument, $SourceDir, "--pdf-dir", $Provider.Directory) - if ($ReferenceFilter) { $ReferenceArgs += @("--filter", $ReferenceFilter) } - if ($ForceReference) { $ReferenceArgs += "--force" } - & $Python -X utf8 @ReferenceArgs - if ($LASTEXITCODE -ne 0) { throw "$($Provider.Label) generation failed." } - } - } -} - -foreach ($Case in $Cases) { - $Case.reference_exists = Test-Path (Join-Path $ReferenceDir ($Case.name + ".pdf")) - $Case.auxiliary_reference_exists = Test-Path (Join-Path $AuxiliaryReferenceDir ($Case.name + ".pdf")) -} - -$PassedConversions = @($Cases | Where-Object { $_.conversion_status -eq "passed" }).Count -$FailedConversions = $Cases.Count - $PassedConversions -$MissingReferences = @($Cases | Where-Object { -not $_.reference_exists }).Count -$MissingAuxiliaryReferences = @($Cases | Where-Object { -not $_.auxiliary_reference_exists }).Count -$Coverage = [pscustomobject]@{ - suite = $Suite - format = $Format - reference_engine = "o365" - reference_label = $ReferenceLabel - auxiliary_reference_engine = "libreoffice" - auxiliary_reference_label = "$AuxiliaryReferenceLabel (auxiliary)" - fixture_scope = "shared-on-disk-fixtures" - executes_dotnet_xunit = $false - max_compare_pages = $MaxComparePages - selected_cases = $Cases.Count - passed_conversions = $PassedConversions - failed_conversions = $FailedConversions - missing_references = $MissingReferences - missing_auxiliary_references = $MissingAuxiliaryReferences - comparison_completed = $false - comparison_results = 0 - average_score = $null - cases = $Cases -} -$Coverage | ConvertTo-Json -Depth 6 | Set-Content $CoverageManifest -Encoding UTF8 - -$CompareArgs = @( - $CompareScript, - "--minipdf-dir", $CandidateDir, - "--reference-dir", $ReferenceDir, - "--auxiliary-dir", $AuxiliaryReferenceDir, - "--report-dir", $ReportDir, - "--manifest", $ComparisonManifest, - "--report-scope", "rust-$Suite-$Format", - "--composite-images", - "--heatmaps", - "--candidate-label", "Rust MiniPdf", - "--reference-label", $ReferenceLabel, - "--auxiliary-label", $AuxiliaryReferenceLabel -) -if ($MaxComparePages -gt 0) { $CompareArgs += @("--max-pages", $MaxComparePages) } -& $Python -X utf8 @CompareArgs -if ($LASTEXITCODE -ne 0) { throw "PDF comparison failed." } - -$Results = @(Get-Content (Join-Path $ReportDir "comparison_report.json") -Raw | ConvertFrom-Json) -$BelowThreshold = @($Results | Where-Object { $null -eq $_.overall_score -or $_.overall_score -lt $MinimumScore }) -$Average = ($Results | Where-Object { $null -ne $_.overall_score } | Measure-Object -Property overall_score -Average).Average -$Coverage.comparison_completed = $true -$Coverage.comparison_results = $Results.Count -$Coverage.average_score = $Average -$Coverage | ConvertTo-Json -Depth 6 | Set-Content $CoverageManifest -Encoding UTF8 - -Write-Host "Coverage: selected=$($Cases.Count), converted=$PassedConversions, failed=$FailedConversions, missing O365 references=$MissingReferences, missing LibreOffice references=$MissingAuxiliaryReferences" -Write-Host "Visual results: compared=$($Results.Count), average score=$([math]::Round($Average, 4))" -Write-Host "Coverage manifest: $CoverageManifest" -Write-Host "Visual report: $(Join-Path $ReportDir 'comparison_report.md')" - -if ($FailedConversions -gt 0 -or $MissingReferences -gt 0 -or $MissingAuxiliaryReferences -gt 0 -or $BelowThreshold.Count -gt 0) { - $ThresholdFailures = ($BelowThreshold | ForEach-Object { "$($_.name)=$($_.overall_score)" }) -join ", " - throw "Rust benchmark failed: conversion failures=$FailedConversions, missing O365 references=$MissingReferences, missing LibreOffice references=$MissingAuxiliaryReferences, below $MinimumScore=[$ThresholdFailures]" -} \ No newline at end of file +param() +& (Join-Path $PSScriptRoot "Invoke-LanguageVisualBenchmark.ps1") -Language rust @args \ No newline at end of file diff --git a/tests/MiniPdf.Benchmark/README.md b/tests/MiniPdf.Benchmark/README.md index 26ac8b4f..d10a1c2b 100644 --- a/tests/MiniPdf.Benchmark/README.md +++ b/tests/MiniPdf.Benchmark/README.md @@ -1,6 +1,6 @@ # MiniPdf Self-Evolution Benchmark -Automatically compares PDFs generated by MiniPdf against LibreOffice (reference implementation), driving continuous rendering quality improvements. +Automatically compares PDFs generated by MiniPdf against Microsoft 365 and LibreOffice, driving continuous rendering quality improvements. ## Architecture Overview @@ -39,43 +39,48 @@ Automatically compares PDFs generated by MiniPdf against LibreOffice (reference ### Shared Cross-Language Corpus -All implementations use the Git-tracked fixture roots declared in -`shared-office-corpus.json`. The runner resolves the same XLSX, DOCX, and PPTX -files for every language, records each source SHA-256, and reuses one set of -LibreOffice reference PDFs. +All implementations use the same classic or issue fixture directory selected by +`-Suite` and `-Format`. Microsoft 365 is always the primary scored reference, +and LibreOffice is a required auxiliary reference displayed in the report. +`-Engine` is retained for compatibility and does not change the primary +reference. Each implementation has an independent entry point and isolated output under -`artifacts/benchmark//`: +`artifacts/-benchmark///`: ```powershell -.\scripts\Run-DotNet-VisualBenchmark.ps1 -Format all -.\scripts\Run-Rust-VisualBenchmark.ps1 -Format all -.\scripts\Run-Java-VisualBenchmark.ps1 -Format all -.\scripts\Run-Go-VisualBenchmark.ps1 -Format all -.\scripts\Run-Python-VisualBenchmark.ps1 -Format all -.\scripts\Run-Node-VisualBenchmark.ps1 -Format all +.\scripts\Run-DotNet-VisualBenchmark.ps1 -Suite classic -Format xlsx +.\scripts\Run-Rust-Benchmark.ps1 -Suite classic -Format xlsx +.\scripts\Run-Java-VisualBenchmark.ps1 -Suite issue -Format docx +.\scripts\Run-Go-VisualBenchmark.ps1 -Suite classic -Format docx +.\scripts\Run-Python-VisualBenchmark.ps1 -Suite issue -Format pptx +.\scripts\Run-Node-VisualBenchmark.ps1 -Suite issue -Format xlsx # Run all six implementations against the same selected cases. -.\scripts\Run-All-Language-VisualBenchmarks.ps1 -Format all +.\scripts\Run-All-Language-VisualBenchmarks.ps1 -Suite issue -Format xlsx -MaxCases 1 ``` -Use `-Filter`, `-MaxCasesPerFormat`, and `-MaxComparePages` for focused runs. -Use `-SkipReference` to reuse shared references and `-SkipBuild` to reuse an -existing language artifact. Each language receives its own resolved manifest, -coverage JSON, candidate PDFs, comparison JSON, Markdown report, images, and -heatmaps. +The default `MinimumScore` is `0.95`. Use `-Filter`, `-MaxCases`, and +`-MaxComparePages` for focused runs. Use `-SkipCandidate` to reuse existing +candidate PDFs and `-SkipReference` to reuse both existing reference sets. +The run fails when a candidate, Microsoft 365 reference, or LibreOffice +reference is missing, or when any score is below the threshold. Each language +receives its own coverage JSON, candidate PDFs, comparison JSON, Markdown +report, images, and heatmaps. ### Prerequisites ```bash -# 1. Python 3.10+ & dependencies -pip install openpyxl pymupdf +# 1. Python 3.10+ & dependencies (pywin32 is required for Microsoft 365 references) +pip install openpyxl pymupdf pywin32 -# 2. LibreOffice (free, used to generate reference PDFs) +# 2. Desktop Microsoft Excel/Word/PowerPoint (primary references) + +# 3. LibreOffice (free, used to generate auxiliary reference PDFs) # Windows: https://www.libreoffice.org/download/ # or: winget install LibreOffice -# 3. .NET 9 SDK +# 4. The toolchain required by the selected MiniPdf implementation ``` ### One-Click Execution