From fbe0bc9b0bd77622fa7c25a249a33bab21b94f71 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 30 Aug 2026 21:24:36 +0200 Subject: [PATCH] fix(svm): decode the text, and close out the plan Closes #772, whose last open defect this is. `read_string_with_encoding` handed back the file's own bytes for every encoding but `UCS2`, so a label in `MS_1252` or an ISO-8859 part emitted invalid utf-8 - and an xml parser refuses that exactly as hard as the unescaped `&` the same defect was raised for, costing the whole image rather than one label. The charset a `FONT` action names now picks a decoder from `internal/encoding`, which already has the single-byte tables. An encoding we have no decoder for is taken for `MS_1252`, what a file of this age most likely means and what vcl falls back to: a label in the wrong characters still draws. The font's own family and style names go through it too - they are bytes as well, and they go straight into `font-family`. The run a text action names is measured in utf-16 code units, because vcl decodes the string before it indexes it. That was a byte `substr` before, which was right only while the string was the raw bytes. Numbers checked against `include/rtl/textenc.h`: `SYMBOL` is 10 and `ASCII_US` 11, 9 being unused. The plan is spent, so `svm/PLAN.md` goes. What had forward value moved rather than went: the corpus frequency table, which is the evidence behind every "occurs nowhere" in the module, and the `FLOATTRANSPARENT` shortcut, both now in `svm/AGENTS.md`. `svm/README.md`'s feature matrix was left behind by it now says what is true, including where the drawing is deliberately approximate. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XQpLmJpJ87qbKoG8B7kbLY --- AGENTS.md | 2 +- CHANGELOG.md | 4 + src/odr/internal/svm/AGENTS.md | 43 +++++++- src/odr/internal/svm/PLAN.md | 149 ---------------------------- src/odr/internal/svm/README.md | 78 ++++++++------- src/odr/internal/svm/svm_format.cpp | 119 ++++++++++++++++++---- src/odr/internal/svm/svm_format.hpp | 31 ++++++ test/src/internal/svm/svm_test.cpp | 64 ++++++++++++ 8 files changed, 283 insertions(+), 207 deletions(-) delete mode 100644 src/odr/internal/svm/PLAN.md diff --git a/AGENTS.md b/AGENTS.md index c8fc4033b..874b3c743 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,7 +72,7 @@ bytes ─▶ magic/open_strategy ─▶ DecodedFile ─▶ Document ─▶ Eleme | `src/odr/internal/markdown/` | Markdown (CommonMark + GFM via md4c), decoded to a text document; see [`markdown/AGENTS.md`](src/odr/internal/markdown/AGENTS.md) + [`markdown/PLAN.md`](src/odr/internal/markdown/PLAN.md). | | `src/odr/internal/xml/` | XML: the pugixml parse and the escaping every xml-writing engine shares (`xml_util`), plus the source view (`xml_file`); see [`xml/AGENTS.md`](src/odr/internal/xml/AGENTS.md). | | `src/odr/internal/svg/` | SVG, detected by reading it as xml; see [`svg/AGENTS.md`](src/odr/internal/svg/AGENTS.md). | -| `src/odr/internal/svm/` | StarView metafile, the vector image odf/ooxml packages carry for charts and OLE objects; translated to svg. See [`svm/AGENTS.md`](src/odr/internal/svm/AGENTS.md) + [`svm/PLAN.md`](src/odr/internal/svm/PLAN.md). | +| `src/odr/internal/svm/` | StarView metafile, the vector image odf/ooxml packages carry for charts and OLE objects; translated to svg. See [`svm/AGENTS.md`](src/odr/internal/svm/AGENTS.md). | | `src/odr/internal/{csv,json,text}/` | Smaller formats. | | `cli/src/` | CLI tools: `translate`, `back_translate`, `meta`, `server`. | | `python/` | Python bindings (`pyodr`, pybind11); see [`python/AGENTS.md`](python/AGENTS.md). | diff --git a/CHANGELOG.md b/CHANGELOG.md index 9059bf230..3e33cf93c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ The release run heads these entries with the version and opens a fresh ## Unreleased +- A StarView metafile's text is decoded by the charset it names. Every + encoding but `UCS2` used to emit the file's own bytes, and the invalid utf-8 + a non-ascii label made of that cost the whole image. + - A curve in a StarView metafile is drawn as a curve. The polygon flags that say which of its points are bezier control points were read and dropped, so every curve came out as a line through them. diff --git a/src/odr/internal/svm/AGENTS.md b/src/odr/internal/svm/AGENTS.md index d2df5c218..9cea6aa12 100644 --- a/src/odr/internal/svm/AGENTS.md +++ b/src/odr/internal/svm/AGENTS.md @@ -2,7 +2,7 @@ Read the root [`AGENTS.md`](../../../../AGENTS.md) first. This file covers what svm does differently, and why. [`README.md`](README.md) has the feature matrix -and the references, [`PLAN.md`](PLAN.md) the roadmap. +and the references. ## What it is @@ -62,9 +62,44 @@ The format is not specified anywhere. The references, best first: unescaped `&` costs the whole image, not one label. Note that `html::escape_text` is the *wrong* escape here — it emits ` `, which no xml parser knows. -- **Escaping is not yet enough.** `read_string_with_encoding` hands back the - file's own bytes for every encoding but `UCS2`, so a latin-1 label emits - invalid utf-8 and the parser refuses the document all the same. +- **Decoding is the other half of escaping.** A byte string is decoded by the + charset the last `FONT` action named, because handing the bytes through + emits invalid utf-8 for anything but ascii and an xml parser refuses that + exactly as hard as an unescaped `&`. An encoding we have no decoder for is + taken for `MS_1252` rather than passed through. + +## What the corpus holds + +Over 1125 metafiles harvested from the `odt`/`ods` fixtures, which is what +every "occurs nowhere" in this module is measured against: + +| action | occurrences | files (of 1125) | +| --- | --- | --- | +| `PUSH` / `POP` | 22317 each | 1124 | +| `TEXTALIGN` | 21534 | 1117 | +| `STRETCHTEXT` | 20335 | 1097 | +| `ISECTRECTCLIPREGION` | 1124 | 1123 | +| `RECT` | 845 | 324 | +| `TEXTARRAY` | 764 | 20 | +| `POLYLINE` | 477 | 13 | +| `POLYPOLYGON` | 247 | 14 | +| `LINE` | 47 | 2 | +| `BMPEXSCALE` | 1 | 1 | + +Two thirds of every action is text, and half of that text is italic — all of +it a formula variable. `ELLIPSE`, `ARC`, `PIE`, `CHORD`, `ROUNDRECT`, `POINT`, +`PIXEL`, `GRADIENT`, `HATCH`, `TRANSPARENT` and `EPS` do not occur at all, nor +does a polygon flag or a complex poly-polygon. The one real bitmap is the +whole data area of `odr-private/svm/Vyplaty.svm` — 1.59 MB of `BMPEXSCALE` in +a 1.63 MB file. + +So a change here is proved by `svm_test.cpp` and by LibreOffice, not by the +reference output: most of what the module now draws, no fixture exercises. + +## `FLOATTRANSPARENT` nests a whole metafile + +The one unimplemented action whose shape is already worked out: translate the +nested metafile into a `` and put the gradient on that group's `mask`. ## Bitmaps do not go through a decoder diff --git a/src/odr/internal/svm/PLAN.md b/src/odr/internal/svm/PLAN.md deleted file mode 100644 index 4d824050e..000000000 --- a/src/odr/internal/svm/PLAN.md +++ /dev/null @@ -1,149 +0,0 @@ -# PLAN — `internal/svm` - -What is missing from the svm → svg translator and the order it gets fixed in. -Umbrella issue: [#772](https://github.com/opendocument-app/OpenDocument.core/issues/772), -sub-parts [#194](https://github.com/opendocument-app/OpenDocument.core/issues/194) -(bitmaps) and [#95](https://github.com/opendocument-app/OpenDocument.core/issues/95) -(font attributes). - -Read [`AGENTS.md`](AGENTS.md) for how the module is built and -[`README.md`](README.md) for the feature matrix and the references. - -## Stages - -Each stage is one pull request, stacked on the one before it. - -1. **Infrastructure.** `svg::SvgWriter`, so markup is written by something that - escapes; a `Logger` through the translator, so an action we drop says so; - inline-bytes tests, so an action can be tested without a fixture. Fixes - #772's defects 1 (escaping, but see stage 3 for the encoding half of it), 9 - (style dispatch) and 10 (silence). -2. **Fixes to what we already emit.** The graphics state stack (`PUSH`/`POP`), - the fill that killed the stroke, the poly-polygon fill rule, the font size - in the transform, `LineInfo`. #772 defects 2, 3, 5, 8. -3. **Text.** `TEXTALIGN`, the `TEXTARRAY` dx array, the `STRETCHTEXT` width, - the run a text action names, and the #95 font attributes (bold, italic, - underline, strikeout, rotation). `TEXTRECT` is still open - it occurs - nowhere in the corpus - and so is decoding a non-`UCS2` string instead of - passing its bytes through: until then a latin-1 label emits invalid utf-8, - which costs the image exactly as an unescaped `&` did. -4. **Clipping** - done but for `MOVECLIPREGION`, which occurs nowhere and - would have to move path data that is already written out. -5. **Bitmaps** (#194) - done for the `BMP` and `BMPEX` families; `MASK`, - which stencils one colour through a bitmap, and `ZCOMPRESS`, which needs - inflating first, are still open. See the shortcut below. -6. **Primitives** - done. -7. **Fills and transparency** - `GRADIENT`, `GRADIENTEX`, `HATCH` and - `TRANSPARENT` are done. `WALLPAPER` and `FLOATTRANSPARENT` are not: the - first has a format of its own, the second nests a whole metafile. -8. **The map mode** (#772 defect 6) - done. -9. **Stretch.** Bézier flags (#772 defect 4) are done. Left: `WALLPAPER`, - `FLOATTRANSPARENT` and `EPS`, which all nest something of their own; the - `MASK` family; `MOVECLIPREGION`; `ZCOMPRESS`ed bitmaps; and version-1 - (pre-`VCLMTF`) files via `SvmConverter.cxx`. None of them occurs in the - corpus. - -The order follows what files actually contain, not the action list. Over 1125 -metafiles harvested from the `odt`/`ods` fixtures: - -| action | occurrences | files (of 1125) | -| --- | --- | --- | -| `PUSH` / `POP` | 22317 each | 1124 | -| `TEXTALIGN` | 21534 | 1117 | -| `STRETCHTEXT` | 20335 | 1097 | -| `ISECTRECTCLIPREGION` | 1124 | 1123 | -| `RECT` | 845 | 324 | -| `TEXTARRAY` | 764 | 20 | -| `POLYLINE` | 477 | 13 | -| `POLYPOLYGON` | 247 | 14 | -| `LINE` | 47 | 2 | -| `BMPEXSCALE` | 1 | 1 | - -Half the text in the corpus is italic, all of it a formula variable, which is -what puts the #95 font attributes in stage 3 rather than later. - -`ELLIPSE`, `ARC`, `PIE`, `CHORD`, `ROUNDRECT`, `POINT`, `PIXEL`, `GRADIENT`, -`HATCH`, `TRANSPARENT` and `EPS` do not occur at all, which is why they come -after clipping rather than before it. The one bitmap is the whole data area of -`odr-private/svm/Vyplaty.svm` — 1.59 MB of `BMPEXSCALE` in a 1.63 MB file, and -the reason that chart renders as an empty frame today. - -## Clipping is nested groups - -An svg `clip-path` names one shape, and `ISECT…` intersects. Two clips -therefore become two groups, one inside the other, and the state's clip is a -*stack* of shapes rather than one: `ensure_clip` keeps the groups that the -next drawing action still wants, closes the ones it does not, and opens what -is missing. A `POP` that restores the clip is then nothing special - the next -action closes what it has to. - -A region streams as a band list, which is a union of rectangles, and from -version 2 also as the poly-polygon those were rasterised from. The polygons -are the better outline where they are there. Both go into one `` - -disjoint bands make union and even-odd the same thing, and a poly-polygon -wants even-odd anyway, so `clip-rule="evenodd"` covers both. - -vcl does *not* re-scale a clip when the map mode changes (`SetMapMode`: -"clip regions are not re-scaled"), so the shape is transformed once, when the -action sets it. - -## Text is where the corpus lives - -Two thirds of every action in the corpus is text, and three things about it -are worth writing down: - -- **`TextAlign` is vertical only.** It says whether the draw point is the top, - the baseline or the bottom of the run; vcl has no horizontal text alignment, - a run always starts at the point. Its default is `ALIGN_TOP`, which is not - what an svg `` does, so it has to be written out. `svgwriter.cxx` - shifts the point by the font's ascent because it has the metrics; we name - `dominant-baseline` and let the browser do it. -- **A text action names a run**, `(index, length)`, of the string it carries - - and the string is the whole paragraph. Drawing the string rather than the run - overprints the sentence at every run's position. -- **The dx array and the stretch width are the file's own measurements**, and - they are what keeps a formula together when the viewer's font is not the - author's. They map onto an `x` list and `textLength` respectively. - -## Shortcuts worth taking - -- **Bitmaps are `.bmp` files already** — this one is taken. `SvmReader` reads - them with `ReadDIB(…, bFileHeader=true)`, i.e. the action body holds a dib - *with* its `BITMAPFILEHEADER`, so the bytes are a `.bmp` file and the browser - can read them as one. Only the length has to be worked out, and `bfSize` is - no help: it is written from the *uncompressed* size, so for a compressed dib - it lies. The header's own `biSizeImage` (or width, height and bit count) is - the answer. - - The bytes then go out as a png rather than a bmp wherever the pixels can be - copied row by row — uncompressed, 1/4/8/24/32 bits, palette expanded. That is - not decoding so much as re-packing, and it is worth it: the one real bitmap - in the corpus is 1.59 MB as a bmp and 33 KB as a png, so the page it sits on - goes from 2.9 MB to 158 KB. A compressed dib (RLE, bit fields) still goes out - as the bmp it is; browsers read those. - - What is left: `ZCOMPRESS`, a LibreOffice-only compression whose zlib stream - would have to be inflated (miniz is already a dependency) before any of the - above, and the `MASK` family, which stencils one colour through a bitmap. -- **Béziers were cheap once the flags were read** - this one is taken. A - polygon that carries curves is written *twice*, the second time with one - `PolyFlags` per point, and two control points between two corners are an - svg `C`. -- **Gradients, hatches and dashes are declarative in SVG** — - ``, ``, ``, `stroke-dasharray`. No - rasterising, no tiling by hand. -- **`FLOATTRANSPARENT` nests a whole metafile**: translate it into a `` and - put the gradient on that group's `mask`. - -## Testing - -`svm_test.cpp` builds its input as bytes inline (`SvmBuilder`), so an action -gets a test without a fixture. The fixtures -(`odr-public/svm/{chart-1,table-1}.svm`, `odr-private/svm/{test,Vyplaty}.svm`) -stay the end-to-end check, and LibreOffice is the oracle for what the drawing -should look like: - -```sh -/Applications/LibreOffice.app/Contents/MacOS/soffice --headless \ - --convert-to svg --outdir /tmp test/data/input/odr-public/svm/chart-1.svm -``` diff --git a/src/odr/internal/svm/README.md b/src/odr/internal/svm/README.md index 604a4cff3..8ac256db6 100644 --- a/src/odr/internal/svm/README.md +++ b/src/odr/internal/svm/README.md @@ -1,53 +1,61 @@ # SVM implementation StarView Metafile → SVG. See [`AGENTS.md`](AGENTS.md) for how the module is -built and [`PLAN.md`](PLAN.md) for the order the gaps below get closed in. +built, what it does differently, and what the corpus actually contains. ## Features +38 of the 54 action types we name are drawn. + - [x] shapes - - [x] rectangle - - [x] polyline, polygon, poly-polygon - - [ ] fill rule of a poly-polygon (holes are painted over) - - [ ] pixel, point, line, rounded rectangle, ellipse, arc, pie, chord - - [ ] bézier segments (the polygon flags are not read) + - [x] rectangle, rounded rectangle + - [x] polyline, polygon, poly-polygon, with the fill rule that cuts its holes + - [x] pixel, point, line, ellipse, arc, pie, chord + - [x] bézier segments - [x] colour - [x] line, fill, text - - [ ] `LineInfo` (width, dash, join, cap) + - [x] `LineInfo` (width, dash, join) - [ ] text fill, overline (read into the state, never drawn) -- [ ] font - - [x] size (in map-mode units, which the transform does not apply) - - [ ] italic, bold - - [ ] alignment - - [ ] underline, strike through - - [x] colour - - [x] family +- [x] font + - [x] family, size, colour + - [x] italic, bold, underline, strike through, rotation + - [x] alignment — `TextAlign` is vertical only, a run always starts at the + point - [x] text - - [x] `TEXT`, `TEXTARRAY`, `STRETCHTEXT` as plain text at a point - - [ ] the `TEXTARRAY` dx array, `STRETCHTEXT` width, `TEXTRECT` - - [ ] non-`UCS2` encodings (the bytes go out undecoded, see below) -- [ ] transform (e.g. flip, rotate) - - [x] map mode origin and scale - - [ ] map mode unit -- [ ] images (`BMP`, `BMPEX`, `MASK` and their scale/part variants) -- [ ] gradient, hatch, wallpaper -- [ ] clipping regions -- [ ] transparency (`TRANSPARENT`, `FLOATTRANSPARENT`) -- [ ] graphics state stack (`PUSH`/`POP`) + - [x] `TEXT`, `TEXTARRAY`, `STRETCHTEXT`, and the run each of them names + - [x] the `TEXTARRAY` dx array and the `STRETCHTEXT` width + - [x] non-`UCS2` encodings, decoded to utf-8 + - [ ] `TEXTRECT`, `TEXTLINE` — the reader has both, nothing draws them +- [x] map mode + - [x] origin, scale, unit + - [x] the relative map mode, which composes with the one before it +- [x] images + - [x] `BMP`, `BMPEX` and their scale and part variants, transparency mask + included + - [ ] the `MASK` family, which stencils one colour through a bitmap + - [ ] `ZCOMPRESS`ed dibs, whose zlib stream would have to be inflated first +- [x] gradient, hatch, transparency + - [x] `GRADIENT`, `GRADIENTEX`, `HATCH`, `TRANSPARENT` + - [ ] `WALLPAPER`, which has a format of its own + - [ ] `FLOATTRANSPARENT`, which nests a whole metafile +- [x] clipping + - [x] `CLIPREGION`, `ISECTRECTCLIPREGION`, `ISECTREGIONCLIPREGION` + - [ ] `MOVECLIPREGION`, which would have to move path data already written +- [x] graphics state stack (`PUSH`/`POP`), restoring only what the push named - [ ] `EPS` substitute metafile -- [ ] version 1 (pre-`VCLMTF`) files -- [x] output is escaped, so a `&` in a label cannot cost the whole image +- [ ] version 1 (pre-`VCLMTF`) files, via `SvmConverter.cxx` +- [x] output is escaped and decoded, so no label can cost the whole image - [x] every action we skip is logged by name -Anything not implemented is skipped by the action's own length, so the actions -after it still read. +Anything unimplemented is skipped by the action's own length, so the actions +after it still read. None of it occurs in the corpus. -### Known defect +### Where the drawing is approximate -Text in a non-`UCS2` encoding is passed through as the bytes the file holds -(`read_ascii_string`). A latin-1 label therefore emits invalid utf-8, and an -xml parser refuses that exactly as hard as an unescaped `&`. Escaping alone -does not make every label safe. +- A `SQUARE` or `RECT` gradient shrinks a rectangle rather than an ellipse, + which svg has no gradient for; it comes out as the ellipse closest to it. +- A triple hatch's third line set runs corner to corner of the pattern tile, + so those lines sit `distance / √2` apart rather than `distance`. ## References @@ -61,6 +69,8 @@ does not make every label safe. - [`svgwriter.cxx`](https://github.com/LibreOffice/core/blob/master/filter/source/svg/svgwriter.cxx) — LibreOffice's own metafile → svg export, i.e. our problem already solved. The reference for mapping decisions. +- [`textenc.h`](https://github.com/LibreOffice/core/blob/master/include/rtl/textenc.h) + — the `rtl_TextEncoding` numbers a font's charset is one of. - [`SPEC`](https://github.com/ONLYOFFICE/core/blob/master/DesktopEditor/raster/Metafile/StarView/SPEC) — ONLYOFFICE's prose write-up, modelled on [MS-WMF]. Cheap to read, but incomplete: several FIXMEs, `Color` and the polygon flags unfinished. diff --git a/src/odr/internal/svm/svm_format.cpp b/src/odr/internal/svm/svm_format.cpp index 89bc1e64b..11bf12c0e 100644 --- a/src/odr/internal/svm/svm_format.cpp +++ b/src/odr/internal/svm/svm_format.cpp @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -155,20 +156,95 @@ String select_run(const String &text, const std::uint16_t offset, return text.substr(offset, length); } -/// @ref select_run in the units the run is measured in - utf-16 code units for -/// `UCS2`, bytes otherwise. @p text has already been decoded to utf-8, where a -/// `UCS2` offset addresses nothing and a `substr` splits a character. -std::string select_run_with_encoding(const std::string &text, - const svm::TextEncoding encoding, - const std::uint16_t offset, - const std::uint16_t length) { - if (encoding != svm::RTL_TEXTENCODING_UCS2) { - return select_run(text, offset, length); - } +/// @ref select_run in the units the offsets are measured in: vcl decodes the +/// string before it indexes it, so they count utf-16 code units whatever the +/// file's encoding is - a utf-8 `substr` would split a character. +std::string select_run_utf16(const std::string &text, + const std::uint16_t offset, + const std::uint16_t length) { return util::string::u16string_to_string( select_run(util::string::string_to_u16string(text), offset, length)); } +/// The decoder for an `rtl_TextEncoding`, `unknown` where we have none. +odr::TextEncoding get_decoder(const svm::TextEncoding encoding) { + switch (encoding) { + // every byte below 0x80 is the character it is; a stray one above reads as + // latin-1 rather than as the invalid utf-8 it would otherwise emit + case svm::RTL_TEXTENCODING_ASCII_US: + case svm::RTL_TEXTENCODING_ISO_8859_1: + return odr::TextEncoding::iso_8859_1; + case svm::RTL_TEXTENCODING_MS_1252: + return odr::TextEncoding::windows_1252; + case svm::RTL_TEXTENCODING_APPLE_ROMAN: + return odr::TextEncoding::macintosh; + case svm::RTL_TEXTENCODING_ISO_8859_2: + return odr::TextEncoding::iso_8859_2; + case svm::RTL_TEXTENCODING_ISO_8859_3: + return odr::TextEncoding::iso_8859_3; + case svm::RTL_TEXTENCODING_ISO_8859_4: + return odr::TextEncoding::iso_8859_4; + case svm::RTL_TEXTENCODING_ISO_8859_5: + return odr::TextEncoding::iso_8859_5; + case svm::RTL_TEXTENCODING_ISO_8859_6: + return odr::TextEncoding::iso_8859_6; + case svm::RTL_TEXTENCODING_ISO_8859_7: + return odr::TextEncoding::iso_8859_7; + case svm::RTL_TEXTENCODING_ISO_8859_8: + return odr::TextEncoding::iso_8859_8; + case svm::RTL_TEXTENCODING_ISO_8859_10: + return odr::TextEncoding::iso_8859_10; + case svm::RTL_TEXTENCODING_ISO_8859_13: + return odr::TextEncoding::iso_8859_13; + case svm::RTL_TEXTENCODING_ISO_8859_14: + return odr::TextEncoding::iso_8859_14; + case svm::RTL_TEXTENCODING_ISO_8859_15: + return odr::TextEncoding::iso_8859_15; + case svm::RTL_TEXTENCODING_IBM_866: + return odr::TextEncoding::ibm866; + case svm::RTL_TEXTENCODING_MS_874: + return odr::TextEncoding::windows_874; + case svm::RTL_TEXTENCODING_MS_1250: + return odr::TextEncoding::windows_1250; + case svm::RTL_TEXTENCODING_MS_1251: + return odr::TextEncoding::windows_1251; + case svm::RTL_TEXTENCODING_MS_1253: + return odr::TextEncoding::windows_1253; + case svm::RTL_TEXTENCODING_MS_1254: + return odr::TextEncoding::windows_1254; + case svm::RTL_TEXTENCODING_MS_1255: + return odr::TextEncoding::windows_1255; + case svm::RTL_TEXTENCODING_MS_1256: + return odr::TextEncoding::windows_1256; + case svm::RTL_TEXTENCODING_MS_1257: + return odr::TextEncoding::windows_1257; + case svm::RTL_TEXTENCODING_MS_1258: + return odr::TextEncoding::windows_1258; + case svm::RTL_TEXTENCODING_APPLE_CYRILLIC: + return odr::TextEncoding::x_mac_cyrillic; + case svm::RTL_TEXTENCODING_KOI8_R: + return odr::TextEncoding::koi8_r; + case svm::RTL_TEXTENCODING_KOI8_U: + return odr::TextEncoding::koi8_u; + case svm::RTL_TEXTENCODING_UTF8: + return odr::TextEncoding::utf8; + default: + return odr::TextEncoding::unknown; + } +} + +/// @p bytes as utf-8. An encoding we have no decoder for is taken for +/// `MS_1252`, which is what a file of this age most likely means and what +/// vcl falls back to on Windows: a label in the wrong characters still draws, +/// where passing the bytes through emits invalid utf-8 and an xml parser +/// refuses that exactly as hard as an unescaped `&`. +std::string decode(const std::string &bytes, const svm::TextEncoding encoding) { + const odr::TextEncoding decoder = get_decoder(encoding); + return encoding::to_utf8(bytes, decoder == odr::TextEncoding::unknown + ? odr::TextEncoding::windows_1252 + : decoder); +} + std::u16string read_u16string(std::istream &in, const std::uint32_t length) { const std::string bytes = read_bytes(in, static_cast(length) * 2); @@ -218,7 +294,7 @@ std::string svm::read_string_with_encoding(std::istream &in, if (encoding == RTL_TEXTENCODING_UCS2) { return read_uint32_prefixed_utf16_string(in); } - return read_uint16_prefixed_ascii_string(in); + return decode(read_uint16_prefixed_ascii_string(in), encoding); } std::string_view svm::action_type_name(const std::uint16_t type) { @@ -499,10 +575,18 @@ svm::Font svm::read_font(std::istream &in) { Font result; result.vl = read_version_length(in); - result.family_name = read_uint16_prefixed_ascii_string(in); - result.style_name = read_uint16_prefixed_ascii_string(in); + const std::string family_name = read_uint16_prefixed_ascii_string(in); + const std::string style_name = read_uint16_prefixed_ascii_string(in); result.size = read_int_pair(in); read_primitive(in, result.charset); + + // the names are bytes, and the only charset the action carries is the one + // it reads next; vcl decodes them with the stream's instead, which a + // metafile never sets + result.family_name = + decode(family_name, static_cast(result.charset)); + result.style_name = + decode(style_name, static_cast(result.charset)); read_primitive(in, result.family); read_primitive(in, result.pitch); read_primitive(in, result.weight); @@ -654,8 +738,7 @@ svm::TextAction svm::read_text_action(std::istream &in, const VersionLength &vl, result.text = util::string::u16string_to_string(select_run( read_uint16_prefixed_u16string(in), result.offset, result.length)); } else { - result.text = select_run_with_encoding(result.text, encoding, result.offset, - result.length); + result.text = select_run_utf16(result.text, result.offset, result.length); } return result; @@ -684,8 +767,7 @@ svm::TextArrayAction svm::read_text_array_action(std::istream &in, result.text = util::string::u16string_to_string(select_run( read_uint16_prefixed_u16string(in), result.offset, result.length)); } else { - result.text = select_run_with_encoding(result.text, encoding, result.offset, - result.length); + result.text = select_run_utf16(result.text, result.offset, result.length); } return result; @@ -706,8 +788,7 @@ svm::read_stretch_text_action(std::istream &in, const VersionLength &vl, result.text = util::string::u16string_to_string(select_run( read_uint16_prefixed_u16string(in), result.offset, result.length)); } else { - result.text = select_run_with_encoding(result.text, encoding, result.offset, - result.length); + result.text = select_run_utf16(result.text, result.offset, result.length); } return result; diff --git a/src/odr/internal/svm/svm_format.hpp b/src/odr/internal/svm/svm_format.hpp index ae845a1a9..cb8b6b813 100644 --- a/src/odr/internal/svm/svm_format.hpp +++ b/src/odr/internal/svm/svm_format.hpp @@ -13,9 +13,40 @@ // https://github.com/LibreOffice/core/blob/master/include/vcl/metaact.hxx namespace odr::internal::svm { +/// `rtl_TextEncoding`, what a byte string in the file is written in. Only the +/// ones we have a decoder for are named; `read_string_with_encoding` takes +/// anything else for `MS_1252`. enum TextEncoding { RTL_TEXTENCODING_DONTKNOW = 0, + RTL_TEXTENCODING_MS_1252 = 1, + RTL_TEXTENCODING_APPLE_ROMAN = 2, RTL_TEXTENCODING_ASCII_US = 11, + RTL_TEXTENCODING_ISO_8859_1 = 12, + RTL_TEXTENCODING_ISO_8859_2 = 13, + RTL_TEXTENCODING_ISO_8859_3 = 14, + RTL_TEXTENCODING_ISO_8859_4 = 15, + RTL_TEXTENCODING_ISO_8859_5 = 16, + RTL_TEXTENCODING_ISO_8859_6 = 17, + RTL_TEXTENCODING_ISO_8859_7 = 18, + RTL_TEXTENCODING_ISO_8859_8 = 19, + RTL_TEXTENCODING_ISO_8859_14 = 21, + RTL_TEXTENCODING_ISO_8859_15 = 22, + RTL_TEXTENCODING_IBM_866 = 30, + RTL_TEXTENCODING_MS_874 = 32, + RTL_TEXTENCODING_MS_1250 = 33, + RTL_TEXTENCODING_MS_1251 = 34, + RTL_TEXTENCODING_MS_1253 = 35, + RTL_TEXTENCODING_MS_1254 = 36, + RTL_TEXTENCODING_MS_1255 = 37, + RTL_TEXTENCODING_MS_1256 = 38, + RTL_TEXTENCODING_MS_1257 = 39, + RTL_TEXTENCODING_MS_1258 = 40, + RTL_TEXTENCODING_APPLE_CYRILLIC = 44, + RTL_TEXTENCODING_KOI8_R = 74, + RTL_TEXTENCODING_UTF8 = 76, + RTL_TEXTENCODING_ISO_8859_10 = 77, + RTL_TEXTENCODING_ISO_8859_13 = 78, + RTL_TEXTENCODING_KOI8_U = 88, RTL_TEXTENCODING_UCS4 = 0xFFFE, RTL_TEXTENCODING_UCS2 = 0xFFFF, }; diff --git a/test/src/internal/svm/svm_test.cpp b/test/src/internal/svm/svm_test.cpp index da4086c59..2a91e303c 100644 --- a/test/src/internal/svm/svm_test.cpp +++ b/test/src/internal/svm/svm_test.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -306,6 +307,29 @@ std::size_t count_of(const std::string &haystack, const std::string &needle) { return result; } +/// Hand-rolled because pugixml accepts invalid utf-8 and a browser does not. +bool is_utf8(const std::string_view text) { + for (std::size_t i = 0; i < text.size();) { + const auto lead = static_cast(text[i]); + const std::size_t length = lead < 0x80 ? 1 + : lead < 0xc2 ? 0 + : lead < 0xe0 ? 2 + : lead < 0xf0 ? 3 + : lead < 0xf5 ? 4 + : 0; + if (length == 0 || i + length > text.size()) { + return false; + } + for (std::size_t n = 1; n < length; ++n) { + if ((static_cast(text[i + n]) & 0xc0) != 0x80) { + return false; + } + } + i += length; + } + return true; +} + std::string translate(const std::string &data) { const svm::SvmFile file(std::make_shared(data)); std::ostringstream out; @@ -1090,6 +1114,46 @@ TEST(SvmToSvg, a_pop_restores_the_map_mode) { /// Two control points between two corners are a bezier segment, which is a /// `C` - the same path LibreOffice writes for the same polygon. +/// 0x92 is a curly apostrophe in `MS_1252` and a control character in latin-1. +TEST(SvmToSvg, text_is_decoded_by_the_font_charset) { + const auto translate_in = [](const std::uint16_t charset) { + return translate(SvmBuilder() + .font("Arial", 10, 0, 0, 0, 0, 0, charset) + .text(0, 0, "it\x92s") + .file()); + }; + + EXPECT_NE(std::string::npos, + translate_in(svm::RTL_TEXTENCODING_MS_1252).find("it\u2019s")); + EXPECT_NE(std::string::npos, + translate_in(svm::RTL_TEXTENCODING_ISO_8859_1).find("it\u0092s")); +} + +/// An encoding we have no decoder for is taken for `MS_1252`. +TEST(SvmToSvg, an_unknown_encoding_still_decodes) { + const std::string svg = translate(SvmBuilder() + .font("Arial", 10, 0, 0, 0, 0, 0, 0x63) + .text(0, 0, "\xe4\x92") + .file()); + + EXPECT_NE(std::string::npos, svg.find("\u00e4\u2019")); + EXPECT_TRUE(is_utf8(svg)); +} + +/// vcl indexes the decoded string, so an offset counts utf-16 code units. +TEST(SvmToSvg, a_run_offset_counts_characters_not_utf8_bytes) { + const std::string svg = translate( + SvmBuilder() + .font("Arial", 10, 0, 0, 0, 0, 0, svm::RTL_TEXTENCODING_MS_1252) + .text(0, 0, + "\x92\x92" + "ab", + 2, 2) + .file()); + + EXPECT_NE(std::string::npos, svg.find(">ab<")); +} + TEST(SvmToSvg, a_polyline_with_flags_curves) { const std::vector> points = { {100, 700}, {300, 100}, {700, 100}, {900, 700}};