diff --git a/AGENTS.md b/AGENTS.md index e98ffcb4..b9c61a9c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,7 +61,7 @@ bytes ─▶ magic/open_strategy ─▶ DecodedFile ─▶ Document ─▶ Eleme | `src/odr/internal/odf/` | OpenDocument (odt/ods/odp/odg); see [`odf/AGENTS.md`](src/odr/internal/odf/AGENTS.md). | | `src/odr/internal/ooxml/` | OOXML (docx/pptx/xlsx); see [`ooxml/AGENTS.md`](src/odr/internal/ooxml/AGENTS.md) + per-format docs. | | `src/odr/internal/oldms/` | **Legacy MS binary** (.doc/.ppt/.xls). | -| `src/odr/internal/iwork/` | Apple iWork (`.pages` and `.key` today); see [`iwork/AGENTS.md`](src/odr/internal/iwork/AGENTS.md) + [`iwork/PLAN.md`](src/odr/internal/iwork/PLAN.md). | +| `src/odr/internal/iwork/` | Apple iWork (`.pages`, `.key`, `.numbers`); see [`iwork/AGENTS.md`](src/odr/internal/iwork/AGENTS.md) + [`iwork/PLAN.md`](src/odr/internal/iwork/PLAN.md). | | `src/odr/internal/pdf/` | PDF (own parser). | | `src/odr/internal/rtf/` | RTF, read as a text document; see [`rtf/AGENTS.md`](src/odr/internal/rtf/AGENTS.md) + [`rtf/PLAN.md`](src/odr/internal/rtf/PLAN.md). | | `src/odr/internal/markdown/` | Markdown (CommonMark + GFM via md4c), decoded to a text document; see [`markdown/AGENTS.md`](src/odr/internal/markdown/AGENTS.md) + [`markdown/PLAN.md`](src/odr/internal/markdown/PLAN.md). | @@ -228,11 +228,12 @@ Dispatch `release.yml` against main, publish the draft that appears — ## Apple iWork (`iwork`) -`.pages` opens as a text document and `.key` as a presentation; `.numbers` is -named but not decoded. There is no spec — the module cites fixtures instead, -keeps its own Snappy and protobuf readers, and fails soft on archive types it -has not mapped. Archive type ids are namespaced per app, so a `.key` and a -`.numbers` share them and the component list is what tells the two apart. Read [`iwork/AGENTS.md`](src/odr/internal/iwork/AGENTS.md) +`.pages` opens as a text document, `.key` as a presentation and `.numbers` as a +spreadsheet. There is no spec — the module cites fixtures instead, keeps its own +Snappy and protobuf readers, and fails soft on archive types it has not mapped. +Archive type ids are namespaced per app, so a `.key` and a `.numbers` share them +and the component list is what tells the two apart. Tables everywhere go +through one tile reader, and a cell value stays the decimal the file stores. Read [`iwork/AGENTS.md`](src/odr/internal/iwork/AGENTS.md) before touching it, and [`iwork/PLAN.md`](src/odr/internal/iwork/PLAN.md) for what comes next. diff --git a/CHANGELOG.md b/CHANGELOG.md index 95b40b08..1160ea45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,12 +24,16 @@ The release run heads these entries with the version and opens a fresh `office:binary-data` image now decodes in a package too. - `Document::as_filesystem` answers with an empty filesystem for a document that is one file rather than a package. -- A `.pages` file opens as a text document and renders its body text instead of - the zip it is made of; styles, images and tables are not read yet. `.numbers` - is named but has no decoder. +- A `.pages` file opens as a text document and renders its body text and the + tables it anchors, instead of the zip it is made of; styles and images are not + read yet. - A `.key` file opens as a presentation and renders the text of each slide in boxes where the file positions them, instead of the zip it is made of; slide masters, styles, images, tables and presenter notes are not read yet. +- A `.numbers` file opens as a spreadsheet instead of the zip it is made of, + with one sheet per Numbers table named ` – `. Cells show the + values the app last computed; number formats, merged ranges, charts and + formulas are not read yet. - An rtf opens and renders as a text document instead of throwing `UnknownFileType`. Its text, encoding, paragraphs and tabs are read; formatting, tables and pictures are not yet. diff --git a/CMakeLists.txt b/CMakeLists.txt index f83ef7b5..79a73865 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -153,13 +153,13 @@ set(ODR_SOURCE_FILES "src/odr/internal/html/xml_file.cpp" "src/odr/internal/iwork/iwork_archive.cpp" - "src/odr/internal/iwork/iwork_budget.cpp" "src/odr/internal/iwork/iwork_document.cpp" "src/odr/internal/iwork/iwork_element_registry.cpp" "src/odr/internal/iwork/iwork_file.cpp" "src/odr/internal/iwork/iwork_parser.cpp" "src/odr/internal/iwork/iwork_protobuf.cpp" "src/odr/internal/iwork/iwork_snappy.cpp" + "src/odr/internal/iwork/iwork_table.cpp" "src/odr/internal/iwork/iwork_text.cpp" "src/odr/internal/json/json_file.cpp" diff --git a/README.md b/README.md index a7ad8dff..091e0b95 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,12 @@ C++ library to visualize files, especially documents, in HTML. - [csv](https://github.com/opendocument-app/OpenDocument.core/issues/107) - [doc](https://github.com/opendocument-app/OpenDocument.core/issues/104), [ppt](https://github.com/opendocument-app/OpenDocument.core/issues/106), [xls](https://github.com/opendocument-app/OpenDocument.core/issues/105) - [pdf](https://github.com/opendocument-app/OpenDocument.core/issues/108) -- pages (Apple Pages — body text only; styles, page geometry, images and - tables are not read yet) +- pages (Apple Pages — body text and the tables it anchors; styles, page + geometry and images are not read yet) - key (Apple Keynote — the text of each slide, in boxes where the file puts them; styles, masters, images, tables and presenter notes are not read yet) +- numbers (Apple Numbers — one sheet per table, showing the values the app last + computed; number formats, merged cells, charts and formulas are not read yet) - rtf (body text only; character and paragraph formatting, tables and pictures are not read yet) - txt @@ -40,7 +42,6 @@ opening one throws: - wpd (WordPerfect) - xlsb (Excel binary workbook — an OOXML package whose workbook parts are binary rather than spreadsheetml) -- numbers (Apple Numbers) ## Asking what is supported diff --git a/src/odr/internal/file_type_table.cpp b/src/odr/internal/file_type_table.cpp index 8a421c64..85986fa2 100644 --- a/src/odr/internal/file_type_table.cpp +++ b/src/odr/internal/file_type_table.cpp @@ -790,15 +790,16 @@ constexpr std::array table{ .open = true, .translate_html = true, .color_scheme = true}}, - // Classified so a caller can name it and hand its MIME types to a file - // picker; no engine reads one yet. Row{FileType::iwork_numbers, "numbers"sv, numbers_extensions, numbers_mimetypes, FileCategory::document, DocumentType::spreadsheet, - {}}, + {.detect_by_content = true, + .open = true, + .translate_html = true, + .color_scheme = true}}, Row{FileType::iwork_keynote, "key"sv, keynote_extensions, diff --git a/src/odr/internal/iwork/AGENTS.md b/src/odr/internal/iwork/AGENTS.md index fecb7b3b..0a417fd7 100644 --- a/src/odr/internal/iwork/AGENTS.md +++ b/src/odr/internal/iwork/AGENTS.md @@ -5,10 +5,12 @@ Read the root [`AGENTS.md`](../../../../AGENTS.md) first, then This file is what the landed stages decided, and why. Landed: **stage 1** (detection and the container), **stage 2** (Pages body -text) and **stage 5** (Keynote slides). A `.pages` opens as a text document and -renders its paragraphs; a `.key` opens as a presentation and renders each -slide's text boxes as frames where the geometry puts them. Everything else in -`PLAN.md` is still ahead. +text), **stage 5** (Keynote slides), **stage 6** (the tile reader) and +**stage 7** (Numbers). A `.pages` opens as a text document and renders its +paragraphs and the tables its text anchors; a `.key` opens as a presentation +and renders each slide's text boxes as frames where the geometry puts them; a +`.numbers` opens as a spreadsheet, one odr sheet per Numbers table. +Everything else in `PLAN.md` is still ahead. ## The files split by framework, not by app @@ -17,13 +19,27 @@ serve all three apps. That is `odf/`'s shape rather than `ooxml/`'s, for a reason the two analogies do not quite carry: Apple factored the format by **framework**, so `TSWP` text, `TSD` drawables, `TST` tables and `TSS` styles mean the same thing in a `.pages`, a `.key` and a `.numbers`, and only the -spine above them is per-app. Parsing splits along those seams — `iwork_text.cpp` -is the `TSWP` layer, `iwork_parser.cpp` the three spines that call it — never -along app lines. `PLAN.md` carries the argument and the files each stage adds. - -`Budget` has its own header because it is a parse-wide meter, not a text one: -the spines spend for slides and frames as the text layer spends for paragraphs -and runs. +spine above them is per-app. Parsing splits along those seams — `iwork_table.cpp` +reads `TST` tiles into a `TableModel`, `iwork_text.cpp` turns a `TSWP` storage +into elements, and `iwork_parser.cpp` holds the three spines that call them — +never along app lines. `PLAN.md` carries the argument and the files each stage +adds. + +The line between the two readers is the registry: `iwork_table.cpp` touches +none, so a tile is decoded as data and tested as data, and everything that +builds elements — a table's rows, a cell's own storage, a drawable a paragraph +anchors — lives in `iwork_text.cpp`, which is where the recursion between them +closes. + +`Budget` is a header of its own because it is a parse-wide meter, not a text +one: the spines spend for slides, sheets and frames as the text layer spends +for paragraphs, runs and cells. **A table's extent is the file's word**, so +`parse_table` spends per row and per cell — a `TST.TableModelArchive` declaring +a grid of millions is a few bytes on the wire. + +`reference_identifier` and `reference_identifiers` live in `iwork_archive.cpp` +with the rest of `TSP`: all three readers resolve references, and a copy per +reader is how that drifts. ## There is no spec, so a fixture is the citation @@ -50,6 +66,15 @@ throw: framing that overruns the file, a Snappy block that does not fill its declared length, a varint that does not terminate, an identifier the package does not hold, and text that is not UTF-8. +**A tile is spent as it is decoded, not once it is a model.** `read_table` +takes the `Budget` and spends per cell and per byte of cell text as `read_tile` +produces them, because a tile list may name one tile any number of times and +`Package::object` hands every repeat back from its cache — a model built first +and metered afterwards is a gigabyte from a package of a few kilobytes. What a +repeat costs a second time is nothing: `TableCache` reads each table once per +parse, so a drawable list naming one table a million times spends a million +elements rather than re-decoding a million tiles. + A declared length is the file's word, so nothing is allocated or written against one before it is known to fit: `snappy_decompress_block` caps its reservation at what the compressed bytes could expand to and checks every tag @@ -60,10 +85,12 @@ identifier and `Package::object` memoises, a `TSP.Reference` list that names one object a million times costs four bytes a repeat on the wire and a fresh subtree — elements plus a copy of the storage's text — every time it is resolved. The repeats are siblings rather than ancestors, so no cycle check -sees them. `parse_pages_tree` and `parse_keynote_tree` therefore spend every -element and every byte of text against a `Budget` set far above what an -authored document reaches, which keeps a package built to expand a thrown -`std::runtime_error` rather than an allocation the process dies on. +sees them. Every tree the module parses therefore spends each element and each +byte of text against a `Budget` set far above what an authored document +reaches, which keeps a package built to expand a thrown `std::runtime_error` +rather than an allocation the process dies on. A nested storage — a cell +holding a table holding a cell — is bounded a second way, by `Context::deeper`, +because depth costs stack rather than budget. ## No new dependencies @@ -128,9 +155,9 @@ all six fixtures, and pinned by the negative the rule rests on — That costs `Index/Metadata.iwa`, which detection otherwise avoids reading, so it is only reached once the root archive has already come back as type 1. -`iwork_numbers` still has a `file_type_table.cpp` row with no capabilities so a -caller can name it and hand a file picker its MIME types; nothing decodes one -yet. +A package whose root archive is type 1 and that holds no `Slide` component is +therefore a `.numbers` — by elimination, which is as strong a claim as the +format allows. ## A slide is drawables, and the drawable list is the one to walk @@ -163,6 +190,93 @@ Slides carry no name in the archive, so they are numbered in presentation order the way `oldms/presentation` numbers `.ppt` slides. The slide size is the show's (field 4), 1024×768 points on both fixtures. +## A table is tiles, and a tile is packed cell records + +`TST.TableInfoArchive` (6000) is the drawable; `TST.TableModelArchive` (6001) +behind it carries the name, the extent (**field 6 is rows and field 7 is +columns** — pinned by the `Wide` table of `style-various-1.numbers`, three rows +of six columns) and a `TST.DataStore`. + +The data store holds the **tiles** — each covering a range of rows, `256` on +every fixture — and the side lists cells reference by key. Two are read: the +string list and the rich text list. Formats, formulas and styles are the +others, and none is read, which is why `0.075` renders as `0.075` where Numbers +shows `7.5%`. + +A tile carries a `TST.TileRowInfo` per row that holds anything. Its cells are +packed back to back in one buffer, addressed by an `std::int16_t` per column +that is `-1` where the row has no cell there — so a sparse row costs nothing +and the reader gets its extent from the offsets rather than from a count. + +A cell record is a twelve-byte header — a version byte, a type byte, six bytes +nothing reads, then a flags word — followed by the optional fields the flags +name, **in ascending bit order**. The five low bits are the value: a decimal128, +a double, a date's seconds, a string key, a rich text key. The rest name styles +and formats, so the walk stops at the value and never needs their widths. + +`TST.TileRowInfo` also carries the same cells in an older encoding in two other +fields. The version byte says which to read; a record that declares a version +we have not seen reads as an **empty** cell rather than a wrong one, which is +the whole point of the pinned table. The same holds one level up: a drawable +whose archive is not a table info, or a table info naming no model, costs the +table rather than the file — `read_table` is a per-drawable reader, where a +skip loses one table and a throw loses the document. (The spine entry points +do throw on a wrong root type, because there a mismatch leaves nothing to +render either way.) `IworkNumbers.a_record_we_have_not_mapped_is_an_empty_cell` +pins the version and the type byte; the framing below them — +`a_row_that_contradicts_its_own_offsets_throws` — still fails fast. + +**A number is a decimal, and stays one.** Apple stores cell values as IEEE 754 +decimal128 precisely so `0.1 + 0.2` is `0.3`; converting through a `double` on +the way out would put back the rounding the type exists to avoid. The reader +divides the 113-bit coefficient down by ten and formats the digits against the +exponent, so what comes out is what the file says. `util::number::to_string_significant` +is not that tool — it is documented for CSS lengths and clamps at fifteen +decimals. + +Cell types the fixtures pin: number, string, date, boolean, duration and rich +text. A date is seconds from **2001-01-01T00:00:00Z** and renders ISO 8601; a +duration renders `1d 2h 3m 4s`; a boolean renders `TRUE`/`FALSE`. Only a number +reports `ValueType::float_number`, which is what right-aligns a cell. + +## A Numbers sheet is many tables, and an odr sheet is one grid + +`TN.DocumentArchive` repeats its sheets in field 1; a `TN.SheetArchive` names +itself in field 1 and repeats its drawables in field 2. Each table drawable +becomes **an odr sheet of its own**, named ` –
` — taking only the +first table of a sheet would drop data with nothing to show for it. A drawable +that is not a table (a chart, a text box) is skipped; `sheet_first_shape` is +where those go when they are read. + +A sheet's cells are looked up by coordinate rather than walked, so they are not +in its child chain — that chain is for shapes. `append_sheet_cell` sets only +the parent, as `odf` does. A position no tile carries has no element, and the +public `Sheet::cell` hands back an empty `SheetCell` for it. + +The declared extent is the table's; the **content** extent is computed from the +cells, so a blank table of 22 rows renders as one empty row rather than 22. + +## A Pages table hangs off the text that anchors it + +The `U+FFFC` stage 2 drops is named by the storage's **attachment run table** +(field 9), whose entries pair a UTF-16 index with the object anchored there — +verified against the three anchors of `style-various-1.pages` at indices 18, +460 and 469. That object is a `TSWP.DrawableAttachmentArchive` (2003) whose +field 1 is the drawable; where that drawable is a `TST.TableInfoArchive` the +table is read with the same reader Numbers uses. + +The table is emitted **after the paragraph its anchor sits in**, as a sibling +rather than a child: the anchor is an inline character but a table is not +something a paragraph can hold. Anchors that resolve to something else — a +table of contents (2241), an image (3005) — are skipped, and the anchor stays +dropped from the text. + +A Pages table's cells are **rich text**: the tile holds a key into the rich +text list, whose entry references a `TST.RichTextPayloadArchive` (6218) whose +field 1 is an ordinary `TSWP.StorageArchive`. So a cell's paragraphs come from +the same walk a body does — which is why that walk takes a depth bound: a cell +holds a storage which may hold a table again. + ## Paragraphs come from the run table A `TSWP.StorageArchive` holds its text as a few large strings plus run tables @@ -192,9 +306,11 @@ drawable list is therefore absent, which must come back as one empty slide. ## Not read yet `Index/DocumentStylesheet.iwa` (so `text_root_page_layout` is empty and every -style is the default), images, `Index/Tables/`, Keynote masters -(`Index/TemplateSlide-*.iwa`, so `slide_master_page` is null) and presenter -notes, drawables anchored in a Pages text flow, and everything `PLAN.md` lists -as deferred. `password_encrypted()` is not answered either: an encrypted +style is the default), images, Keynote masters (`Index/TemplateSlide-*.iwa`, so +`slide_master_page` is null) and presenter notes, non-table drawables in a +Pages text flow or on a Numbers sheet, number formats, `CalculationEngine.iwa` +(a cell shows what the app last computed, which is the position `.xls` takes), +merged cell ranges (they live outside the tiles, so a merge renders as separate +cells), and everything `PLAN.md` lists as deferred. `password_encrypted()` is not answered either: an encrypted package is one whose `Index/Document.iwa` does not decompress, which falls back to reporting the file as a zip. diff --git a/src/odr/internal/iwork/PLAN.md b/src/odr/internal/iwork/PLAN.md index ff354dc5..5e5981fc 100644 --- a/src/odr/internal/iwork/PLAN.md +++ b/src/odr/internal/iwork/PLAN.md @@ -1,30 +1,28 @@ # iWork plan Where an iwork module goes, and in what order. Written before stage 1; kept -honest as stages land. **Stages 1, 2 and 5 have landed** — see +honest as stages land. **Stages 1, 2 and 5 through 7 have landed** — see [`AGENTS.md`](AGENTS.md) for what they decided. Stage 5 was pulled ahead of 3 and 4 because it needed neither: a slide is a container above the text storage -stage 2 already read. Stage 6 is next. +stage 2 already read. Stages 3 and 4 are what is left. ## Today -A `.pages` opens as a text document and renders its body text; a `.key` opens -as a presentation and renders each slide's text boxes as positioned frames. -`.numbers` has a `FileType` entry and a `file_type_table.cpp` row so a caller -can name it, but no capabilities and no engine behind it. +All three formats decode. A `.pages` opens as a text document and renders its +body text and the tables its text anchors; a `.key` opens as a presentation and +renders each slide's text boxes as positioned frames; a `.numbers` opens as a +spreadsheet, one odr sheet per Numbers table, showing the values the app last +computed. Six fixtures are committed: `test/data/input/odr-public/pages/{empty.pages,style-various-1.pages}`, written -by iWork 13.2, and `test/data/input/odr-public/{key/{empty.key, -style-various-1.key},numbers/{empty.numbers,style-various-1.numbers}}`, written -by iWork 14.4 (`Metadata/BuildVersionHistory.plist`). None is listed in -`index.csv` — they do not need to be, `TestData` picks up anything the file -type table knows an extension for — and each gained reference output when its -format turned `translate_html` on. Nothing decodes the `.numbers` pair yet; -they are what pins the Keynote-versus-Numbers detection rule. -`style-various-1.pages` carries `Index/Tables/` and nine files under `Data/`, -and `style-various-1.key` a table on its last slide, which is most of the -surface below. +by iWork 13.2, and `key/{empty.key,style-various-1.key}` plus +`numbers/{empty.numbers,style-various-1.numbers}`, written by iWork 14.4 +(`Metadata/BuildVersionHistory.plist`). None is listed in `index.csv` — they do +not need to be, `TestData` picks up anything the file type table knows an +extension for — and each gained reference output when its format turned +`translate_html` on. What is left below is styles and drawables, which none of +them is blocked on. ## Spec @@ -82,20 +80,22 @@ sheets) is per-app. So `iwork/` stays flat, with **one** `ElementRegistry`, one |---|---| | `iwork_text.cpp` | `TSWP` storage → paragraphs *(landed)* | | `iwork_drawable.cpp` | `TSD` geometry, shapes, images *(stage 4)* | -| `iwork_table.cpp` | `TST` tiles → table, row, cell *(stage 6)* | +| `iwork_table.cpp` | `TST` tiles → a `TableModel` *(landed)* | | `iwork_style.cpp` | `TSS` property-set inheritance *(stage 3)* | | `iwork_parser.cpp` | the three spines, calling the above | -The evidence is stage 6: the tile reader is written for **Pages** tables, a -`.key` slide carries a `TST.TableInfoArchive` too, and stage 7 puts Numbers on -top of the same reader. The most expensive component still ahead is shared by -all three apps, which an app-shaped split would either duplicate or push into a -`common/` holding most of the module. +Stage 6 is what proved it: the tile reader was written for **Pages** tables, a +`.key` slide carries a `TST.TableInfoArchive` too, and stage 7 put Numbers on +top of the same reader without touching it. The most expensive component in the +module is shared by all three apps, which an app-shaped split would either have +duplicated or pushed into a `common/` holding most of the module. -Only `iwork_text.cpp` and the `Budget` it spends against are split out today — -the rest follows the stage that writes it. Merging three registries later is -not mechanical; splitting one is, so the unified side is the cheap side to be -wrong on. +`iwork_table.cpp` reads bytes and returns a `TableModel` — no registry, no +elements — so what turns a model into elements sits in `iwork_text.cpp` beside +the storage walk that a cell recurses into. `iwork_drawable.cpp` and +`iwork_style.cpp` follow the stage that writes them. Merging three registries +later is not mechanical; splitting one is, so the unified side is the cheap +side to be wrong on. **No new dependencies.** Two pieces would normally be a conan line each, and both are wrong here: @@ -302,7 +302,7 @@ Landed with three deviations: needs for Pages; what stage 4 still owes is images, shapes and the anchoring a Pages text flow does. -## Stage 6 — the tile reader +## Stage 6 — the tile reader *(landed)* Tables in iWork are stored as **tiles** — row ranges holding packed cell records — with strings, formats and formulas kept in side "data lists" that @@ -314,7 +314,20 @@ Do this for **Pages tables first** (`Table`, `TableRow`, `TableCell`), because `style-various-1.pages` already carries `Index/Tables/` and exercises the reader without any Numbers archive being mapped. -## Stage 7 — Numbers +Landed as planned, with two things the fixtures settled: + +- **One record layout, not several.** iWork 13.2 and 14.4 both write version 5, + and both keep an older encoding of the same cells in two further fields of + the row. The version byte on the record is what decides, and anything else + reads as an empty cell. +- **A Pages table cell is rich text and a Numbers one usually is not.** The + same reader covers both because the cell record only carries a key; what it + keys into is a string in one case and a storage in the other. + +A number is decoded as the IEEE 754 decimal128 it is stored as, exactly, rather +than through a `double` — see `AGENTS.md`. + +## Stage 7 — Numbers *(landed)* - sheets → one odr `Sheet` per table (see decisions), on top of stage 6. - cached values only. `CalculationEngine.iwa` holds the formula graph and is not @@ -322,6 +335,10 @@ without any Numbers archive being mapped. takes. - `iwork_numbers` gains `.translate_html = true`. +Landed as planned. Number formats are not applied — a cell Numbers shows as +`7.5%` renders `0.075` — and merged ranges are recorded outside the tiles, so a +merge renders as separate cells. + --- ## Deferred, by decision @@ -361,13 +378,13 @@ without any Numbers archive being mapped. knows — and reference output was regenerated when stage 2 flipped `translate_html` on. -`empty.numbers` and `style-various-1.numbers` are in -`test/data/input/odr-public/numbers/`. Stage 7 is what will decode them; today -they are the negative that pins detection -(`IworkKeynote.a_numbers_package_is_not_keynote`). Everything at container -level stays inline, per stage 1. +Every stage that has landed has a fixture. Everything at container level stays +inline, per stage 1. -The `.key` fixtures were authored on macOS with Keynote 14.4 rather than found: -there is no spec, so a file the app wrote is the only citation available, and -one written to order can carry exactly the shapes a stage needs — a title -slide, a bulleted body, an empty placeholder, a free text box and a table. +The `.key` and `.numbers` fixtures were authored on macOS with iWork 14.4 +rather than found: there is no spec, so a file the app wrote is the only +citation available, and one written to order can carry exactly the shapes a +stage needs. `style-various-1.numbers` is built around that — a table wider +than it is tall so rows and columns cannot be confused, one cell of every type +the reader maps, a row the tiles skip entirely, and two formula cells whose +cached values are what a reader sees. diff --git a/src/odr/internal/iwork/iwork_archive.cpp b/src/odr/internal/iwork/iwork_archive.cpp index 7b579265..79348145 100644 --- a/src/odr/internal/iwork/iwork_archive.cpp +++ b/src/odr/internal/iwork/iwork_archive.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -184,4 +185,31 @@ std::vector iwork::read_objects(const std::string_view data) { return result; } +std::optional +iwork::reference_identifier(const Message &message, + const std::uint32_t number) { + const std::optional bytes = message.bytes_field(number); + if (!bytes.has_value()) { + return {}; + } + return Message(*bytes).number_field(reference::identifier); +} + +std::vector +iwork::reference_identifiers(const Message &message, + const std::uint32_t number) { + std::vector result; + for (const Field &field : message.repeated_field(number)) { + if (field.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed reference"); + } + if (const std::optional identifier = + Message(field.bytes).number_field(reference::identifier); + identifier.has_value()) { + result.push_back(*identifier); + } + } + return result; +} + } // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_archive.hpp b/src/odr/internal/iwork/iwork_archive.hpp index c1697ffb..c1736356 100644 --- a/src/odr/internal/iwork/iwork_archive.hpp +++ b/src/odr/internal/iwork/iwork_archive.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ class ReadableFilesystem; } // namespace odr::internal::abstract namespace odr::internal::iwork { +class Message; /// One object of an `.iwa`, per `TSP.ArchiveInfo`. Where it holds more than /// one message, only the first is modelled. @@ -88,4 +90,12 @@ std::string read_iwa(const abstract::ReadableFilesystem &filesystem, /// Splits a decompressed `.iwa` into its objects, viewing @p data. std::vector read_objects(std::string_view data); +/// The object the `TSP.Reference` in field @p number names. +std::optional reference_identifier(const Message &message, + std::uint32_t number); + +/// The objects the repeated `TSP.Reference` field @p number names, in order. +std::vector reference_identifiers(const Message &message, + std::uint32_t number); + } // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_budget.cpp b/src/odr/internal/iwork/iwork_budget.cpp deleted file mode 100644 index 5c14aa5a..00000000 --- a/src/odr/internal/iwork/iwork_budget.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include - -#include - -namespace odr::internal::iwork { - -void Budget::spend_element() { - if (++m_elements > element_limit) { - throw std::runtime_error("iwork: document holds too many elements"); - } -} - -void Budget::spend_text(const std::size_t bytes) { - m_text += bytes; - if (m_text > text_limit) { - throw std::runtime_error("iwork: document holds too much text"); - } -} - -} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_budget.hpp b/src/odr/internal/iwork/iwork_budget.hpp index f0b28b95..cb7cfe10 100644 --- a/src/odr/internal/iwork/iwork_budget.hpp +++ b/src/odr/internal/iwork/iwork_budget.hpp @@ -1,19 +1,27 @@ #pragma once #include +#include namespace odr::internal::iwork { -/// What one parse may expand to. An `.iwa` is an object graph, so a reference -/// list may name the same object any number of times and `Package::object` -/// hands every repeat back from its cache — a few kilobytes of references -/// would otherwise build elements and copy text without bound. Spending -/// against a budget keeps such a package the thrown error every caller already -/// handles rather than an allocation the process dies on. +/// What one parse may expand to. A reference list may name the same object any +/// number of times, so what a walk builds is spent against a limit rather than +/// left to grow with what a few bytes of references ask for. class Budget final { public: - void spend_element(); - void spend_text(std::size_t bytes); + void spend_element() { + if (++m_elements > element_limit) { + throw std::runtime_error("iwork: document holds too many elements"); + } + } + + void spend_text(const std::size_t bytes) { + m_text += bytes; + if (m_text > text_limit) { + throw std::runtime_error("iwork: document holds too much text"); + } + } private: /// Far above what an authored document reaches, and far below what the diff --git a/src/odr/internal/iwork/iwork_document.cpp b/src/odr/internal/iwork/iwork_document.cpp index ee321d85..530b762d 100644 --- a/src/odr/internal/iwork/iwork_document.cpp +++ b/src/odr/internal/iwork/iwork_document.cpp @@ -4,11 +4,15 @@ #include #include #include +#include +#include #include #include #include +#include +#include #include #include @@ -27,6 +31,8 @@ ElementIdentifier parse_tree(ElementRegistry ®istry, return parse_pages_tree(registry, files); case FileType::iwork_keynote: return parse_keynote_tree(registry, files); + case FileType::iwork_numbers: + return parse_numbers_tree(registry, files); default: throw UnsupportedFileType(file_type); } @@ -70,6 +76,12 @@ namespace { class ElementAdapter final : public abstract::ElementAdapter, public abstract::TextRootAdapter, public abstract::SlideAdapter, + public abstract::SheetAdapter, + public abstract::SheetCellAdapter, + public abstract::TableAdapter, + public abstract::TableColumnAdapter, + public abstract::TableRowAdapter, + public abstract::TableCellAdapter, public abstract::FrameAdapter, public abstract::LineBreakAdapter, public abstract::ParagraphAdapter, @@ -133,6 +145,31 @@ class ElementAdapter final : public abstract::ElementAdapter, slide_adapter(const ElementIdentifier element_id) const override { return element_type(element_id) == ElementType::slide ? this : nullptr; } + [[nodiscard]] const SheetAdapter * + sheet_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::sheet ? this : nullptr; + } + [[nodiscard]] const SheetCellAdapter * + sheet_cell_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::sheet_cell ? this : nullptr; + } + [[nodiscard]] const TableAdapter * + table_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::table ? this : nullptr; + } + [[nodiscard]] const TableColumnAdapter * + table_column_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::table_column ? this + : nullptr; + } + [[nodiscard]] const TableRowAdapter * + table_row_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::table_row ? this : nullptr; + } + [[nodiscard]] const TableCellAdapter * + table_cell_adapter(const ElementIdentifier element_id) const override { + return element_type(element_id) == ElementType::table_cell ? this : nullptr; + } [[nodiscard]] const FrameAdapter * frame_adapter(const ElementIdentifier element_id) const override { return element_type(element_id) == ElementType::frame ? this : nullptr; @@ -185,6 +222,121 @@ class ElementAdapter final : public abstract::ElementAdapter, return null_element_id; } + [[nodiscard]] std::string + sheet_name(const ElementIdentifier element_id) const override { + return m_registry->sheet_element_at(element_id).name; + } + [[nodiscard]] TableDimensions + sheet_dimensions(const ElementIdentifier element_id) const override { + const ElementRegistry::Sheet &sheet = + m_registry->sheet_element_at(element_id); + return {sheet.rows, sheet.columns}; + } + [[nodiscard]] TableDimensions + sheet_content(const ElementIdentifier element_id, + const std::optional range) const override { + const ElementRegistry::Sheet &sheet = + m_registry->sheet_element_at(element_id); + if (!range.has_value()) { + return {sheet.content_rows, sheet.content_columns}; + } + return {std::min(sheet.content_rows, range->rows), + std::min(sheet.content_columns, range->columns)}; + } + [[nodiscard]] ElementIdentifier + sheet_cell(const ElementIdentifier element_id, const std::uint32_t column, + const std::uint32_t row) const override { + return m_registry->sheet_element_at(element_id).cell(column, row); + } + [[nodiscard]] ElementIdentifier sheet_first_shape( + [[maybe_unused]] const ElementIdentifier element_id) const override { + // a chart or a text box on the sheet; none read yet + return null_element_id; + } + [[nodiscard]] TableStyle sheet_style( + [[maybe_unused]] const ElementIdentifier element_id) const override { + return {}; + } + [[nodiscard]] TableColumnStyle sheet_column_style( + [[maybe_unused]] const ElementIdentifier element_id, + [[maybe_unused]] const std::uint32_t column) const override { + return {}; + } + [[nodiscard]] TableRowStyle + sheet_row_style([[maybe_unused]] const ElementIdentifier element_id, + [[maybe_unused]] const std::uint32_t row) const override { + return {}; + } + [[nodiscard]] TableCellStyle + sheet_cell_style([[maybe_unused]] const ElementIdentifier element_id, + [[maybe_unused]] const std::uint32_t column, + [[maybe_unused]] const std::uint32_t row) const override { + return {}; + } + + [[nodiscard]] TablePosition + sheet_cell_position(const ElementIdentifier element_id) const override { + const ElementRegistry::Cell &cell = m_registry->cell_element_at(element_id); + // `TablePosition` is (column, row); `TableDimensions` is (rows, columns) + return TablePosition(cell.column, cell.row); + } + [[nodiscard]] bool sheet_cell_is_covered( + [[maybe_unused]] const ElementIdentifier element_id) const override { + return false; + } + [[nodiscard]] TableDimensions sheet_cell_span( + [[maybe_unused]] const ElementIdentifier element_id) const override { + // merged ranges are recorded outside the tiles and are not read yet + return {1, 1}; + } + [[nodiscard]] ValueType + sheet_cell_value_type(const ElementIdentifier element_id) const override { + return m_registry->cell_element_at(element_id).value_type; + } + + [[nodiscard]] TableDimensions + table_dimensions(const ElementIdentifier element_id) const override { + const ElementRegistry::Table &table = + m_registry->table_element_at(element_id); + return {table.rows, table.columns}; + } + [[nodiscard]] ElementIdentifier + table_first_column(const ElementIdentifier element_id) const override { + return m_registry->table_element_at(element_id).first_column_id; + } + [[nodiscard]] ElementIdentifier + table_first_row(const ElementIdentifier element_id) const override { + return element_first_child(element_id); + } + [[nodiscard]] TableStyle table_style( + [[maybe_unused]] const ElementIdentifier element_id) const override { + return {}; + } + [[nodiscard]] TableColumnStyle table_column_style( + [[maybe_unused]] const ElementIdentifier element_id) const override { + return {}; + } + [[nodiscard]] TableRowStyle table_row_style( + [[maybe_unused]] const ElementIdentifier element_id) const override { + return {}; + } + [[nodiscard]] bool table_cell_is_covered( + [[maybe_unused]] const ElementIdentifier element_id) const override { + return false; + } + [[nodiscard]] TableDimensions table_cell_span( + [[maybe_unused]] const ElementIdentifier element_id) const override { + return {1, 1}; + } + [[nodiscard]] ValueType + table_cell_value_type(const ElementIdentifier element_id) const override { + return m_registry->cell_element_at(element_id).value_type; + } + [[nodiscard]] TableCellStyle table_cell_style( + [[maybe_unused]] const ElementIdentifier element_id) const override { + return {}; + } + [[nodiscard]] AnchorType frame_anchor_type( [[maybe_unused]] const ElementIdentifier element_id) const override { return AnchorType::at_page; diff --git a/src/odr/internal/iwork/iwork_document.hpp b/src/odr/internal/iwork/iwork_document.hpp index 8b734df2..5759ad54 100644 --- a/src/odr/internal/iwork/iwork_document.hpp +++ b/src/odr/internal/iwork/iwork_document.hpp @@ -10,7 +10,8 @@ namespace odr::internal::iwork { /// An iWork package, read as the kind of document the app that wrote it -/// makes: a `.pages` as text, a `.key` as a presentation. +/// makes: a `.pages` as text, a `.key` as a presentation, a `.numbers` as a +/// spreadsheet. class Document final : public internal::Document { public: Document(FileType file_type, diff --git a/src/odr/internal/iwork/iwork_element_registry.cpp b/src/odr/internal/iwork/iwork_element_registry.cpp index aab2f992..d188b533 100644 --- a/src/odr/internal/iwork/iwork_element_registry.cpp +++ b/src/odr/internal/iwork/iwork_element_registry.cpp @@ -1,6 +1,7 @@ #include #include +#include namespace odr::internal::iwork { @@ -9,6 +10,15 @@ void ElementRegistry::clear() noexcept { m_texts.clear(); m_frames.clear(); m_slides.clear(); + m_tables.clear(); + m_sheets.clear(); + m_cells.clear(); +} + +ElementIdentifier ElementRegistry::Sheet::cell(const std::uint32_t column, + const std::uint32_t row) const { + const auto it = cells.find({row, column}); + return it == cells.end() ? null_element_id : it->second; } [[nodiscard]] std::size_t ElementRegistry::size() const noexcept { @@ -47,6 +57,30 @@ ElementRegistry::create_slide_element() { return {element_id, element, it->second}; } +std::tuple +ElementRegistry::create_table_element() { + const auto &[element_id, element] = create_element(ElementType::table); + auto [it, success] = m_tables.emplace(element_id, Table{}); + return {element_id, element, it->second}; +} + +std::tuple +ElementRegistry::create_sheet_element() { + const auto &[element_id, element] = create_element(ElementType::sheet); + auto [it, success] = m_sheets.emplace(element_id, Sheet{}); + return {element_id, element, it->second}; +} + +std::tuple +ElementRegistry::create_cell_element(const ElementType type) { + const auto &[element_id, element] = create_element(type); + auto [it, success] = m_cells.emplace(element_id, Cell{}); + return {element_id, element, it->second}; +} + ElementRegistry::Element & ElementRegistry::element_at(const ElementIdentifier id) { check_element_id(id); @@ -71,6 +105,24 @@ ElementRegistry::slide_element_at(const ElementIdentifier id) { return m_slides.at(id); } +ElementRegistry::Table & +ElementRegistry::table_element_at(const ElementIdentifier id) { + check_table_id(id); + return m_tables.at(id); +} + +ElementRegistry::Sheet & +ElementRegistry::sheet_element_at(const ElementIdentifier id) { + check_sheet_id(id); + return m_sheets.at(id); +} + +ElementRegistry::Cell & +ElementRegistry::cell_element_at(const ElementIdentifier id) { + check_cell_id(id); + return m_cells.at(id); +} + const ElementRegistry::Element & ElementRegistry::element_at(const ElementIdentifier id) const { check_element_id(id); @@ -95,6 +147,24 @@ ElementRegistry::slide_element_at(const ElementIdentifier id) const { return m_slides.at(id); } +const ElementRegistry::Table & +ElementRegistry::table_element_at(const ElementIdentifier id) const { + check_table_id(id); + return m_tables.at(id); +} + +const ElementRegistry::Sheet & +ElementRegistry::sheet_element_at(const ElementIdentifier id) const { + check_sheet_id(id); + return m_sheets.at(id); +} + +const ElementRegistry::Cell & +ElementRegistry::cell_element_at(const ElementIdentifier id) const { + check_cell_id(id); + return m_cells.at(id); +} + void ElementRegistry::append_child(const ElementIdentifier parent_id, const ElementIdentifier child_id) { check_element_id(parent_id); @@ -118,6 +188,43 @@ void ElementRegistry::append_child(const ElementIdentifier parent_id, element_at(parent_id).last_child_id = child_id; } +void ElementRegistry::append_table_column(const ElementIdentifier table_id, + const ElementIdentifier column_id) { + check_table_id(table_id); + check_element_id(column_id); + if (element_at(column_id).parent_id != null_element_id) { + throw std::invalid_argument( + "ElementRegistry::append_table_column: column already has a parent"); + } + + Table &table = table_element_at(table_id); + const ElementIdentifier previous_id = table.last_column_id; + + element_at(column_id).parent_id = table_id; + element_at(column_id).previous_sibling_id = previous_id; + if (previous_id == null_element_id) { + table.first_column_id = column_id; + } else { + element_at(previous_id).next_sibling_id = column_id; + } + table.last_column_id = column_id; +} + +void ElementRegistry::append_sheet_cell(const ElementIdentifier sheet_id, + const ElementIdentifier cell_id) { + check_sheet_id(sheet_id); + check_cell_id(cell_id); + if (element_at(cell_id).parent_id != null_element_id) { + throw std::invalid_argument( + "ElementRegistry::append_sheet_cell: cell already has a parent"); + } + + const Cell &cell = cell_element_at(cell_id); + element_at(cell_id).parent_id = sheet_id; + sheet_element_at(sheet_id).cells.emplace(std::pair(cell.row, cell.column), + cell_id); +} + void ElementRegistry::check_element_id(const ElementIdentifier id) const { if (id == null_element_id) { throw std::out_of_range("ElementRegistry::check_id: null identifier"); @@ -149,4 +256,25 @@ void ElementRegistry::check_slide_id(const ElementIdentifier id) const { } } +void ElementRegistry::check_table_id(const ElementIdentifier id) const { + check_element_id(id); + if (!m_tables.contains(id)) { + throw std::out_of_range("ElementRegistry::check_id: identifier not found"); + } +} + +void ElementRegistry::check_sheet_id(const ElementIdentifier id) const { + check_element_id(id); + if (!m_sheets.contains(id)) { + throw std::out_of_range("ElementRegistry::check_id: identifier not found"); + } +} + +void ElementRegistry::check_cell_id(const ElementIdentifier id) const { + check_element_id(id); + if (!m_cells.contains(id)) { + throw std::out_of_range("ElementRegistry::check_id: identifier not found"); + } +} + } // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_element_registry.hpp b/src/odr/internal/iwork/iwork_element_registry.hpp index 6f7d1c75..ab9a9c39 100644 --- a/src/odr/internal/iwork/iwork_element_registry.hpp +++ b/src/odr/internal/iwork/iwork_element_registry.hpp @@ -4,10 +4,13 @@ #include #include +#include +#include #include #include #include #include +#include #include namespace odr::internal::iwork { @@ -53,6 +56,40 @@ class ElementRegistry final { std::optional size; }; + /// Where a cell sits in the table or sheet it belongs to, and what kind of + /// value it holds. + struct Cell final { + std::uint32_t row{}; + std::uint32_t column{}; + ValueType value_type{ValueType::unknown}; + }; + + /// A table anchored in a text flow. Its rows are its children; its columns + /// are a chain of their own, as they are in `odf`. + struct Table final { + std::uint32_t rows{}; + std::uint32_t columns{}; + ElementIdentifier first_column_id{null_element_id}; + ElementIdentifier last_column_id{null_element_id}; + }; + + /// One odr sheet, which is one Numbers table. Cells are reached by + /// coordinate rather than by walking, so they are not in the child chain — + /// that is what a sheet's shapes would use. + struct Sheet final { + std::string name; + std::uint32_t rows{}; + std::uint32_t columns{}; + /// The extent that actually holds something, which is what a sheet of a + /// hundred empty rows should render. + std::uint32_t content_rows{}; + std::uint32_t content_columns{}; + std::map, ElementIdentifier> cells; + + [[nodiscard]] ElementIdentifier cell(std::uint32_t column, + std::uint32_t row) const; + }; + void clear() noexcept; [[nodiscard]] std::size_t size() const noexcept; @@ -61,29 +98,53 @@ class ElementRegistry final { std::tuple create_text_element(); std::tuple create_frame_element(); std::tuple create_slide_element(); + std::tuple create_table_element(); + std::tuple create_sheet_element(); + std::tuple + create_cell_element(ElementType type); [[nodiscard]] Element &element_at(ElementIdentifier id); [[nodiscard]] Text &text_element_at(ElementIdentifier id); [[nodiscard]] Frame &frame_element_at(ElementIdentifier id); [[nodiscard]] Slide &slide_element_at(ElementIdentifier id); + [[nodiscard]] Table &table_element_at(ElementIdentifier id); + [[nodiscard]] Sheet &sheet_element_at(ElementIdentifier id); + [[nodiscard]] Cell &cell_element_at(ElementIdentifier id); [[nodiscard]] const Element &element_at(ElementIdentifier id) const; [[nodiscard]] const Text &text_element_at(ElementIdentifier id) const; [[nodiscard]] const Frame &frame_element_at(ElementIdentifier id) const; [[nodiscard]] const Slide &slide_element_at(ElementIdentifier id) const; + [[nodiscard]] const Table &table_element_at(ElementIdentifier id) const; + [[nodiscard]] const Sheet &sheet_element_at(ElementIdentifier id) const; + [[nodiscard]] const Cell &cell_element_at(ElementIdentifier id) const; void append_child(ElementIdentifier parent_id, ElementIdentifier child_id); + /// Links @p column_id into @p table_id's column chain. Columns are not + /// children: a table's child chain is its rows. + void append_table_column(ElementIdentifier table_id, + ElementIdentifier column_id); + /// Files @p cell_id at its coordinate in @p sheet_id. A sheet's cells are + /// looked up rather than walked, so this only sets the parent. + void append_sheet_cell(ElementIdentifier sheet_id, ElementIdentifier cell_id); + private: std::vector m_elements; std::unordered_map m_texts; std::unordered_map m_frames; std::unordered_map m_slides; + std::unordered_map m_tables; + std::unordered_map m_sheets; + std::unordered_map m_cells; void check_element_id(ElementIdentifier id) const; void check_text_id(ElementIdentifier id) const; void check_frame_id(ElementIdentifier id) const; void check_slide_id(ElementIdentifier id) const; + void check_table_id(ElementIdentifier id) const; + void check_sheet_id(ElementIdentifier id) const; + void check_cell_id(ElementIdentifier id) const; }; } // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_file.cpp b/src/odr/internal/iwork/iwork_file.cpp index 8e2348e7..9dc9c9f9 100644 --- a/src/odr/internal/iwork/iwork_file.cpp +++ b/src/odr/internal/iwork/iwork_file.cpp @@ -32,7 +32,7 @@ FileType app_by_components(const abstract::ReadableFilesystem &filesystem) { if (package.has_component(slide_component)) { return FileType::iwork_keynote; } - return FileType::unknown; + return FileType::iwork_numbers; } /// Reads the root archive of the package's `Document` component. The component @@ -97,6 +97,7 @@ std::shared_ptr IworkFile::document() const { switch (file_type()) { case FileType::iwork_pages: case FileType::iwork_keynote: + case FileType::iwork_numbers: return std::make_shared(file_type(), m_filesystem); default: throw UnsupportedFileType(file_type()); diff --git a/src/odr/internal/iwork/iwork_parser.cpp b/src/odr/internal/iwork/iwork_parser.cpp index e332e96d..65bd89ba 100644 --- a/src/odr/internal/iwork/iwork_parser.cpp +++ b/src/odr/internal/iwork/iwork_parser.cpp @@ -4,9 +4,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -19,33 +21,6 @@ namespace odr::internal::iwork { namespace { -/// The object a `TSP.Reference` in field @p number names. -std::optional reference_identifier(const Message &message, - const std::uint32_t number) { - const std::optional bytes = message.bytes_field(number); - if (!bytes.has_value()) { - return {}; - } - return Message(*bytes).number_field(reference::identifier); -} - -/// The objects the repeated `TSP.Reference` field @p number names, in order. -std::vector reference_identifiers(const Message &message, - const std::uint32_t number) { - std::vector result; - for (const Field &field : message.repeated_field(number)) { - if (field.type != WireType::length_delimited) { - throw std::runtime_error("iwork: malformed reference"); - } - if (const std::optional identifier = - Message(field.bytes).number_field(reference::identifier); - identifier.has_value()) { - result.push_back(*identifier); - } - } - return result; -} - /// The root archive of the package's `Document` component, checked against the /// type the app is expected to write. const Object &root_archive(Package &package, const std::uint32_t type) { @@ -147,8 +122,10 @@ std::optional text_shape_of(const Object &drawable) { } /// Appends one slide's text boxes to @p slide_id as frames. -void parse_slide(ElementRegistry ®istry, Budget &budget, Package &package, - const ElementIdentifier slide_id, const Object &slide) { +void parse_slide(const Context &context, const ElementIdentifier slide_id, + const Object &slide) { + ElementRegistry ®istry = *context.registry; + Package &package = *context.package; const Message slide_message(slide.payload); for (const std::uint64_t identifier : @@ -169,7 +146,7 @@ void parse_slide(ElementRegistry ®istry, Budget &budget, Package &package, continue; } - budget.spend_element(); + context.budget->spend_element(); auto [frame_id, frame, payload] = registry.create_frame_element(); if (const std::optional inner = shape->bytes_field(text_shape::shape); @@ -178,7 +155,58 @@ void parse_slide(ElementRegistry ®istry, Budget &budget, Package &package, } registry.append_child(slide_id, frame_id); - parse_storage(registry, budget, frame_id, Message(storage.payload)); + parse_storage(context, frame_id, Message(storage.payload)); + } +} + +/// One Numbers sheet holds many tables and our `Sheet` is one grid, so each +/// table becomes an odr sheet of its own — taking only the first would drop +/// data with nothing to show for it. +void parse_sheet(const Context &context, const ElementIdentifier root_id, + const Object &sheet) { + ElementRegistry ®istry = *context.registry; + Budget &budget = *context.budget; + const Message sheet_message(sheet.payload); + const std::string name = + std::string(sheet_message.bytes_field(sheet_archive::name) + .value_or(std::string_view())); + + for (const std::uint64_t identifier : + reference_identifiers(sheet_message, sheet_archive::drawables)) { + const Object &drawable = context.package->object(identifier); + if (drawable.type != archive_type::table_info) { + // a chart, a text box or an image on the sheet; none read yet + continue; + } + const TableModel &model = + context.tables->table(*context.package, *context.budget, identifier); + + budget.spend_element(); + auto [sheet_id, element, payload] = registry.create_sheet_element(); + // ` –
`, less whichever half the archive leaves empty + payload.name = name; + if (!model.name.empty()) { + payload.name += payload.name.empty() ? model.name : " – " + model.name; + } + payload.rows = model.rows; + payload.columns = model.columns; + registry.append_child(root_id, sheet_id); + + for (const TableModel::Cell &cell : model.cells) { + budget.spend_element(); + auto [cell_id, cell_element, entry] = + registry.create_cell_element(ElementType::sheet_cell); + entry.row = cell.row; + entry.column = cell.column; + entry.value_type = cell.value_type; + registry.append_sheet_cell(sheet_id, cell_id); + + fill_cell(context, cell_id, cell); + + payload.content_rows = std::max(payload.content_rows, cell.row + 1); + payload.content_columns = + std::max(payload.content_columns, cell.column + 1); + } } } @@ -207,8 +235,10 @@ iwork::parse_pages_tree(ElementRegistry ®istry, } Budget budget; + TableCache tables; auto [root_id, root] = registry.create_element(ElementType::root); - parse_storage(registry, budget, root_id, Message(body_object.payload)); + parse_storage({®istry, &package, &budget, &tables, 0}, root_id, + Message(body_object.payload)); return root_id; } @@ -234,6 +264,7 @@ iwork::parse_keynote_tree(ElementRegistry ®istry, read_size(show, show_archive::size); Budget budget; + TableCache tables; auto [root_id, root] = registry.create_element(ElementType::root); // a show that carries no slide tree is a deck with no slides @@ -264,7 +295,31 @@ iwork::parse_keynote_tree(ElementRegistry ®istry, payload.size = slide_size; registry.append_child(root_id, slide_id); - parse_slide(registry, budget, package, slide_id, slide); + parse_slide({®istry, &package, &budget, &tables, 0}, slide_id, slide); + } + + return root_id; +} + +ElementIdentifier +iwork::parse_numbers_tree(ElementRegistry ®istry, + const abstract::ReadableFilesystem &files) { + Package package(files); + + const Message document( + root_archive(package, archive_type::app_document).payload); + + Budget budget; + TableCache tables; + auto [root_id, root] = registry.create_element(ElementType::root); + + for (const std::uint64_t identifier : + reference_identifiers(document, document_archive::sheets)) { + const Object &sheet = package.object(identifier); + if (sheet.type != archive_type::numbers_sheet) { + continue; + } + parse_sheet({®istry, &package, &budget, &tables, 0}, root_id, sheet); } return root_id; diff --git a/src/odr/internal/iwork/iwork_parser.hpp b/src/odr/internal/iwork/iwork_parser.hpp index 0ac9cc1e..bdbe4efc 100644 --- a/src/odr/internal/iwork/iwork_parser.hpp +++ b/src/odr/internal/iwork/iwork_parser.hpp @@ -21,4 +21,10 @@ ElementIdentifier parse_pages_tree(ElementRegistry ®istry, ElementIdentifier parse_keynote_tree(ElementRegistry ®istry, const abstract::ReadableFilesystem &files); +/// Parses a `.numbers` package into root → sheet → cell elements, one sheet +/// per Numbers table. +/// \return the root element id. +ElementIdentifier parse_numbers_tree(ElementRegistry ®istry, + const abstract::ReadableFilesystem &files); + } // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_table.cpp b/src/odr/internal/iwork/iwork_table.cpp new file mode 100644 index 00000000..9a5b02b2 --- /dev/null +++ b/src/odr/internal/iwork/iwork_table.cpp @@ -0,0 +1,514 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace odr::internal::iwork { + +namespace { + +/// The bias an IEEE 754 decimal128 exponent carries. +constexpr std::int32_t decimal128_bias = 6176; + +/// Divides the 128-bit @p value, four little-endian 32-bit limbs, by ten. +/// \return the remainder. +std::uint32_t divide_by_ten(std::array &value) { + std::uint64_t remainder = 0; + for (std::size_t i = value.size(); i-- > 0;) { + const std::uint64_t current = (remainder << 32) | value[i]; + value[i] = static_cast(current / 10); + remainder = current % 10; + } + return static_cast(remainder); +} + +bool is_zero(const std::array &value) { + return std::ranges::all_of( + value, [](const std::uint32_t limb) { return limb == 0; }); +} + +std::uint64_t read_uint64(const std::string_view bytes, + const std::size_t offset) { + return util::byte::from_little_endian(bytes.substr(offset), 8); +} + +std::uint32_t read_uint32(const std::string_view bytes, + const std::size_t offset) { + return util::byte::from_little_endian(bytes.substr(offset), 4); +} + +double read_double(const std::string_view bytes, const std::size_t offset) { + return std::bit_cast(read_uint64(bytes, offset)); +} + +/// One `TST.DataList`, as key → entry. The entries are read once per table +/// because a cell names its value by key and the keys are not dense. +std::unordered_map +read_data_list(Package &package, + const std::optional identifier) { + std::unordered_map result; + if (!identifier.has_value()) { + return result; + } + + const Object &object = package.object(*identifier); + if (object.type != archive_type::data_list) { + return result; + } + + for (const Field &entry : + Message(object.payload).repeated_field(data_list::entries)) { + if (entry.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed data list"); + } + Message message(entry.bytes); + const std::optional key = + message.number_field(data_list_entry::key); + if (!key.has_value()) { + continue; + } + result.emplace(*key, std::move(message)); + } + return result; +} + +/// Where each of the value flags puts its payload, and how wide it is. A cell +/// holds one value, so the walk stops once the flag the type asks for is +/// found; the offsets before it still have to be stepped over. +struct Value final { + std::uint32_t flag{}; + std::size_t size{}; +}; + +constexpr std::array value_layout{{ + {cell::flag::decimal, 16}, + {cell::flag::number, 8}, + {cell::flag::seconds, 8}, + {cell::flag::string_key, 4}, + {cell::flag::rich_text_key, 4}, +}}; + +/// The largest instant either formatter reads, in seconds. `std::chrono::year` +/// runs to ±32767, so a second beyond this names no date a calendar has — +/// and the `std::int64_t` the seconds are cast to holds this with room to +/// spare, which is what keeps the cast defined. +constexpr double max_instant = 1e12; + +/// The offset of the value @p flag names within @p record, or nothing when the +/// flags do not carry it or it would run past the record. +std::optional value_offset(const std::string_view record, + const std::uint32_t flags, + const std::uint32_t flag) { + std::size_t offset = cell::header_size; + for (const auto &[candidate, size] : value_layout) { + if ((flags & candidate) == 0) { + continue; + } + if (candidate == flag) { + return offset + size <= record.size() ? std::optional(offset) + : std::nullopt; + } + offset += size; + } + return {}; +} + +/// Reads one packed cell record. A type or a version we have not mapped comes +/// back empty — there is no spec, so it is a shape Apple ships and we have not +/// seen, not a corrupt file. +TableModel::Cell +read_cell(const std::string_view record, const std::uint32_t row, + const std::uint32_t column, + const std::unordered_map &strings, + const std::unordered_map &rich_texts) { + TableModel::Cell result; + result.row = row; + result.column = column; + + if (record.size() < cell::header_size || + static_cast(record[cell::version_offset]) != + cell::version) { + return result; + } + + const auto type = static_cast(record[cell::type_offset]); + const std::uint32_t flags = read_uint32(record, cell::flags_offset); + + const auto key_at = + [&](const std::uint32_t flag) -> std::optional { + const std::optional offset = value_offset(record, flags, flag); + if (!offset.has_value()) { + return {}; + } + return read_uint32(record, *offset); + }; + const auto number_at = + [&](const std::uint32_t flag) -> std::optional { + const std::optional offset = value_offset(record, flags, flag); + if (!offset.has_value()) { + return {}; + } + return read_double(record, *offset); + }; + + switch (type) { + case cell::type::number: { + const std::optional offset = + value_offset(record, flags, cell::flag::decimal); + if (!offset.has_value()) { + break; + } + result.text = decimal128_to_string(record.substr(*offset, 16)); + result.value_type = ValueType::float_number; + } break; + case cell::type::string: { + const std::optional key = key_at(cell::flag::string_key); + if (!key.has_value()) { + break; + } + const auto it = strings.find(*key); + if (it == strings.end()) { + break; + } + result.text = std::string(it->second.bytes_field(data_list_entry::string) + .value_or(std::string_view())); + result.value_type = ValueType::string; + } break; + case cell::type::date: { + const std::optional seconds = number_at(cell::flag::seconds); + if (!seconds.has_value()) { + break; + } + result.text = date_to_string(*seconds); + result.value_type = ValueType::string; + } break; + case cell::type::boolean: { + const std::optional value = number_at(cell::flag::number); + if (!value.has_value()) { + break; + } + result.text = *value != 0.0 ? "TRUE" : "FALSE"; + result.value_type = ValueType::string; + } break; + case cell::type::duration: { + const std::optional seconds = number_at(cell::flag::number); + if (!seconds.has_value()) { + break; + } + result.text = duration_to_string(*seconds); + result.value_type = ValueType::string; + } break; + case cell::type::rich_text: { + const std::optional key = key_at(cell::flag::rich_text_key); + if (!key.has_value()) { + break; + } + const auto it = rich_texts.find(*key); + if (it == rich_texts.end()) { + break; + } + result.storage_identifier = + reference_identifier(it->second, data_list_entry::rich_text); + result.value_type = ValueType::string; + } break; + default: + break; + } + + return result; +} + +/// Reads the cells of one `TST.Tile`, whose row indices are relative to the +/// tile's own start. +void read_tile(Package &package, Budget &budget, const std::uint64_t identifier, + const std::uint32_t first_row, + const std::unordered_map &strings, + const std::unordered_map &rich_texts, + std::vector &out) { + const Object &object = package.object(identifier); + if (object.type != archive_type::tile) { + return; + } + + for (const Field &row : Message(object.payload).repeated_field(tile::rows)) { + if (row.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed tile row"); + } + const Message info(row.bytes); + + const std::uint32_t index = + first_row + static_cast( + info.number_field(tile_row::index).value_or(0)); + const std::string_view storage = + info.bytes_field(tile_row::storage).value_or(std::string_view()); + const std::string_view offsets = + info.bytes_field(tile_row::offsets).value_or(std::string_view()); + + // one `std::int16_t` per column, `-1` where the row holds no cell; a cell + // runs to the next column that has one + const std::size_t columns = offsets.size() / 2; + std::vector> starts; + for (std::size_t column = 0; column < columns; ++column) { + const auto offset = static_cast( + util::byte::from_little_endian( + offsets.substr(column * 2), 2)); + if (offset < 0) { + continue; + } + if (static_cast(offset) > storage.size()) { + throw std::runtime_error("iwork: cell runs past its row storage"); + } + starts.emplace_back(static_cast(column), + static_cast(offset)); + } + + for (std::size_t i = 0; i < starts.size(); ++i) { + const auto [column, begin] = starts[i]; + const std::size_t end = + i + 1 < starts.size() ? starts[i + 1].second : storage.size(); + if (end < begin) { + throw std::runtime_error("iwork: cell offsets are out of order"); + } + TableModel::Cell cell = read_cell(storage.substr(begin, end - begin), + index, column, strings, rich_texts); + if (cell.value_type != ValueType::unknown) { + // a tile list may name one tile any number of times, so a cell is + // spent as it is produced rather than once the model is complete + budget.spend_element(); + budget.spend_text(cell.text.size()); + out.push_back(std::move(cell)); + } + } + } +} + +} // namespace + +} // namespace odr::internal::iwork + +namespace odr::internal { + +std::string iwork::decimal128_to_string(const std::string_view bytes) { + if (bytes.size() < 16) { + throw std::runtime_error("iwork: decimal128 is cut off"); + } + + const std::uint64_t low = read_uint64(bytes, 0); + const std::uint64_t high = read_uint64(bytes, 8); + + const bool negative = (high >> 63) != 0; + std::int32_t exponent = 0; + std::array coefficient{}; + + if (((high >> 61) & 0x3) == 0x3) { + // the form with a coefficient above 10^34, which IEEE 754 leaves + // non-canonical and reads as zero + exponent = static_cast((high >> 47) & 0x3fff); + } else { + exponent = static_cast((high >> 49) & 0x3fff); + const std::uint64_t significand_high = high & ((1ULL << 49) - 1); + coefficient = {static_cast(low), + static_cast(low >> 32), + static_cast(significand_high), + static_cast(significand_high >> 32)}; + } + exponent -= decimal128_bias; + + std::string digits; + while (!is_zero(coefficient)) { + digits.push_back(static_cast('0' + divide_by_ten(coefficient))); + } + if (digits.empty()) { + return "0"; + } + + // Numbers stores more precision than it shows — 0.075 arrives as + // 750000000000000e-16 — so the zeros that would only pad the fraction go + std::size_t significant = 0; + while (exponent < 0 && significant < digits.size() && + digits[significant] == '0') { + ++significant; + ++exponent; + } + digits.erase(0, significant); + std::ranges::reverse(digits); + + std::string result; + if (exponent >= 0) { + result = digits + std::string(static_cast(exponent), '0'); + } else { + const auto fraction = static_cast(-exponent); + if (fraction >= digits.size()) { + result = "0." + std::string(fraction - digits.size(), '0') + digits; + } else { + result = digits.substr(0, digits.size() - fraction) + "." + + digits.substr(digits.size() - fraction); + } + } + + return negative ? "-" + result : result; +} + +std::string iwork::date_to_string(const double seconds) { + // an instant is cast to `std::int64_t` below, which is undefined for a value + // the type cannot hold, and no `std::chrono::year` can name one this far out + // anyway — so a value beyond the calendar is read as no date at all + if (!std::isfinite(seconds) || std::abs(seconds) > max_instant) { + return {}; + } + + // days since the epoch, and the second within the day, with a negative + // instant flooring rather than truncating toward zero + const auto total = static_cast(std::floor(seconds)); + auto days = static_cast(std::floor(total / 86400.0)); + auto rest = static_cast(total - days * 86400); + + // Apple counts from 2001-01-01, which is 11323 days after 1970-01-01 + days += 11323; + + const std::chrono::year_month_day date{ + std::chrono::sys_days(std::chrono::days(days))}; + + const auto pad = [](const std::int32_t value, const std::size_t width) { + std::string digits = std::to_string(value); + return digits.size() >= width + ? digits + : std::string(width - digits.size(), '0') + digits; + }; + + return pad(static_cast(date.year()), 4) + "-" + + pad(static_cast( + static_cast(date.month())), + 2) + + "-" + + pad(static_cast(static_cast(date.day())), + 2) + + "T" + pad(rest / 3600, 2) + ":" + pad(rest / 60 % 60, 2) + ":" + + pad(rest % 60, 2) + "Z"; +} + +std::string iwork::duration_to_string(const double seconds) { + if (!std::isfinite(seconds) || std::abs(seconds) > max_instant) { + return {}; + } + + const bool negative = seconds < 0; + auto rest = static_cast(std::llround(std::abs(seconds))); + + constexpr std::array, 4> units{ + {{86400, 'd'}, {3600, 'h'}, {60, 'm'}, {1, 's'}}}; + + std::string result; + for (const auto &[size, suffix] : units) { + const std::int64_t count = rest / size; + rest %= size; + if (count == 0) { + continue; + } + if (!result.empty()) { + result += ' '; + } + result += std::to_string(count) + suffix; + } + if (result.empty()) { + result = "0s"; + } + + return negative ? "-" + result : result; +} + +iwork::TableModel iwork::read_table(Package &package, Budget &budget, + const std::uint64_t identifier) { + TableModel result; + + // a drawable we have not mapped is a shape Apple ships and we have not seen, + // so it costs the table it names rather than the file it sits in + const Object &info_object = package.object(identifier); + if (info_object.type != archive_type::table_info) { + return result; + } + const std::optional model_identifier = + reference_identifier(Message(info_object.payload), table_info::model); + if (!model_identifier.has_value()) { + return result; + } + + const Object &model_object = package.object(*model_identifier); + if (model_object.type != archive_type::table_model) { + return result; + } + const Message model(model_object.payload); + + result.name = std::string( + model.bytes_field(table_model::name).value_or(std::string_view())); + result.rows = static_cast( + model.number_field(table_model::rows).value_or(0)); + result.columns = static_cast( + model.number_field(table_model::columns).value_or(0)); + + const std::optional store = + model.bytes_field(table_model::data_store); + if (!store.has_value()) { + return result; + } + const Message data_store(*store); + + const std::unordered_map strings = read_data_list( + package, reference_identifier(data_store, data_store::string_list)); + const std::unordered_map rich_texts = read_data_list( + package, reference_identifier(data_store, data_store::rich_text_list)); + + const std::optional tiles = + data_store.bytes_field(data_store::tiles); + if (!tiles.has_value()) { + return result; + } + const Message tile_storage(*tiles); + const auto rows_per_tile = static_cast( + tile_storage.number_field(tile_storage::rows_per_tile).value_or(0)); + + for (const Field &entry : tile_storage.repeated_field(tile_storage::tiles)) { + if (entry.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed tile list"); + } + const Message tile_entry(entry.bytes); + const auto index = static_cast( + tile_entry.number_field(tile_storage_entry::index).value_or(0)); + const std::optional tile_identifier = + reference_identifier(tile_entry, tile_storage_entry::tile); + if (!tile_identifier.has_value()) { + continue; + } + read_tile(package, budget, *tile_identifier, index * rows_per_tile, strings, + rich_texts, result.cells); + } + + return result; +} + +const iwork::TableModel & +iwork::TableCache::table(Package &package, Budget &budget, + const std::uint64_t identifier) { + const auto it = m_tables.find(identifier); + if (it != m_tables.end()) { + return it->second; + } + return m_tables.emplace(identifier, read_table(package, budget, identifier)) + .first->second; +} + +} // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_table.hpp b/src/odr/internal/iwork/iwork_table.hpp new file mode 100644 index 00000000..782151e5 --- /dev/null +++ b/src/odr/internal/iwork/iwork_table.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace odr::internal::iwork { +class Budget; +class Package; + +/// A `TST.TableModelArchive`: its name, the extent it declares, and the cells +/// that hold something. +struct TableModel final { + /// One cell a tile carries. A position the tile does not carry is empty and + /// has no entry here. + struct Cell final { + std::uint32_t row{}; + std::uint32_t column{}; + ValueType value_type{ValueType::unknown}; + /// The value as text. Empty for a rich text cell, whose paragraphs live in + /// the storage @ref storage_identifier names. + std::string text; + std::optional storage_identifier; + }; + + std::string name; + std::uint32_t rows{}; + std::uint32_t columns{}; + std::vector cells; +}; + +/// Reads the table the `TST.TableInfoArchive` @p identifier names, spending +/// what it decodes against @p budget. A drawable kind we have not mapped comes +/// back as an empty table rather than throwing, as every other per-drawable +/// reader does. +TableModel read_table(Package &package, Budget &budget, + std::uint64_t identifier); + +/// Every table one parse reads, by identifier. A reference list may name one +/// table any number of times, and `Package::object` memoises the archive but +/// not the tiles below it — so a repeat that decoded them again would spend +/// the bytes without building an element the budget could count. +class TableCache final { +public: + const TableModel &table(Package &package, Budget &budget, + std::uint64_t identifier); + +private: + std::unordered_map m_tables; +}; + +/// An IEEE 754 decimal128, in the binary integer form Apple writes, as an +/// exact decimal string. Exposed for its tests. +std::string decimal128_to_string(std::string_view bytes); + +/// @p seconds since 2001-01-01T00:00:00Z as an ISO 8601 instant. Exposed for +/// its tests. +std::string date_to_string(double seconds); + +/// @p seconds as the `1d 2h 3m 4s` form Numbers shows a duration in. Exposed +/// for its tests. +std::string duration_to_string(double seconds); + +} // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_text.cpp b/src/odr/internal/iwork/iwork_text.cpp index 52422668..7ba4a26f 100644 --- a/src/odr/internal/iwork/iwork_text.cpp +++ b/src/odr/internal/iwork/iwork_text.cpp @@ -1,13 +1,16 @@ #include +#include #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -22,10 +25,13 @@ namespace { /// `U+2028 LINE SEPARATOR` — a line break inside a paragraph. constexpr std::string_view line_separator = "\xe2\x80\xa8"; /// `U+FFFC OBJECT REPLACEMENT CHARACTER` — where a drawable is anchored in the -/// text. Nothing reads drawables anchored in a text flow yet, so the anchor is -/// dropped rather than rendered as a glyph. +/// text. The anchor itself is dropped; what it stands for is appended after +/// the paragraph that holds it. constexpr std::string_view object_replacement = "\xef\xbf\xbc"; +void parse_table(const Context &context, ElementIdentifier parent_id, + std::uint64_t identifier); + /// The paragraph mark ends the paragraph it belongs to. Pages writes `\n` and /// Keynote `\r` — the run table is what says where a paragraph starts either /// way, so this only decides whether the mark is part of the text. @@ -61,9 +67,12 @@ std::vector paragraph_starts(const Message &storage) { /// Fills @p paragraph_id with the text of one paragraph, breaking it at the /// line separators it holds. -void parse_paragraph(ElementRegistry ®istry, Budget &budget, +void parse_paragraph(const Context &context, const ElementIdentifier paragraph_id, std::string_view content) { + ElementRegistry ®istry = *context.registry; + Budget &budget = *context.budget; + const auto append_text = [&](const std::string_view part) { std::string text(part); util::string::replace_all(text, std::string(object_replacement), ""); @@ -91,15 +100,132 @@ void parse_paragraph(ElementRegistry ®istry, Budget &budget, append_text(content); } +/// The drawable each `U+FFFC` in a storage anchors there, as byte offsets into +/// @p text. +std::vector> +read_attachments(const Message &storage, const std::string_view text) { + std::vector indices; + std::vector identifiers; + + if (const std::optional table = + storage.bytes_field(text_storage::attachments); + table.has_value()) { + for (const Field &entry : + Message(*table).repeated_field(attribute_table::entries)) { + if (entry.type != WireType::length_delimited) { + throw std::runtime_error("iwork: malformed attachment table"); + } + const Message run(entry.bytes); + const std::optional identifier = + reference_identifier(run, attribute_table_entry::object); + if (!identifier.has_value()) { + continue; + } + indices.push_back( + run.number_field(attribute_table_entry::character_index).value_or(0)); + identifiers.push_back(*identifier); + } + } + + const std::vector offsets = + util::string::utf16_offsets(text, indices); + + std::vector> result; + result.reserve(offsets.size()); + for (std::size_t i = 0; i < offsets.size(); ++i) { + result.emplace_back(offsets[i], identifiers[i]); + } + return result; +} + +/// The drawable an attachment stands for, where it is one we read. +void parse_attachment(const Context &context, const ElementIdentifier parent_id, + const std::uint64_t identifier) { + const Object &attachment = context.package->object(identifier); + if (attachment.type != archive_type::drawable_attachment) { + // a table of contents, a footnote mark, an inline shape — none read yet + return; + } + const std::optional drawable_identifier = reference_identifier( + Message(attachment.payload), attachment_archive::drawable); + if (!drawable_identifier.has_value()) { + return; + } + const Object &drawable = context.package->object(*drawable_identifier); + if (drawable.type != archive_type::table_info) { + return; + } + parse_table(context.deeper(), parent_id, *drawable_identifier); +} + +/// Fills a `Table` element from the tile reader's view of it. +void parse_table(const Context &context, const ElementIdentifier parent_id, + const std::uint64_t identifier) { + ElementRegistry ®istry = *context.registry; + Budget &budget = *context.budget; + const TableModel &model = + context.tables->table(*context.package, *context.budget, identifier); + + budget.spend_element(); + auto [table_id, element, payload] = registry.create_table_element(); + payload.rows = model.rows; + payload.columns = model.columns; + registry.append_child(parent_id, table_id); + + // the tiles are sparse and the rows below are dense, so index them once + // rather than searching the list per position + std::map, const TableModel::Cell *> + by_position; + for (const TableModel::Cell &cell : model.cells) { + by_position.emplace(std::pair(cell.row, cell.column), &cell); + } + + for (std::uint32_t column = 0; column < model.columns; ++column) { + budget.spend_element(); + auto [column_id, column_element] = + registry.create_element(ElementType::table_column); + registry.append_table_column(table_id, column_id); + } + + // a table's rows are dense — the renderer walks them rather than asking by + // coordinate, so a row the tiles do not carry is a row of empty cells. The + // extent is the file's word, so every position is spent against the budget + // rather than trusted to be one an app wrote. + for (std::uint32_t row = 0; row < model.rows; ++row) { + budget.spend_element(); + auto [row_id, row_element] = + registry.create_element(ElementType::table_row); + registry.append_child(table_id, row_id); + + for (std::uint32_t column = 0; column < model.columns; ++column) { + budget.spend_element(); + auto [cell_id, cell_element, cell] = + registry.create_cell_element(ElementType::table_cell); + cell.row = row; + cell.column = column; + registry.append_child(row_id, cell_id); + + const auto it = by_position.find({row, column}); + if (it != by_position.end()) { + cell.value_type = it->second->value_type; + fill_cell(context, cell_id, *it->second); + } + } + } +} + } // namespace } // namespace odr::internal::iwork namespace odr::internal { -void iwork::parse_storage(ElementRegistry ®istry, Budget &budget, +void iwork::parse_storage(const Context &context, const ElementIdentifier parent_id, const Message &storage) { + ElementRegistry ®istry = *context.registry; + Budget &budget = *context.budget; + // the text arrives as a small number of large strings; the run tables index // it as one std::string text; @@ -112,6 +238,8 @@ void iwork::parse_storage(ElementRegistry ®istry, Budget &budget, const std::vector starts = util::string::utf16_offsets(text, paragraph_starts(storage)); + const std::vector> attachments = + read_attachments(storage, text); const std::string_view body_text(text); for (std::size_t i = 0; i < starts.size(); ++i) { @@ -129,8 +257,55 @@ void iwork::parse_storage(ElementRegistry ®istry, Budget &budget, auto [paragraph_id, paragraph] = registry.create_element(ElementType::paragraph); registry.append_child(parent_id, paragraph_id); - parse_paragraph(registry, budget, paragraph_id, content); + parse_paragraph(context, paragraph_id, content); + + // a drawable goes after the paragraph its anchor sits in: the anchor is + // an inline character but a table is not something a paragraph can hold + for (const auto &[offset, identifier] : attachments) { + if (offset >= begin && offset < end) { + parse_attachment(context, parent_id, identifier); + } + } + } +} + +void iwork::fill_cell(const Context &context, const ElementIdentifier cell_id, + const TableModel::Cell &cell) { + ElementRegistry ®istry = *context.registry; + Budget &budget = *context.budget; + + if (cell.storage_identifier.has_value()) { + const Object &payload = context.package->object(*cell.storage_identifier); + if (payload.type != archive_type::rich_text_payload) { + return; + } + const std::optional storage_identifier = + reference_identifier(Message(payload.payload), + rich_text_payload::storage); + if (!storage_identifier.has_value()) { + return; + } + const Object &storage = context.package->object(*storage_identifier); + if (storage.type != archive_type::text_storage) { + return; + } + parse_storage(context.deeper(), cell_id, Message(storage.payload)); + return; } + + if (cell.text.empty()) { + return; + } + budget.spend_element(); + auto [paragraph_id, paragraph] = + registry.create_element(ElementType::paragraph); + registry.append_child(cell_id, paragraph_id); + + budget.spend_element(); + budget.spend_text(cell.text.size()); + auto [text_id, element, text] = registry.create_text_element(); + text.text = cell.text; + registry.append_child(paragraph_id, text_id); } } // namespace odr::internal diff --git a/src/odr/internal/iwork/iwork_text.hpp b/src/odr/internal/iwork/iwork_text.hpp index d8d006c3..b3a5afef 100644 --- a/src/odr/internal/iwork/iwork_text.hpp +++ b/src/odr/internal/iwork/iwork_text.hpp @@ -2,14 +2,47 @@ #include +#include + +#include +#include + namespace odr::internal::iwork { class Budget; class ElementRegistry; class Message; +class Package; + +/// What a storage walk needs to resolve what its text anchors. +struct Context final { + /// How deep a storage may nest before the file is treated as malformed: a + /// table cell holds a storage, which may hold a table again. + static constexpr std::uint32_t max_depth = 8; + + ElementRegistry *registry{nullptr}; + Package *package{nullptr}; + Budget *budget{nullptr}; + TableCache *tables{nullptr}; + std::uint32_t depth{}; + + [[nodiscard]] Context deeper() const { + if (depth + 1 >= max_depth) { + throw std::runtime_error("iwork: storages nest too deeply"); + } + return {registry, package, budget, tables, depth + 1}; + } +}; + +/// Appends the paragraphs of a `TSWP.StorageArchive` to @p parent_id, and the +/// drawables its text anchors after the paragraph that holds the anchor. +/// Shared by every place text lives: a Pages body, a Keynote text box, a table +/// cell. +void parse_storage(const Context &context, ElementIdentifier parent_id, + const Message &storage); -/// Appends the paragraphs of a `TSWP.StorageArchive` to @p parent_id. Shared -/// by every place text lives: a Pages body and a Keynote text box. -void parse_storage(ElementRegistry ®istry, Budget &budget, - ElementIdentifier parent_id, const Message &storage); +/// Fills @p cell_id with what the tile carried: a value as text, or the +/// storage a rich text cell holds. +void fill_cell(const Context &context, ElementIdentifier cell_id, + const TableModel::Cell &cell); } // namespace odr::internal::iwork diff --git a/src/odr/internal/iwork/iwork_types.hpp b/src/odr/internal/iwork/iwork_types.hpp index ec8c2541..f06e337f 100644 --- a/src/odr/internal/iwork/iwork_types.hpp +++ b/src/odr/internal/iwork/iwork_types.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include namespace odr::internal::iwork { @@ -44,6 +45,28 @@ constexpr std::uint32_t text_storage = 2001; /// `TSWP.ShapeArchive`, a text box: a drawable plus the storage it holds. /// `style-various-1.key Index/Slide-32281.iwa` object 32369 (iWork 14.4). constexpr std::uint32_t text_shape = 2011; +/// `TSWP.DrawableAttachmentArchive`, what a `U+FFFC` in a text flow points at. +/// `style-various-1.pages Index/Document.iwa` object 1732925 (iWork 13.2). +constexpr std::uint32_t drawable_attachment = 2003; +/// `TN.SheetArchive`, one sheet of a `.numbers` package. Numbered the same as +/// @ref keynote_show — see the note above. +/// `style-various-1.numbers Index/Document.iwa` object 904475 (iWork 14.4). +constexpr std::uint32_t numbers_sheet = 2; +/// `TST.TableInfoArchive`, the drawable a table is placed by. +/// `style-various-1.pages Index/Document.iwa` object 1732842 (iWork 13.2). +constexpr std::uint32_t table_info = 6000; +/// `TST.TableModelArchive`, a table's extent, name and cell storage. +/// `style-various-1.pages Index/Document.iwa` object 1732845 (iWork 13.2). +constexpr std::uint32_t table_model = 6001; +/// `TST.Tile`, a range of rows holding packed cell records. +/// `style-various-1.pages Index/Tables/Tile.iwa` object 1732818 (iWork 13.2). +constexpr std::uint32_t tile = 6002; +/// `TST.DataList`, a side table cells reference by key. +/// `style-various-1.numbers Index/Tables/DataList-904489-2.iwa` (iWork 14.4). +constexpr std::uint32_t data_list = 6005; +/// `TST.RichTextPayloadArchive`, one entry of a rich text data list. +/// `style-various-1.pages Index/Document.iwa` object 1732965 (iWork 13.2). +constexpr std::uint32_t rich_text_payload = 6218; } // namespace archive_type namespace document_archive { @@ -51,8 +74,18 @@ namespace document_archive { constexpr std::uint32_t body_storage = 4; /// The `KN.ShowArchive`, as a `TSP.Reference`. `KN.DocumentArchive` only. constexpr std::uint32_t show = 2; +/// One `TN.SheetArchive`, as a `TSP.Reference`; repeated, in tab order. +/// `TN.DocumentArchive` only. +constexpr std::uint32_t sheets = 1; } // namespace document_archive +namespace sheet_archive { +constexpr std::uint32_t name = 1; +/// One drawable on the sheet, as a `TSP.Reference`; repeated. A `.numbers` +/// sheet holds its tables here, one per table. +constexpr std::uint32_t drawables = 2; +} // namespace sheet_archive + namespace show_archive { /// The slide tree, holding the deck's slides in presentation order. constexpr std::uint32_t slide_tree = 3; @@ -110,12 +143,118 @@ constexpr std::uint32_t x = 1; constexpr std::uint32_t y = 2; } // namespace point +/// `TST.TableInfoArchive` — where a table sits, and the model behind it. +namespace table_info { +constexpr std::uint32_t model = 2; +} // namespace table_info + +/// `TST.TableModelArchive` — a table's extent, its name and its cell storage. +namespace table_model { +constexpr std::uint32_t data_store = 4; +constexpr std::uint32_t rows = 6; +constexpr std::uint32_t columns = 7; +constexpr std::uint32_t name = 8; +} // namespace table_model + +/// `TST.DataStore` — the tiles a table's cells live in, and the side lists +/// they reference by key. The other lists it names are formats, formulas and +/// styles, none of which is read. +namespace data_store { +constexpr std::uint32_t tiles = 3; +constexpr std::uint32_t string_list = 4; +constexpr std::uint32_t rich_text_list = 17; +} // namespace data_store + +namespace tile_storage { +/// One `{index, tile}` pair; repeated. +constexpr std::uint32_t tiles = 1; +/// How many rows a tile covers, so a tile's row index is relative to it. +constexpr std::uint32_t rows_per_tile = 2; +} // namespace tile_storage + +namespace tile_storage_entry { +constexpr std::uint32_t index = 1; +constexpr std::uint32_t tile = 2; +} // namespace tile_storage_entry + +namespace tile { +/// One `TST.TileRowInfo`; repeated, only for rows that hold something. +constexpr std::uint32_t rows = 5; +} // namespace tile + +/// `TST.TileRowInfo` — one row of a tile. Fields 3 and 4 carry an older +/// encoding of the same cells, which @ref cell::version says not to read. +namespace tile_row { +constexpr std::uint32_t index = 1; +constexpr std::uint32_t storage = 6; +/// `std::int16_t` per column, the offset of that column's cell into +/// @ref storage, or `-1` where the row holds no cell there. +constexpr std::uint32_t offsets = 7; +} // namespace tile_row + +namespace data_list { +constexpr std::uint32_t entries = 3; +} // namespace data_list + +namespace data_list_entry { +constexpr std::uint32_t key = 1; +constexpr std::uint32_t string = 3; +constexpr std::uint32_t rich_text = 9; +} // namespace data_list_entry + +namespace rich_text_payload { +constexpr std::uint32_t storage = 1; +} // namespace rich_text_payload + +/// One packed cell record inside a tile row's storage. +/// +/// The layout is a twelve-byte header — a version byte, a type byte, six bytes +/// nothing here reads, then a little-endian `std::uint32_t` of flags — followed +/// by the optional fields the flags name, in ascending bit order. Read off the +/// cells of `style-various-1.pages` (iWork 13.2) and +/// `style-various-1.numbers` (iWork 14.4), which agree. +namespace cell { +/// The only encoding both fixtures write. A record that declares another is +/// one we have not mapped, and reads as an empty cell rather than a wrong one. +constexpr std::uint8_t version = 5; +constexpr std::size_t header_size = 12; +constexpr std::size_t version_offset = 0; +constexpr std::size_t type_offset = 1; +constexpr std::size_t flags_offset = 8; + +/// A cell's type byte. +namespace type { +constexpr std::uint8_t number = 2; ///< a decimal128 in @ref flag::decimal +constexpr std::uint8_t string = 3; ///< a key into the string list +constexpr std::uint8_t date = 5; ///< seconds since 2001-01-01T00:00:00Z +constexpr std::uint8_t boolean = 6; ///< `1.0` or `0.0` +constexpr std::uint8_t duration = 7; ///< a count of seconds +constexpr std::uint8_t rich_text = 9; ///< a key into the rich text list +} // namespace type + +/// The flag bits that name a value, with the width each one occupies. The +/// higher bits name styles and formats, which nothing reads — a value is +/// always in one of these five, so the walk stops after the last of them. +namespace flag { +constexpr std::uint32_t decimal = 1U << 0; ///< 16 bytes, IEEE decimal128 +constexpr std::uint32_t number = 1U << 1; ///< 8 bytes, IEEE double +constexpr std::uint32_t seconds = 1U << 2; ///< 8 bytes, IEEE double +constexpr std::uint32_t string_key = 1U << 3; ///< 4 bytes +constexpr std::uint32_t rich_text_key = 1U << 4; ///< 4 bytes +} // namespace flag +} // namespace cell + namespace text_storage { /// The text, in a small number of large strings. constexpr std::uint32_t text = 3; /// The paragraph style run table: one entry per paragraph, holding the /// character index the paragraph starts at and, where it has one, its style. constexpr std::uint32_t paragraph_styles = 5; +/// The attachment run table: one entry per `U+FFFC` in the text, naming the +/// drawable anchored there. +/// `style-various-1.pages` body storage, indices 18, 460 and 469 against the +/// three anchors the text carries (iWork 13.2). +constexpr std::uint32_t attachments = 9; } // namespace text_storage /// A run table parallel to the text, as `TSWP.ObjectAttributeTable`. @@ -129,6 +268,13 @@ constexpr std::uint32_t identifier = 1; namespace attribute_table_entry { constexpr std::uint32_t character_index = 1; +/// The object the entry attaches there, as a `TSP.Reference`. +constexpr std::uint32_t object = 2; } // namespace attribute_table_entry +/// `TSWP.DrawableAttachmentArchive` — the drawable a text anchor stands for. +namespace attachment_archive { +constexpr std::uint32_t drawable = 1; +} // namespace attachment_archive + } // namespace odr::internal::iwork diff --git a/src/odr/internal/open_strategy.cpp b/src/odr/internal/open_strategy.cpp index 83690234..5da915ee 100644 --- a/src/odr/internal/open_strategy.cpp +++ b/src/odr/internal/open_strategy.cpp @@ -94,7 +94,8 @@ open_file_as(const std::shared_ptr &file, const FileType as, throw NoOpenDocumentFile(); } - if (as == FileType::iwork_pages || as == FileType::iwork_keynote) { + if (as == FileType::iwork_pages || as == FileType::iwork_keynote || + as == FileType::iwork_numbers) { ODR_VERBOSE(logger, "open as iwork"); try { auto zip_file = std::make_unique(file); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dd1cff01..542ebdd8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -65,6 +65,7 @@ add_executable(odr_test "src/internal/iwork/iwork_protobuf_test.cpp" "src/internal/iwork/iwork_snappy_test.cpp" "src/internal/iwork/keynote_test.cpp" + "src/internal/iwork/numbers_test.cpp" "src/internal/iwork/pages_test.cpp" "src/internal/rtf/rtf_document_test.cpp" diff --git a/test/data.cmake b/test/data.cmake index b4b9df26..dc08524d 100644 --- a/test/data.cmake +++ b/test/data.cmake @@ -17,7 +17,7 @@ odr_test_data( odr_test_data( PATH "reference-output/odr-public" URL "https://github.com/opendocument-app/OpenDocument.test.output.git" - REVISION "7c77e0ecb4878185854d397328eba9c120372fda") + REVISION "7dd2d881f7d832d119ab1369d8725c1bbdb9819e") odr_test_data( PATH "reference-output/odr-private" diff --git a/test/src/internal/iwork/iwork_element_util.hpp b/test/src/internal/iwork/iwork_element_util.hpp new file mode 100644 index 00000000..37f62cbd --- /dev/null +++ b/test/src/internal/iwork/iwork_element_util.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include + +#include + +/// Reading an iWork document back out, for the tests that assert what one +/// decodes to. The sibling `iwork_test_util.hpp` assembles the input; this +/// reads the output. +namespace odr::test::iwork { + +/// The text of one element, a line break reading as a newline and a paragraph +/// boundary as well. The engine emits a `line_break` only where a storage +/// carried a line separator inside a paragraph, so both really are newlines. +inline std::string element_text(const Element element) { + std::string result; + for (const Element paragraph : element.children()) { + if (!result.empty()) { + result += '\n'; + } + for (const Element child : paragraph.children()) { + if (child.type() == ElementType::line_break) { + result += '\n'; + } else { + result += child.as_text().content(); + } + } + } + return result; +} + +} // namespace odr::test::iwork diff --git a/test/src/internal/iwork/iwork_test_util.hpp b/test/src/internal/iwork/iwork_test_util.hpp index a9c5e810..a3bb0549 100644 --- a/test/src/internal/iwork/iwork_test_util.hpp +++ b/test/src/internal/iwork/iwork_test_util.hpp @@ -324,4 +324,244 @@ inline std::shared_ptr pages_package( return package({{"Document", document}}); } +/// One packed cell record: the twelve-byte header, then the value the flags +/// name. @p version and @p type are taken as given so a test can state a +/// record the reader has not mapped. +inline std::string +cell_record(const std::uint8_t type, const std::uint32_t flags, + const std::string &value, + const std::uint8_t version = types::cell::version) { + std::string result(types::cell::header_size, '\0'); + result[types::cell::version_offset] = static_cast(version); + result[types::cell::type_offset] = static_cast(type); + for (std::size_t i = 0; i < 4; ++i) { + result[types::cell::flags_offset + i] = + static_cast((flags >> (i * 8)) & 0xff); + } + return result + value; +} + +/// A cell holding a `double` in the field @p flag names. +inline std::string number_cell(const std::uint8_t type, + const std::uint32_t flag, const double value) { + const auto bits = std::bit_cast(value); + std::string bytes; + for (std::uint32_t shift = 0; shift < 64; shift += 8) { + bytes.push_back(static_cast((bits >> shift) & 0xff)); + } + return cell_record(type, flag, bytes); +} + +/// One row of a tile: @p cells packed back to back, and the offsets table that +/// says where each column's record starts. An empty entry writes `-1`, the +/// column the row holds no cell for. +inline std::string tile_row(const std::uint32_t index, + const std::vector &cells) { + std::string storage; + std::string offsets; + for (const std::string &cell : cells) { + const auto offset = cell.empty() + ? static_cast(-1) + : static_cast(storage.size()); + const auto bits = static_cast(offset); + offsets.push_back(static_cast(bits & 0xff)); + offsets.push_back(static_cast((bits >> 8) & 0xff)); + storage += cell; + } + + return number_field(types::tile_row::index, index) + + message_field(types::tile_row::storage, storage) + + message_field(types::tile_row::offsets, offsets); +} + +/// One row of a tile, stated as the bytes it carries rather than as its cells, +/// so a test can write an offsets table the storage does not back. +inline std::string tile_row_bytes(const std::uint32_t index, + const std::string &storage, + const std::vector &offsets) { + std::string table; + for (const std::int16_t offset : offsets) { + const auto bits = static_cast(offset); + table.push_back(static_cast(bits & 0xff)); + table.push_back(static_cast((bits >> 8) & 0xff)); + } + + return number_field(types::tile_row::index, index) + + message_field(types::tile_row::storage, storage) + + message_field(types::tile_row::offsets, table); +} + +/// A `TST.Tile` holding @p rows, each already built by @ref tile_row. +inline std::string tile(const std::vector &rows) { + std::string result; + for (const std::string &row : rows) { + result += message_field(types::tile::rows, row); + } + return result; +} + +/// One entry of a `TST.DataList`: a key and the string it stands for. +inline std::string string_entry(const std::uint64_t key, + const std::string &text) { + return message_field(types::data_list::entries, + number_field(types::data_list_entry::key, key) + + message_field(types::data_list_entry::string, text)); +} + +/// What a synthetic table is made of. The extent is what the model *declares*, +/// which the tiles need not fill — that is the shape the budget guards. +struct TableSpec final { + std::string name{}; + std::uint32_t rows{}; + std::uint32_t columns{}; + /// Rows already built by @ref tile_row, all in one tile. + std::vector tile_rows{}; + /// Entries already built by @ref string_entry. + std::vector strings{}; + /// How many times the tile list names the one tile. + std::size_t tile_repeats{1}; + /// When set, a rich text list holding one entry under key 1, whose payload + /// names this storage — which a `rich_text` cell reaches by that key. + std::optional rich_text_storage{}; + /// When set, the tile's payload verbatim, in place of @ref tile_rows. + std::optional raw_tile{}; + /// When set, the tile storage's tile list verbatim, in place of the entries + /// @ref tile_repeats would write. + std::optional raw_tile_list{}; +}; + +/// The identifiers a synthetic table's objects are filed under. +constexpr std::uint64_t table_info_identifier = 20; +constexpr std::uint64_t table_model_identifier = 21; +constexpr std::uint64_t table_tile_identifier = 22; +constexpr std::uint64_t table_strings_identifier = 23; +constexpr std::uint64_t table_rich_texts_identifier = 25; +constexpr std::uint64_t rich_text_payload_identifier = 26; + +/// The objects a table is made of: a `TST.TableInfoArchive`, the model behind +/// it, its tile and its string list. +inline std::string table_objects(const TableSpec &spec) { + std::string entries; + for (const std::string &entry : spec.strings) { + entries += entry; + } + + std::string tiles = spec.raw_tile_list.value_or(std::string()); + for (std::size_t repeat = 0; + !spec.raw_tile_list.has_value() && repeat < spec.tile_repeats; + ++repeat) { + tiles += message_field(types::tile_storage::tiles, + number_field(types::tile_storage_entry::index, 0) + + reference_field(types::tile_storage_entry::tile, + table_tile_identifier)); + } + + std::string store = + message_field(types::data_store::tiles, tiles) + + reference_field(types::data_store::string_list, table_strings_identifier); + std::string rich_text_objects; + if (spec.rich_text_storage.has_value()) { + store += reference_field(types::data_store::rich_text_list, + table_rich_texts_identifier); + + const std::string payload = reference_field( + types::rich_text_payload::storage, *spec.rich_text_storage); + const std::string list = + message_field(types::data_list::entries, + number_field(types::data_list_entry::key, 1) + + reference_field(types::data_list_entry::rich_text, + rich_text_payload_identifier)); + + rich_text_objects = + object(table_rich_texts_identifier, + {{types::archive_type::data_list, list.size()}}, list) + + object(rich_text_payload_identifier, + {{types::archive_type::rich_text_payload, payload.size()}}, + payload); + } + + const std::string model = + message_field(types::table_model::data_store, store) + + number_field(types::table_model::rows, spec.rows) + + number_field(types::table_model::columns, spec.columns) + + message_field(types::table_model::name, spec.name); + const std::string info = + reference_field(types::table_info::model, table_model_identifier); + const std::string tile_payload = spec.raw_tile.value_or(tile(spec.tile_rows)); + + return object(table_info_identifier, + {{types::archive_type::table_info, info.size()}}, info) + + object(table_model_identifier, + {{types::archive_type::table_model, model.size()}}, model) + + object(table_tile_identifier, + {{types::archive_type::tile, tile_payload.size()}}, + tile_payload) + + object(table_strings_identifier, + {{types::archive_type::data_list, entries.size()}}, entries) + + rich_text_objects; +} + +/// The identifier a synthetic attachment is filed under. +constexpr std::uint64_t attachment_identifier = 24; + +/// A `.pages` package whose body anchors @p spec as a table: the body text +/// carries a `U+FFFC` at @p anchor_index, the attachment run table names the +/// `TSWP.DrawableAttachmentArchive` there, and that names the table. +inline std::shared_ptr +pages_table_package(const TableSpec &spec, + const std::string &text = "\xef\xbf\xbc", + const std::uint64_t anchor_index = 0) { + const std::string storage = + message_field(types::text_storage::text, text) + + message_field( + types::text_storage::attachments, + message_field( + types::attribute_table::entries, + number_field(types::attribute_table_entry::character_index, + anchor_index) + + reference_field(types::attribute_table_entry::object, + attachment_identifier))); + + const std::string attachment = reference_field( + types::attachment_archive::drawable, table_info_identifier); + + const std::string root = document_archive(body_identifier); + const std::string document = + object(1, {{types::archive_type::pages_document, root.size()}}, root) + + object(body_identifier, + {{types::archive_type::text_storage, storage.size()}}, storage) + + object(attachment_identifier, + {{types::archive_type::drawable_attachment, attachment.size()}}, + attachment) + + table_objects(spec); + + return package({{"Document", document}}); +} + +/// A `.numbers` package: a root archive naming one sheet, which names @p spec +/// as its only drawable. @p table_repeats is how many times the sheet's +/// drawable list names that one table. +inline std::shared_ptr +numbers_package(const TableSpec &spec, const std::string &sheet_name = "Sheet", + const std::size_t table_repeats = 1) { + constexpr std::uint64_t sheet_identifier = 2; + + const std::string root = + reference_field(types::document_archive::sheets, sheet_identifier); + + std::string sheet = message_field(types::sheet_archive::name, sheet_name); + for (std::size_t repeat = 0; repeat < table_repeats; ++repeat) { + sheet += + reference_field(types::sheet_archive::drawables, table_info_identifier); + } + + const std::string document = + object(1, {{types::archive_type::app_document, root.size()}}, root) + + object(sheet_identifier, + {{types::archive_type::numbers_sheet, sheet.size()}}, sheet) + + table_objects(spec); + + return package({{"Document", document}}); +} + } // namespace odr::test::iwork diff --git a/test/src/internal/iwork/keynote_test.cpp b/test/src/internal/iwork/keynote_test.cpp index 7c2a2668..3fe252f2 100644 --- a/test/src/internal/iwork/keynote_test.cpp +++ b/test/src/internal/iwork/keynote_test.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -31,24 +32,13 @@ namespace iwork = odr::internal::iwork; namespace { -/// The text of one frame, a line break reading as a newline and a paragraph -/// boundary as well. +/// The text of one frame. Every child of a frame is a paragraph, which the +/// shared reader does not assert, so state it here. std::string frame_text(const Element frame) { - std::string result; for (const Element paragraph : frame.children()) { EXPECT_EQ(paragraph.type(), ElementType::paragraph); - if (!result.empty()) { - result += '\n'; - } - for (const Element child : paragraph.children()) { - if (child.type() == ElementType::line_break) { - result += '\n'; - } else { - result += child.as_text().content(); - } - } } - return result; + return builder::element_text(frame); } /// The text of every frame of every slide, one vector per slide. @@ -269,13 +259,14 @@ TEST(IworkKeynote, a_deck_without_slides_has_an_empty_root) { } // A package whose root archive is type 1 is a Keynote one only when it holds -// slide components — Numbers numbers its root archive the same. -TEST(IworkKeynote, a_root_archive_without_slide_components_is_not_keynote) { +// slide components — Numbers numbers its root archive the same, so one +// without them reads as a `.numbers`. +TEST(IworkKeynote, a_root_archive_without_slide_components_is_numbers) { const auto files = builder::pages_package(builder::text_storage("", std::nullopt), builder::types::archive_type::app_document); - EXPECT_THROW(iwork::IworkFile{files}, NoIworkFile); + EXPECT_EQ(iwork::IworkFile{files}.file_type(), FileType::iwork_numbers); } // A drawable list may name the same object any number of times, and every diff --git a/test/src/internal/iwork/numbers_test.cpp b/test/src/internal/iwork/numbers_test.cpp new file mode 100644 index 00000000..e7f0107a --- /dev/null +++ b/test/src/internal/iwork/numbers_test.cpp @@ -0,0 +1,402 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +using namespace odr; +using odr::test::TestData; +namespace builder = odr::test::iwork; +namespace iwork = odr::internal::iwork; + +namespace { + +/// A sheet's content extent, row by row. +std::vector> grid(const Sheet sheet) { + const TableDimensions content = sheet.content({}); + + std::vector> result; + for (std::uint32_t row = 0; row < content.rows; ++row) { + std::vector cells; + for (std::uint32_t column = 0; column < content.columns; ++column) { + cells.push_back(builder::element_text(sheet.cell(column, row))); + } + result.push_back(std::move(cells)); + } + return result; +} + +std::vector sheet_names(const Element root) { + std::vector result; + for (const Element sheet : root.children()) { + EXPECT_EQ(sheet.type(), ElementType::sheet); + result.push_back(sheet.as_sheet().name()); + } + return result; +} + +/// The document a synthetic table decodes to. +Document numbers_document(const builder::TableSpec &spec, + const std::string &sheet_name = "Sheet", + const std::size_t table_repeats = 1) { + return Document(std::make_shared( + FileType::iwork_numbers, + builder::numbers_package(spec, sheet_name, table_repeats))); +} + +Sheet sheet_at(const Element root, const std::size_t index) { + std::size_t i = 0; + for (const Element sheet : root.children()) { + if (i++ == index) { + return sheet.as_sheet(); + } + } + return {}; +} + +} // namespace + +TEST(IworkNumbers, is_detected_by_content) { + const Logger logger = Logger::create_stdio("odr-test", LogLevel::verbose); + const std::string path = + TestData::test_file_path("odr-public/numbers/style-various-1.numbers"); + + EXPECT_THAT(list_file_types(path, logger), + testing::Contains(FileType::iwork_numbers)); + + const DecodedFile file(path, logger); + EXPECT_EQ(file.file_type(), FileType::iwork_numbers); + EXPECT_EQ(file.file_category(), FileCategory::document); + EXPECT_EQ(file.as_document_file().document_type(), DocumentType::spreadsheet); +} + +// The blank template's one sheet holds one table with nothing in it, which +// must come back as a sheet of its declared extent and no content. +TEST(IworkNumbers, empty) { + const DocumentFile document_file( + TestData::test_file_path("odr-public/numbers/empty.numbers"), + Logger::null()); + EXPECT_EQ(document_file.file_type(), FileType::iwork_numbers); + + const Document document = document_file.document(); + EXPECT_EQ(document.document_type(), DocumentType::spreadsheet); + EXPECT_FALSE(document.is_editable()); + + const Element root = document.root_element(); + EXPECT_EQ(sheet_names(root), (std::vector{"Sheet 1 – Table 1"})); + + const Sheet sheet = sheet_at(root, 0); + EXPECT_EQ(sheet.dimensions().rows, 22); + EXPECT_EQ(sheet.dimensions().columns, 7); + EXPECT_EQ(sheet.content({}).rows, 0); + EXPECT_EQ(sheet.content({}).columns, 0); +} + +// A Numbers sheet holds many tables and our `Sheet` is one grid, so each +// table is a sheet of its own rather than only the first one surviving. +TEST(IworkNumbers, one_sheet_per_table) { + const DocumentFile document_file( + TestData::test_file_path("odr-public/numbers/style-various-1.numbers"), + Logger::null()); + const Document document = document_file.document(); + + EXPECT_EQ(sheet_names(document.root_element()), + (std::vector{"Sales – Quarterly", "Sales – Wide", + "Types – Values"})); +} + +TEST(IworkNumbers, cell_values) { + const DocumentFile document_file( + TestData::test_file_path("odr-public/numbers/style-various-1.numbers"), + Logger::null()); + const Document document = document_file.document(); + + EXPECT_EQ(grid(sheet_at(document.root_element(), 0)), + (std::vector>{ + {"Quarter", "Revenue", "Growth"}, + {"Q1", "1000", "0.125"}, + {"Q2", "1250.5", "0.25"}, + // the cached value of `=SUM(B2:B3)` and `=AVERAGE(C2:C3)`; + // `CalculationEngine.iwa` is not read + {"Total", "2250.5", "0.1875"}, + })); +} + +// Rows and columns cannot be confused: this table is three rows of six +// columns, and holds cells only at its corners. +TEST(IworkNumbers, a_table_wider_than_it_is_tall) { + const DocumentFile document_file( + TestData::test_file_path("odr-public/numbers/style-various-1.numbers"), + Logger::null()); + const Document document = document_file.document(); + + const Sheet sheet = sheet_at(document.root_element(), 1); + EXPECT_EQ(sheet.dimensions().rows, 3); + EXPECT_EQ(sheet.dimensions().columns, 6); + + EXPECT_EQ(grid(sheet), (std::vector>{ + {"a", "", "", "", "", "f"}, + {"", "", "", "", "", ""}, + {"bottom left", "", "", "", "", "bottom right"}, + })); +} + +TEST(IworkNumbers, every_cell_type_the_fixtures_hold) { + const DocumentFile document_file( + TestData::test_file_path("odr-public/numbers/style-various-1.numbers"), + Logger::null()); + const Document document = document_file.document(); + + EXPECT_EQ(grid(sheet_at(document.root_element(), 2)), + (std::vector>{ + {"Kind", "Value"}, + {"text", "A cell holding a rather longer piece of text"}, + {"number", "-42"}, + {"boolean", "TRUE"}, + {"date", "2024-01-01T10:30:00Z"}, + {"duration", "1h 30m"}, + // a row whose second cell the tile does not carry at all + {"empty", ""}, + // Numbers shows 7.5%; the number format is not applied + {"percent", "0.075"}, + {"two lines", "line one\nline two"}, + })); +} + +// A number cell is right-aligned by the renderer; nothing else is. +TEST(IworkNumbers, only_a_number_reports_a_float_value_type) { + const DocumentFile document_file( + TestData::test_file_path("odr-public/numbers/style-various-1.numbers"), + Logger::null()); + const Document document = document_file.document(); + const Sheet sheet = sheet_at(document.root_element(), 2); + + EXPECT_EQ(sheet.cell(1, 2).value_type(), ValueType::float_number); + EXPECT_EQ(sheet.cell(1, 1).value_type(), ValueType::string); + EXPECT_EQ(sheet.cell(1, 4).value_type(), ValueType::string); + // a position the tile carries no cell for + EXPECT_EQ(sheet.cell(1, 6).value_type(), ValueType::unknown); +} + +// A spreadsheet value is a decimal by construction, which is why Apple stores +// one; going through a `double` would put back the rounding it avoids. +TEST(IworkDecimal128, reads_the_values_the_fixtures_hold) { + const auto decimal = [](const std::string &hex) { + std::string bytes; + for (std::size_t i = 0; i < hex.size(); i += 2) { + bytes.push_back( + static_cast(std::stoul(hex.substr(i, 2), nullptr, 16))); + } + return iwork::decimal128_to_string(bytes); + }; + + EXPECT_EQ(decimal("e8030000000000000000000000004030"), "1000"); + EXPECT_EQ(decimal("7d000000000000000000000000003a30"), "0.125"); + EXPECT_EQ(decimal("d9300000000000000000000000003e30"), "1250.5"); + EXPECT_EQ(decimal("2a000000000000000000000000004030"), "42"); + EXPECT_EQ(decimal("2a0000000000000000000000000040b0"), "-42"); + // Numbers stores 0.075 as 750000000000000e-16 + EXPECT_EQ(decimal("00e094fb1eaa02000000000000002030"), "0.075"); + EXPECT_EQ(decimal("00000000000000000000000000004030"), "0"); +} + +// IEEE 754 leaves a coefficient above 10^34 non-canonical and reads it as +// zero, and the same combination field is how an infinity or a NaN is written +// — none of which a cell should render as a number. +TEST(IworkDecimal128, a_non_canonical_combination_field_reads_as_zero) { + const auto decimal = [](const std::string &hex) { + std::string bytes; + for (std::size_t i = 0; i < hex.size(); i += 2) { + bytes.push_back( + static_cast(std::stoul(hex.substr(i, 2), nullptr, 16))); + } + return iwork::decimal128_to_string(bytes); + }; + + // +infinity: the top bits of the last byte select the combination field + EXPECT_EQ(decimal("00000000000000000000000000000078"), "0"); + // a coefficient too large to be canonical + EXPECT_EQ(decimal("ffffffffffffffffffffffffffffff60"), "0"); +} + +TEST(IworkDecimal128, is_cut_off) { + EXPECT_ANY_THROW(std::ignore = iwork::decimal128_to_string("short")); +} + +// The instant stored is UTC, whatever the clock said when it was typed. +TEST(IworkDate, reads_seconds_since_2001) { + EXPECT_EQ(iwork::date_to_string(0), "2001-01-01T00:00:00Z"); + EXPECT_EQ(iwork::date_to_string(725797800), "2024-01-01T10:30:00Z"); + EXPECT_EQ(iwork::date_to_string(-1), "2000-12-31T23:59:59Z"); +} + +// A cell carries its seconds as a raw double, so the value is the file's word: +// one no calendar can name is read as no date rather than cast into undefined +// behaviour. +TEST(IworkDate, a_value_no_calendar_can_name_is_no_date) { + EXPECT_EQ(iwork::date_to_string(std::numeric_limits::quiet_NaN()), + ""); + EXPECT_EQ(iwork::date_to_string(std::numeric_limits::infinity()), ""); + EXPECT_EQ(iwork::date_to_string(1e300), ""); + EXPECT_EQ(iwork::date_to_string(-1e300), ""); +} + +TEST(IworkDuration, reads_a_count_of_seconds) { + EXPECT_EQ(iwork::duration_to_string(5400), "1h 30m"); + EXPECT_EQ(iwork::duration_to_string(0), "0s"); + EXPECT_EQ(iwork::duration_to_string(90061), "1d 1h 1m 1s"); + EXPECT_EQ(iwork::duration_to_string(-60), "-1m"); +} + +TEST(IworkDuration, a_value_no_count_can_hold_is_no_duration) { + EXPECT_EQ(iwork::duration_to_string(std::numeric_limits::quiet_NaN()), + ""); + EXPECT_EQ(iwork::duration_to_string(1e300), ""); +} + +// The decision stage 6 rests on: a record declaring an encoding we have not +// mapped reads as an **empty** cell rather than a wrong one. Both bytes that +// can say so — the version and the type — take the same way out, and the +// control is the same record with the version this reader was written against. +TEST(IworkNumbers, a_record_we_have_not_mapped_is_an_empty_cell) { + constexpr std::uint32_t decimal = iwork::cell::flag::decimal; + // decimal128 `42` + const std::string value("\x2a\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x40\x30", + 16); + + const auto document_of = [&](const std::string &record) { + return numbers_document({.rows = 1, + .columns = 1, + .tile_rows = {builder::tile_row(0, {record})}}); + }; + // a cell points into the document it came from, so each one outlives its use + const Document mapped = document_of( + builder::cell_record(iwork::cell::type::number, decimal, value)); + const Document other_version = document_of(builder::cell_record( + iwork::cell::type::number, decimal, value, iwork::cell::version + 1)); + const Document other_type = + document_of(builder::cell_record(200, decimal, value)); + + const auto cell = [](const Document &document) { + return sheet_at(document.root_element(), 0).cell(0, 0); + }; + + EXPECT_EQ(builder::element_text(cell(mapped)), "42"); + + // a version byte Apple has not shipped yet + EXPECT_EQ(cell(other_version).as_sheet_cell().value_type(), + ValueType::unknown); + EXPECT_EQ(builder::element_text(cell(other_version)), ""); + + // a type byte we have no reader for + EXPECT_EQ(cell(other_type).as_sheet_cell().value_type(), ValueType::unknown); + EXPECT_EQ(builder::element_text(cell(other_type)), ""); +} + +// The framing below a cell is the file's word about its own bytes, so a row +// that contradicts itself is a broken file rather than a shape we have not +// seen — the one place the tile reader throws instead of skipping. +TEST(IworkNumbers, a_row_that_contradicts_its_own_offsets_throws) { + const std::string record = + builder::cell_record(iwork::cell::type::number, 0, ""); + + const auto read = [](const std::string &row) { + std::ignore = + numbers_document({.rows = 1, .columns = 2, .tile_rows = {row}}); + }; + + EXPECT_THAT([&] { read(builder::tile_row_bytes(0, record, {0, 400})); }, + testing::ThrowsMessage( + testing::HasSubstr("cell runs past its row storage"))); + + EXPECT_THAT( + [&] { + read(builder::tile_row_bytes( + 0, record + record, {static_cast(record.size()), 0})); + }, + testing::ThrowsMessage( + testing::HasSubstr("cell offsets are out of order"))); +} + +// A field the reader expects to be a message, written as a varint instead, is +// framing that contradicts itself rather than a shape we have not seen. +TEST(IworkNumbers, a_field_that_is_not_the_message_it_should_be_throws) { + const auto throws = [](const std::string &message) { + return testing::ThrowsMessage( + testing::HasSubstr(message)); + }; + + builder::TableSpec spec{.rows = 1, .columns = 1}; + spec.tile_rows = {}; + + // a tile whose row list holds a varint + EXPECT_THAT( + [&] { + builder::TableSpec malformed = spec; + malformed.raw_tile = builder::number_field(iwork::tile::rows, 1); + std::ignore = numbers_document(malformed); + }, + throws("malformed tile row")); + + // a tile storage whose tile list holds a varint + EXPECT_THAT( + [&] { + builder::TableSpec malformed = spec; + malformed.raw_tile_list = + builder::number_field(iwork::tile_storage::tiles, 1); + std::ignore = numbers_document(malformed); + }, + throws("malformed tile list")); + + // a data list whose entry list holds a varint + EXPECT_THAT( + [&] { + builder::TableSpec malformed = spec; + malformed.strings = { + builder::number_field(iwork::data_list::entries, 1)}; + std::ignore = numbers_document(malformed); + }, + throws("malformed data list")); +} + +// A tile list may name one tile any number of times, and `Package::object` +// hands every repeat back from its cache — so the cells a repeat appends are +// spent as they are produced rather than once the model is complete. +TEST(IworkNumbers, a_repeated_tile_is_capped_by_the_cells_it_carries) { + const std::string record = builder::cell_record( + iwork::cell::type::string, iwork::cell::flag::string_key, + std::string("\x01\x00\x00\x00", 4)); + + EXPECT_THAT( + [&] { + std::ignore = numbers_document( + {.rows = 1, + .columns = 1, + .tile_rows = {builder::tile_row(0, {record})}, + .strings = {builder::string_entry(1, std::string(1024, 'a'))}, + .tile_repeats = 100'000}); + }, + testing::ThrowsMessage( + testing::HasSubstr("too much text"))); +} diff --git a/test/src/internal/iwork/pages_test.cpp b/test/src/internal/iwork/pages_test.cpp index 4d6aab82..fab2fc3e 100644 --- a/test/src/internal/iwork/pages_test.cpp +++ b/test/src/internal/iwork/pages_test.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -13,6 +14,7 @@ #include #include +#include #include #include @@ -32,12 +34,15 @@ namespace iwork = odr::internal::iwork; namespace { -/// The paragraphs of a text root, a line break reading as a newline. +/// The paragraphs of a text root, a line break reading as a newline. A body +/// also holds the tables its text anchors, which this passes over. std::vector paragraphs(const Element root) { std::vector result; for (const Element paragraph : root.children()) { - EXPECT_EQ(paragraph.type(), ElementType::paragraph); + if (paragraph.type() != ElementType::paragraph) { + continue; + } std::string text; for (const Element child : paragraph.children()) { @@ -110,7 +115,8 @@ TEST(Iwork, pages_body_text) { // separate its sections ASSERT_EQ(text.size(), 54); EXPECT_EQ(text[0], "Table of Contents"); - // the anchor of a drawable is dropped: nothing reads drawables yet + // the anchor of a drawable is dropped from the text; a table anchored there + // becomes an element of its own, and an image is still not read EXPECT_EQ(text[1], ""); EXPECT_EQ(text[4], "Headline"); EXPECT_EQ(text[5], "Nested Headline"); @@ -121,6 +127,52 @@ TEST(Iwork, pages_body_text) { EXPECT_EQ(text.back(), "image"); } +// A `U+FFFC` in the body anchors a drawable, which the attachment run table +// names. The one that is a table becomes a `Table` after the paragraph its +// anchor sits in; its cells hold rich text, one storage each. +TEST(Iwork, pages_table) { + const DocumentFile document_file( + TestData::test_file_path("odr-public/pages/style-various-1.pages"), + Logger::null()); + const Document document = document_file.document(); + + std::vector tables; + for (const Element child : document.root_element().children()) { + if (child.type() == ElementType::table) { + tables.push_back(child); + } + } + ASSERT_EQ(tables.size(), 1); + + // the rule the reader decided: a table is a sibling after the paragraph its + // anchor sits in, not a child of it and not appended to the body. The anchor + // paragraph carries nothing but the `U+FFFC` itself, so it reads empty, and + // the body continues after the table rather than ending there. + const Element anchor = tables.front().previous_sibling(); + ASSERT_TRUE(anchor); + EXPECT_EQ(anchor.type(), ElementType::paragraph); + EXPECT_EQ(builder::element_text(anchor), ""); + EXPECT_TRUE(tables.front().next_sibling()); + + const Table table = tables.front().as_table(); + EXPECT_EQ(table.dimensions().rows, 2); + EXPECT_EQ(table.dimensions().columns, 2); + + std::vector> cells; + for (const Element row : table.rows()) { + std::vector texts; + for (const Element cell : row.children()) { + texts.push_back(builder::element_text(cell)); + } + cells.push_back(std::move(texts)); + } + + EXPECT_EQ(cells, (std::vector>{ + {"A1", "B1\nasdf"}, + {"A2", "B2\n\n"}, + })); +} + // Components share names — `style-various-1.pages` holds two dozen called // `Tables/DataList` — so the package has to load them by locator. Keying on // the name hands back the wrong file and leaves the rest never loaded, which @@ -207,3 +259,40 @@ TEST(Iwork, package_without_a_document_component_is_not_an_iwork_file) { EXPECT_THROW(iwork::IworkFile{files}, NoIworkFile); } + +// A model's extent is two varints, so a grid of millions is a few bytes on the +// wire. A Pages table is walked densely — its rows are children — so the +// budget is what stands between that and the memory it asks for. +TEST(Iwork, a_declared_table_extent_is_capped_by_the_elements_it_builds) { + EXPECT_THAT( + [] { + std::ignore = Document(std::make_shared( + FileType::iwork_pages, + builder::pages_table_package( + {.rows = 1'000'000, .columns = 1'000'000}))); + }, + testing::ThrowsMessage( + testing::HasSubstr("too many elements"))); +} + +// A cell's rich text is an ordinary storage, which may anchor a table again. +// The budget counts what a walk builds, not how deep it goes, so the depth +// bound is the only thing that ends a storage that reaches itself. +TEST(Iwork, a_storage_that_reaches_itself_is_bounded_by_its_depth) { + EXPECT_THAT( + [] { + const std::string record = builder::cell_record( + iwork::cell::type::rich_text, iwork::cell::flag::rich_text_key, + std::string("\x01\x00\x00\x00", 4)); + std::ignore = Document(std::make_shared( + FileType::iwork_pages, + builder::pages_table_package( + {.rows = 1, + .columns = 1, + .tile_rows = {builder::tile_row(0, {record})}, + // the cell's rich text is the body storage that anchors it + .rich_text_storage = builder::body_identifier}))); + }, + testing::ThrowsMessage( + testing::HasSubstr("nest too deeply"))); +}