diff --git a/CHANGELOG.md b/CHANGELOG.md index 636f818a6..7a5c1958c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,9 @@ The release run heads these entries with the version and opens a fresh missing. - An odf shape or frame that is not filled no longer paints the colour of one that was. +- Text in a pdf can be selected, searched and copied where it came out as CJK + before. A simple font's codes are one byte each, whatever codespace its + `ToUnicode` map declares, and producers routinely declare two. ## v6.10.1 - 2026-08-21 diff --git a/src/odr/internal/pdf/AGENTS.md b/src/odr/internal/pdf/AGENTS.md index 132c0757c..320b48408 100644 --- a/src/odr/internal/pdf/AGENTS.md +++ b/src/odr/internal/pdf/AGENTS.md @@ -53,7 +53,13 @@ directly; legacy CJK CMaps via `pdf_cid`/`pdf_cid_data`: code → CID → Unicod `/CIDSystemInfo` collection for `Identity-H`/embedded-CMap → embedded-font reverse map (code → glyph → `code_point_for_glyph`). Only a genuinely unmapped code yields "no Unicode" — never byte-garbage. The point of the chain is that each link -recovers a class of real-world PDF the previous one misses. +recovers a class of real-world PDF the previous one misses. A **simple font's +codes are one byte** (9.10.3), so `to_unicode` imposes that width rather than +read it off the `/ToUnicode` codespace — producers write the two-byte +`<0000> ` boilerplate there regardless, and splitting by it pairs the +bytes into CJK. The entries themselves are keyed either way, so an imposed +one-byte code is also looked up zero-padded. Composite codes keep splitting by +the codespace, as `Font::codes` splits them. **`std::any`-based object model.** `Object` holds its value in `std::any` with typed `is_*`/`as_*` accessors (mirrors `oldms/`'s `Entry`). Pro: one value type diff --git a/src/odr/internal/pdf/pdf_cmap.cpp b/src/odr/internal/pdf/pdf_cmap.cpp index aa14c682c..63d0dff41 100644 --- a/src/odr/internal/pdf/pdf_cmap.cpp +++ b/src/odr/internal/pdf/pdf_cmap.cpp @@ -4,6 +4,7 @@ #include #include +#include namespace odr::internal::pdf { @@ -62,13 +63,16 @@ std::size_t CMap::code_length(const std::string &codes, return code_width(static_cast(codes[pos])); } -std::string CMap::translate_string(const std::string &codes) const { +std::string CMap::translate_string(const std::string &codes, + const bool single_byte_codes) const { std::u16string result; std::size_t pos = 0; while (pos < codes.size()) { const std::size_t width = - std::min(code_length(codes, pos), codes.size() - pos); + single_byte_codes + ? 1 + : std::min(code_length(codes, pos), codes.size() - pos); const std::string code = codes.substr(pos, width); pos += width; @@ -77,6 +81,16 @@ std::string CMap::translate_string(const std::string &codes) const { continue; } + // Only for an imposed width — a declared mixed codespace keeps `<20>` and + // `<0020>` distinct. + if (single_byte_codes) { + if (const auto it = m_map.find(std::string(1, '\0') + code); + it != m_map.end()) { + result += it->second; + continue; + } + } + // Unknown code: fall back to its numeric value as a single UTF-16 unit // (identity for single-byte codes). These "no Unicode" runs are left for // later re-encoding. diff --git a/src/odr/internal/pdf/pdf_cmap.hpp b/src/odr/internal/pdf/pdf_cmap.hpp index fb8f788bf..3da8e89d5 100644 --- a/src/odr/internal/pdf/pdf_cmap.hpp +++ b/src/odr/internal/pdf/pdf_cmap.hpp @@ -56,7 +56,12 @@ class CMap { /// does, keeping a mixed 1-/2-byte codespace aligned across both. [[nodiscard]] std::size_t code_width(std::uint8_t first) const; - [[nodiscard]] std::string translate_string(const std::string &codes) const; + /// `single_byte_codes` overrides the codespace ranges. An imposed + /// single-byte code is also looked up zero-padded to two bytes, producers + /// keying the entries either way. + [[nodiscard]] std::string + translate_string(const std::string &codes, + bool single_byte_codes = false) const; /// True when at least one `cidchar`/`cidrange` mapping was parsed (an /// embedded CID `/Encoding` CMap). When false the composite code -> CID is diff --git a/src/odr/internal/pdf/pdf_document.cpp b/src/odr/internal/pdf/pdf_document.cpp index ec8443906..46278a96d 100644 --- a/src/odr/internal/pdf/pdf_document.cpp +++ b/src/odr/internal/pdf/pdf_document.cpp @@ -161,8 +161,11 @@ std::uint16_t Font::glyph_for_code(const std::uint32_t code) const { } std::string Font::to_unicode(const std::string &codes) const { + // A simple font's codes are one byte each (ISO 32000-1 9.10.3); its + // `ToUnicode` codespace is not to be trusted, producers writing the + // two-byte `<0000> ` boilerplate there regardless. if (!cmap.empty()) { - return cmap.translate_string(codes); + return cmap.translate_string(codes, !composite); } if (composite) { // A composite (Type0) font with no `ToUnicode` CMap. A predefined @@ -210,7 +213,7 @@ std::string Font::to_unicode(const std::string &codes) const { !unicode.empty()) { return unicode; } - return cmap.translate_string(codes); + return cmap.translate_string(codes, true); } } // namespace odr::internal::pdf diff --git a/test/src/internal/pdf/pdf_cmap.cpp b/test/src/internal/pdf/pdf_cmap.cpp index 137c6cec5..9e751cfc1 100644 --- a/test/src/internal/pdf/pdf_cmap.cpp +++ b/test/src/internal/pdf/pdf_cmap.cpp @@ -199,3 +199,46 @@ TEST(PdfCMap, usecmap_disables_local_codespace_authority) { EXPECT_TRUE(cmap.has_cid_map()); EXPECT_EQ(cmap.cid_for_code(std::string_view("\x20", 1)), 1u); } + +TEST(PdfCMap, imposed_code_width_overrides_codespace) { + // A simple font's `ToUnicode` CMap carrying the two-byte `<0000> ` + // boilerplate over one-byte entries; splitting by it pairs the codes up. + CMap cmap = parse("1 begincodespacerange\n" + "<0000> \n" + "endcodespacerange\n" + "2 beginbfchar\n" + "<41> <0041>\n" + "<42> <0042>\n" + "endbfchar\n"); + + EXPECT_EQ(cmap.translate_string("\x41\x42"), "\xe4\x85\x82"); // U+4142 + EXPECT_EQ(cmap.translate_string("\x41\x42", true), "AB"); +} + +TEST(PdfCMap, imposed_code_width_falls_back_to_a_padded_entry) { + CMap cmap = parse("1 begincodespacerange\n" + "<0000> \n" + "endcodespacerange\n" + "2 beginbfchar\n" + "<0041> <0041>\n" + "<0042> <0042>\n" + "endbfchar\n"); + + EXPECT_EQ(cmap.translate_string("\x41\x42", true), "AB"); +} + +TEST(PdfCMap, imposed_code_width_keeps_a_mixed_codespace_distinct) { + // Padding is only for an imposed width; a declared mixed codespace keeps + // `<20>` and `<2120>` apart. + CMap cmap = parse("2 begincodespacerange\n" + "<00> <20>\n" + "<2100> \n" + "endcodespacerange\n" + "2 beginbfchar\n" + "<20> <0041>\n" + "<2120> <0042>\n" + "endbfchar\n"); + + EXPECT_EQ(cmap.translate_string("\x20"), "A"); + EXPECT_EQ(cmap.translate_string("\x21\x20"), "B"); +} diff --git a/test/src/internal/pdf/pdf_font.cpp b/test/src/internal/pdf/pdf_font.cpp index d0f7261b1..d7ec64cd7 100644 --- a/test/src/internal/pdf/pdf_font.cpp +++ b/test/src/internal/pdf/pdf_font.cpp @@ -184,6 +184,24 @@ TEST(PdfFont, to_unicode_prefers_cmap_over_reverse_map) { EXPECT_EQ(font.to_unicode(codes2({1})), "Z"); } +TEST(PdfFont, simple_font_to_unicode_ignores_cmap_codespace) { + Font font; + font.cmap.add_codespace_range(codes2({0}), codes2({0xffff})); + font.cmap.map_single("\x41", u"A"); + font.cmap.map_single("\x42", u"B"); + + EXPECT_EQ(font.to_unicode("\x41\x42"), "AB"); +} + +TEST(PdfFont, composite_to_unicode_splits_by_cmap_codespace) { + Font font; + font.composite = true; + font.cmap.add_codespace_range(codes2({0}), codes2({0xffff})); + font.cmap.map_single(codes2({0x4142}), u"Z"); + + EXPECT_EQ(font.to_unicode("\x41\x42"), "Z"); +} + TEST(PdfFont, simple_font_glyph_for_code_via_cmap) { // A simple (1-byte) TrueType font: the code's Unicode reaches the glyph // through the embedded (3,1) cmap.