From bfb46c19ae4ec59581d569d0649f4db786b79f36 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Sun, 13 Sep 2026 16:33:33 -0400 Subject: [PATCH 1/4] Shrink compressed translations: dense alphabet, per-class Huffman, optimal parsing Rework the translation compressor in `py/maketranslationdata.py` and the decoder in `translate.c`: - Renumber a language's non-ASCII characters into 0x80.. by frequency and map them back through `alphabet[]`, so every symbol fits in 8 bits and the dictionary gets the remaining code points. Languages with more than 127 distinct non-ASCII characters (ja, ko) keep the 16-bit fallback. - Choose Huffman codes per class of the previously decoded byte (start or space, letter, other, ...) through `class_map[]`; the generator tries several class presets and keeps the smallest result. - Tokenize each string by dynamic programming for the fewest bits instead of greedy longest match, deciding per occurrence whether a qstr reference pays off. Dictionary words left unused are dropped. The format is documented in `compressed_string.h`. Same-commit gains on the tightest languages: metro_m4_express fr +1,208 bytes free, pl +1,160; metro_m0_express pl +672. The decoder grows 76-108 bytes. Verified on the unix coverage port (979 tests) and on a Metro M4 Express in fr and en_US. Co-Authored-By: Claude Fable 5.1 --- py/maketranslationdata.py | 540 ++++++++++++------ .../shared/translate/compressed_string.h | 64 ++- supervisor/shared/translate/translate.c | 53 +- 3 files changed, 451 insertions(+), 206 deletions(-) diff --git a/py/maketranslationdata.py b/py/maketranslationdata.py index c3c3442e311..19bcb0970c5 100644 --- a/py/maketranslationdata.py +++ b/py/maketranslationdata.py @@ -6,13 +6,13 @@ supported CPython version are unintended. For documentation about the format of compressed translated strings, see -supervisor/shared/translate/translate.h +supervisor/shared/translate/compressed_string.h """ from __future__ import print_function import bisect -from dataclasses import dataclass +from dataclasses import dataclass, field import re import sys @@ -73,6 +73,46 @@ '"': '\\"', } +# Reserved symbol values. 2 and 3 are unused for now but must not appear in +# translated text either. +QSTR_ESC = "\1" +RESERVED_CHARS = {"\1", "\2", "\3"} + +# The first non-ASCII symbol value in the dense alphabet. +ALPHABET_BASE = 0x80 + +# Huffman codes are chosen per "class" of the previously decoded character. +# These base classes are fixed in the C decoder (translate.c, base_class()); +# the generator collapses them with class_map[] into TRANSLATION_CLASSES tables. +NUM_BASE_CLASSES = 7 +# Presets to try, as class_map. Each maps a base class to a table index. +CLASS_PRESETS = ( + (0, 0, 0, 0, 0, 0, 0), # a single table: the tables cost more than they save + (0, 1, 1, 2, 2, 2, 1), # space, letter, other + (0, 1, 2, 3, 4, 5, 1), # non-ASCII shares the lowercase table + (0, 1, 2, 3, 4, 5, 6), # non-ASCII has its own table +) + + +def base_class(c): + """Class of the character preceding the next symbol; None means start of string. + Computed on the (possibly remapped) character: remapped characters are >= 0x80 + exactly when the original is, which is all the classification looks at.""" + if c is None or c == " ": + return 0 + o = ord(c) + if 0x61 <= o <= 0x7A: + return 1 + if 0x41 <= o <= 0x5A: + return 2 + if 0x30 <= o <= 0x39: + return 3 + if o == 0x25: # '%' + return 5 + if o >= 0x80: + return 6 + return 4 + # this must match the equivalent function in qstr.c def compute_hash(qstr, bytes_hash): @@ -140,90 +180,188 @@ def iter_substrings(s, minlen, maxlen): yield s[begin : begin + n] -translation_requires_uint16 = {"cs", "ja", "ko", "pl", "tr", "zh_Latn_pinyin"} +# Languages whose non-ASCII alphabet does not fit the dense 8-bit alphabet and +# therefore use 16-bit table entries. +translation_requires_uint16 = {"ja", "ko"} -def compute_unicode_offset(texts): - all_ch = set(" ".join(texts)) - ch_160 = sorted(c for c in all_ch if 160 <= ord(c) < 255) - ch_256 = sorted(c for c in all_ch if 255 < ord(c)) - if not ch_256: - return 0, 0 - min_256 = ord(min(ch_256)) - span = ord(max(ch_256)) - ord(min(ch_256)) + 1 +@dataclass +class EncodingTable: + # One entry per class table: list of symbols in canonical order. + values: list + # One entry per class table: list of code-length counts, padded to lengths_row. + lengths: list + lengths_row: int + class_map: tuple + # Per class table: symbol -> canonical code (string of '0'/'1'). + canonical: list + # Dictionary words, in remapped characters, sorted by length. + words: list + word_start: int + # Dense alphabet: index -> original character. Empty in uint16 mode. + alphabet: list + # Original character -> remapped character (identity for ASCII). + remap: dict + translation_qstr_bits: int + qstrs: dict + qstrs_inv: dict + values_type: str + # Remapped text -> token list used to encode it. A token is a str (character + # or word) or a ("q", qstr) tuple. + tokens: dict = field(default_factory=dict) - if ch_160: - max_160 = ord(max(ch_160)) + 1 - else: - max_160 = max(160, 255 - span) - if max_160 + span > 256: - return 0, 0 +def remap_text(text, remap): + return "".join(remap.get(c, c) for c in text) - offstart = max_160 - offset = min_256 - max_160 - return offstart, offset +def is_qstr(token): + return isinstance(token, tuple) -@dataclass -class EncodingTable: - values: object - lengths: object - words: object - canonical: object - extractor: object - apply_offset: object - remove_offset: object - translation_qstr_bits: int - qstrs: object - qstrs_inv: object + +def token_len(token): + return len(token[1]) if is_qstr(token) else len(token) + + +def token_symbol(token): + """The Huffman symbol a token is coded as (qstrs share one escape symbol).""" + return QSTR_ESC if is_qstr(token) else token + + +def code_lengths(counter): + """Huffman code lengths for a symbol counter. A lone symbol gets a 1-bit code.""" + if len(counter) == 1: + return {next(iter(counter)): 1} + cb = huffman.codebook(counter.items()) + return {k: len(v) for k, v in cb.items()} + + +def canonical_codes(lengths): + """Canonical Huffman codes from a symbol -> length dict. + Returns (values in canonical order, per-length counts, symbol -> code).""" + if not lengths: + return [], [], {} + values = [] + length_count = collections.Counter() + canonical = {} + renumbered = 0 + last_length = None + for atom, length in sorted(lengths.items(), key=lambda x: (x[1], x[0])): + values.append(atom) + length_count[length] += 1 + if last_length: + renumbered <<= length - last_length + canonical[atom] = "{0:0{width}b}".format(renumbered, width=length) + renumbered += 1 + last_length = length + counts = [length_count.get(i, 0) for i in range(1, max(length_count) + 2)] + return values, counts, canonical + + +def parse_optimal( + text, lens, class_map, words_by_first, qstrs_by_first, qstr_bits, unknown_len=32 +): + """Tokenize text with the fewest bits given per-class code lengths. + Symbols missing from a class table are allowed at a penalty so a parse always exists.""" + n = len(text) + best = [math.inf] * (n + 1) + back = [None] * (n + 1) + best[0] = 0 + for i in range(n): + if best[i] == math.inf: + continue + table = lens[class_map[base_class(text[i - 1] if i else None)]] + c = text[i] + cands = [c] + for w in words_by_first.get(c, ()): + if text.startswith(w, i): + cands.append(w) + for q in qstrs_by_first.get(c, ()): + if text.startswith(q, i): + cands.append(("q", q)) + for tok in cands: + cost = table.get(token_symbol(tok), unknown_len) + if is_qstr(tok): + cost += qstr_bits + j = i + token_len(tok) + if best[i] + cost < best[j]: + best[j] = best[i] + cost + back[j] = tok + out = [] + i = n + while i > 0: + tok = back[i] + out.append(tok) + i -= token_len(tok) + out.reverse() + return out + + +def tally(texts, tokens, class_map): + """Symbol counts per class table for the given tokenizations.""" + counts = collections.defaultdict(collections.Counter) + for text in texts: + pos = 0 + for tok in tokens[text]: + cls = class_map[base_class(text[pos - 1] if pos else None)] + counts[cls][token_symbol(tok)] += 1 + pos += token_len(tok) + return counts + + +def encoded_bits(text, tokens, lens, class_map, qstr_bits): + bits = 0 + pos = 0 + for tok in tokens: + cls = class_map[base_class(text[pos - 1] if pos else None)] + bits += lens[cls][token_symbol(tok)] + if is_qstr(tok): + bits += qstr_bits + pos += token_len(tok) + return bits def compute_huffman_coding(qstrs, translation_name, translations, f, compression_level): # possible future improvement: some languages are better when consider len(k) > 2. try both? qstrs = dict((k, v) for k, v in qstrs.items() if len(k) > 3) qstr_strs = list(qstrs.keys()) - texts = [t[1] for t in translations] + original_texts = [t[1] for t in translations] words = [] - start_unused = 0x80 - end_unused = 0xFF - max_ord = 0 - offstart, offset = compute_unicode_offset(texts) - - def apply_offset(c): - oc = ord(c) - if oc >= offstart: - oc += offset - return chr(oc) - - def remove_offset(c): - oc = ord(c) - if oc >= offstart: - oc = oc - offset - try: - return chr(oc) - except Exception as e: - raise ValueError(f"remove_offset {offstart=} {oc=}") from e + for text in original_texts: + bad = RESERVED_CHARS.intersection(text) + if bad: + raise ValueError(f"Translation contains reserved character {bad!r}: {text!r}") - for text in texts: - for c in text: - c = remove_offset(c) - ord_c = ord(c) - max_ord = max(ord_c, max_ord) - if 0x80 <= ord_c < 0xFF: - end_unused = min(ord_c, end_unused) - max_words = end_unused - 0x80 + translation_name = translation_name.split("/")[-1].split(".")[0] + + # Dense alphabet: non-ASCII characters are renumbered from 0x80 by decreasing + # frequency so that all symbols fit in 8 bits. If there are too many of them, + # fall back to raw 16-bit code points. + hi_count = collections.Counter(c for t in original_texts for c in t if ord(c) >= 0x80) + if len(hi_count) <= 0x7F: + alphabet = [c for c, _ in hi_count.most_common()] + remap = {c: chr(ALPHABET_BASE + i) for i, c in enumerate(alphabet)} + word_start = ALPHABET_BASE + len(alphabet) + max_words = 0x100 - word_start + values_type = "uint8_t" + else: + if translation_name not in translation_requires_uint16: + raise ValueError( + f"Translation {translation_name} expected to fit in 8 bits but required 16 bits" + ) + alphabet = [] + remap = {} + # Words take the unused code points from 0x80 up to the lowest one in use. + end_unused = min([0xFF] + [o for o in map(ord, hi_count) if o < 0xFF]) + word_start = ALPHABET_BASE + max_words = end_unused - word_start + values_type = "uint16_t" if compression_level < 5: max_words = 0 + bits_per_codepoint = 16 if values_type == "uint16_t" else 8 - bits_per_codepoint = 16 if max_ord > 255 else 8 - values_type = "uint16_t" if max_ord > 255 else "uint8_t" - translation_name = translation_name.split("/")[-1].split(".")[0] - if max_ord > 255 and translation_name not in translation_requires_uint16: - raise ValueError( - f"Translation {translation_name} expected to fit in 8 bits but required 16 bits" - ) + texts = [remap_text(t, remap) for t in original_texts] # Prune the qstrs to only those that appear in the texts qstr_counters = collections.Counter() @@ -248,7 +386,7 @@ def remove_offset(c): for t in texts: for atom in extractor.iter(t): if atom in qstrs: - atom = "\1" + atom = QSTR_ESC counter[atom] += 1 cb = huffman.codebook(counter.items()) lengths = sorted(dict((v, len(cb[k])) for k, v in counter.items()).items()) @@ -320,89 +458,142 @@ def est_net_savings(s, occ): word = scores[0][0] words.append(word) - splitters = words[:] - if compression_level > 3: - splitters.extend(qstr_strs) + # Now that the dictionary is fixed, find the tokenization of each string that + # takes the fewest bits, with Huffman codes chosen per class of the previous + # character. Code lengths and tokenization depend on each other, so iterate + # a few rounds from the greedy tokenization; the final tokenization and the + # codes built from it are what get emitted, so they are always consistent. + use_qstrs = compression_level > 3 + dp_qstrs = qstr_strs if use_qstrs else [] + # Upper bound on the qstr index width while parsing; the real width is computed below. + est_qstr_bits = max([0] + [qstrs[q] for q in dp_qstrs]).bit_length() + words_by_first = collections.defaultdict(list) + for w in words: + words_by_first[w[0]].append(w) + qstrs_by_first = collections.defaultdict(list) + for q in dp_qstrs: + qstrs_by_first[q[0]].append(q) + + greedy = TextSplitter(words + dp_qstrs) + greedy_tokens = {} + for t in texts: + greedy_tokens[t] = [("q", a) if a in qstrs else a for a in greedy.iter(t)] + + max_translation_encoded_length = max(len(t.encode("utf-8")) for t in original_texts) + encoded_length_bits = max_translation_encoded_length.bit_length() + + def table_bytes(counts, used_words): + nclasses = max(class_map) + 1 + row = max(max(code_lengths(cn).values()) for cn in counts.values()) + 1 + mchar = bits_per_codepoint // 8 + n = sum(len(cn) for cn in counts.values()) * mchar # values + n += nclasses * row # lengths + n += 2 * (nclasses + 1) # values_offset + n += NUM_BASE_CLASSES # class_map + n += sum(len(w) for w in used_words) * mchar # words + if used_words: + n += len(used_words[-1]) - len(used_words[0]) + 1 # wlencount + n += 2 * len(alphabet) + return n + + best = None + for class_map in CLASS_PRESETS: + tokens = greedy_tokens + counts = tally(texts, tokens, class_map) + lens = {cls: code_lengths(cn) for cls, cn in counts.items()} + for _ in range(3): + tokens = { + t: parse_optimal(t, lens, class_map, words_by_first, qstrs_by_first, est_qstr_bits) + for t in texts + } + counts = tally(texts, tokens, class_map) + lens = {cls: code_lengths(cn) for cls, cn in counts.items()} + used_symbols = set().union(*counts.values()) + used_words = sorted((w for w in words if w in used_symbols), key=len) + size = table_bytes(counts, used_words) + sum( + (encoded_length_bits + encoded_bits(t, tokens[t], lens, class_map, est_qstr_bits) + 7) + // 8 + for t in texts + ) + if best is None or size < best[0]: + best = (size, class_map, tokens, counts, lens, used_words) + size, class_map, tokens, counts, lens, words = best - words.sort(key=len) - extractor = TextSplitter(splitters) - counter = collections.Counter() used_qstr = 0 - for t in texts: - for atom in extractor.iter(t): - if atom in qstrs: - used_qstr = max(used_qstr, qstrs[atom]) - atom = "\1" - counter[atom] += 1 - cb = huffman.codebook(counter.items()) + for toks in tokens.values(): + for tok in toks: + if is_qstr(tok): + used_qstr = max(used_qstr, qstrs[tok[1]]) + translation_qstr_bits = used_qstr.bit_length() - word_start = start_unused word_end = word_start + len(words) - 1 - f.write(f"// # words {len(words)}\n") - f.write(f"// words {words}\n") + word_code = {w: chr(word_start + i) for i, w in enumerate(words)} - values = [] - length_count = {} - renumbered = 0 - last_length = None - canonical = {} - for atom, code in sorted(cb.items(), key=lambda x: (len(x[1]), x[0])): - if atom in qstr_strs: - atom = "\1" - values.append(atom) - length = len(code) - if length not in length_count: - length_count[length] = 0 - length_count[length] += 1 - if last_length: - renumbered <<= length - last_length - # print(f"atom={repr(atom)} code={code}", file=sys.stderr) - canonical[atom] = "{0:0{width}b}".format(renumbered, width=length) - if len(atom) > 1: - o = words.index(atom) + 0x80 - s = "".join(C_ESCAPES.get(ch1, ch1) for ch1 in atom) - f.write(f"// {o} {s} {counter[atom]} {canonical[atom]} {renumbered}\n") - else: - s = C_ESCAPES.get(atom, atom) - canonical[atom] = "{0:0{width}b}".format(renumbered, width=length) - o = ord(atom) - f.write(f"// {o} {s} {counter[atom]} {canonical[atom]} {renumbered}\n") - renumbered += 1 - last_length = length - lengths = bytearray() - f.write(f"// length count {length_count}\n") + def symbol_value(sym): + return ord(word_code.get(sym, sym)) - for i in range(1, max(length_count) + 2): - lengths.append(length_count.get(i, 0)) - f.write(f"// values {values} lengths {len(lengths)} {lengths}\n") + nclasses = max(class_map) + 1 + values = [] + lengths_rows = [] + canonical = [] + for cls in range(nclasses): + # Symbols are ordered by (length, value) so that the C decoder's canonical + # walk lands on the same index. + table = lens.get(cls, {}) + by_value = {chr(symbol_value(s)): l for s, l in table.items()} + v, counts_row, canon = canonical_codes(by_value) + values.append([ord(x) for x in v]) + lengths_rows.append(counts_row) + canonical.append({s: canon[chr(symbol_value(s))] for s in table}) + lengths_row = max(1, max(len(r) for r in lengths_rows)) + lengths_rows = [r + [0] * (lengths_row - len(r)) for r in lengths_rows] + assert all(len(code) >= 1 for canon in canonical for code in canon.values()) - f.write(f"// {values} {lengths}\n") - values = [(atom if len(atom) == 1 else chr(0x80 + words.index(atom))) for atom in values] - max_translation_encoded_length = max( - len(translation.encode("utf-8")) for (original, translation) in translations + f.write(f"// # words {len(words)}\n") + f.write( + "// words {}\n".format([remap_text(w, {v: k for k, v in remap.items()}) for w in words]) ) + f.write(f"// # alphabet {len(alphabet)}\n") + f.write(f"// class_map {class_map} tables {nclasses}\n") + for cls in range(nclasses): + f.write(f"// class {cls}: {len(values[cls])} symbols, lengths {lengths_rows[cls]}\n") maxlen = len(words[-1]) if words else 0 minlen = len(words[0]) if words else 0 wlencount = [len([None for w in words if len(w) == l]) for l in range(minlen, maxlen + 1)] - translation_qstr_bits = used_qstr.bit_length() - f.write("typedef {} mchar_t;\n".format(values_type)) - f.write("const uint8_t lengths[] = {{ {} }};\n".format(", ".join(map(str, lengths)))) + f.write("#define compress_max_length_bits ({})\n".format(encoded_length_bits)) + f.write("#define TRANSLATION_CLASSES {}\n".format(nclasses)) + f.write("#define LENGTHS_ROW {}\n".format(lengths_row)) + f.write( + "const uint8_t class_map[{}] = {{ {} }};\n".format( + NUM_BASE_CLASSES, ", ".join(map(str, class_map)) + ) + ) f.write( - "const mchar_t values[] = {{ {} }};\n".format( - ", ".join(str(ord(remove_offset(u))) for u in values) + "const uint8_t lengths[] = {{ {} }};\n".format( + ", ".join(str(x) for row in lengths_rows for x in row) ) ) + offsets = [0] + for v in values: + offsets.append(offsets[-1] + len(v)) f.write( - "#define compress_max_length_bits ({})\n".format( - max_translation_encoded_length.bit_length() + "const uint16_t values_offset[{}] = {{ {} }};\n".format( + nclasses + 1, ", ".join(map(str, offsets)) ) ) + f.write( + "const mchar_t values[] = {{ {} }};\n".format(", ".join(str(x) for v in values for x in v)) + ) + f.write("#define alphabet_size {}\n".format(len(alphabet))) + f.write( + "const uint16_t alphabet[] = {{ {} }};\n".format(", ".join(str(ord(c)) for c in alphabet)) + ) f.write( "const mchar_t words[] = {{ {} }};\n".format( - ", ".join(str(ord(remove_offset(c))) for w in words for c in w) + ", ".join(str(ord(c)) for w in words for c in w) ) ) f.write("const uint8_t wlencount[] = {{ {} }};\n".format(", ".join(str(p) for p in wlencount))) @@ -410,30 +601,32 @@ def est_net_savings(s, occ): f.write("#define word_end {}\n".format(word_end)) f.write("#define minlen {}\n".format(minlen)) f.write("#define maxlen {}\n".format(maxlen)) - f.write("#define translation_offstart {}\n".format(offstart)) - f.write("#define translation_offset {}\n".format(offset)) f.write("#define translation_qstr_bits {}\n".format(translation_qstr_bits)) qstrs_inv = dict((v, k) for k, v in qstrs.items()) return EncodingTable( values, - lengths, - words, + lengths_rows, + lengths_row, + class_map, canonical, - extractor, - apply_offset, - remove_offset, + words, + word_start, + alphabet, + remap, translation_qstr_bits, qstrs, qstrs_inv, + values_type, + tokens, ) def decompress(encoding_table, encoded, encoded_length_bits): - qstrs_inv = encoding_table.qstrs_inv - values = encoding_table.values - lengths = encoding_table.lengths - words = encoding_table.words + """Decode as the C decoder does (translate.c). Returns the original text.""" + et = encoding_table + alphabet_size = len(et.alphabet) + word_end = et.word_start + len(et.words) - 1 def bititer(): for byte in encoded: @@ -449,10 +642,23 @@ def getnbits(n): return bits dec = [] + last = None length = getnbits(encoded_length_bits) + decoded = 0 - i = 0 - while i < length: + def emit(u): + nonlocal last, decoded + if ALPHABET_BASE <= u < ALPHABET_BASE + alphabet_size: + c = et.alphabet[u - ALPHABET_BASE] + else: + c = chr(u) + dec.append(c) + decoded += len(c.encode("utf-8")) + last = c + + while decoded < length: + cls = et.class_map[base_class(last)] + lengths = et.lengths[cls] bits = 0 bit_length = 0 max_code = lengths[0] @@ -461,28 +667,31 @@ def getnbits(n): bits = (bits << 1) | nextbit() bit_length += 1 if max_code > 0 and bits < max_code: - # print('{0:0{width}b}'.format(bits, width=bit_length)) break max_code = (max_code << 1) + lengths[bit_length] searched_length += lengths[bit_length] - - v = values[searched_length + bits - max_code] - if v == chr(1): - qstr_idx = getnbits(encoding_table.translation_qstr_bits) - v = qstrs_inv[qstr_idx] - elif v >= chr(0x80) and v < chr(0x80 + len(words)): - v = words[ord(v) - 0x80] - i += len(v.encode("utf-8")) - dec.append(v) + v = et.values[cls][searched_length + bits - max_code] + if v == 1: + qstr_idx = getnbits(et.translation_qstr_bits) + s = et.qstrs_inv[qstr_idx] + dec.append(s) + decoded += len(s.encode("utf-8")) + last = s[-1] + elif et.word_start <= v <= word_end: + for c in et.words[v - et.word_start]: + emit(ord(c)) + else: + emit(v) return "".join(dec) def compress(encoding_table, decompressed, encoded_length_bits, len_translation_encoded): + """Encode a translation using the tokenization chosen in compute_huffman_coding().""" if not isinstance(decompressed, str): raise TypeError() - qstrs = encoding_table.qstrs - canonical = encoding_table.canonical - extractor = encoding_table.extractor + et = encoding_table + text = remap_text(decompressed, et.remap) + tokens = et.tokens[text] enc = 1 @@ -494,17 +703,20 @@ def put_bits(enc, b, n): enc = put_bit(enc, b & (1 << i)) return enc + def put_code(enc, cls, sym): + for b in et.canonical[cls][sym]: + enc = put_bit(enc, b == "1") + return enc + enc = put_bits(enc, len_translation_encoded, encoded_length_bits) - for atom in extractor.iter(decompressed): - if atom in qstrs: - can = canonical["\1"] - else: - can = canonical[atom] - for b in can: - enc = put_bit(enc, b == "1") - if atom in qstrs: - enc = put_bits(enc, qstrs[atom], encoding_table.translation_qstr_bits) + pos = 0 + for tok in tokens: + cls = et.class_map[base_class(text[pos - 1] if pos else None)] + enc = put_code(enc, cls, token_symbol(tok)) + if is_qstr(tok): + enc = put_bits(enc, et.qstrs[tok[1]], et.translation_qstr_bits) + pos += token_len(tok) while enc.bit_length() % 8 != 1: enc = put_bit(enc, 0) diff --git a/supervisor/shared/translate/compressed_string.h b/supervisor/shared/translate/compressed_string.h index 89dd75c22ae..5d1bc260c76 100644 --- a/supervisor/shared/translate/compressed_string.h +++ b/supervisor/shared/translate/compressed_string.h @@ -13,38 +13,39 @@ // The format of the compressed data is: // - the size of the uncompressed string in UTF-8 bytes, encoded as a // (compress_max_length_bits)-bit number. compress_max_length_bits is -// computed during dictionary generation time, and happens to be 8 -// for all current platforms. However, it'll probably end up being -// 9 in some translations sometime in the future. This length excludes +// computed during dictionary generation time, and is 8 for all +// current translations except ru, which needs 9. This length excludes // the trailing NUL, though notably decompress_length includes it. // -// - followed by the huffman encoding of the individual code -// points that make up the string. The trailing "\0" is not -// represented by a huffman code, but is implied by the length. -// (building the huffman encoding on UTF-16 code points gave better -// compression than building it on UTF-8 bytes) +// - followed by the huffman encoding of the symbols that make up the +// string. The trailing "\0" is not represented by a huffman code, but +// is implied by the length. // -// - If possible, the code points are represented as uint8_t values, with -// 0..127 representing themselves and 160..255 representing another range -// of Unicode, controlled by translation_offset and translation_offstart. -// If this is not possible, uint16_t values are used. At present, no translation -// requires code points not in the BMP, so this is adequate. +// - Each symbol is coded with one of TRANSLATION_CLASSES canonical Huffman +// tables, chosen by the class of the previously decoded byte: start of +// string or space, lowercase, uppercase, digit, '%', other, or >= 0x80. +// base_class() computes that seven-way class and class_map[] collapses it +// to a table index; the generator picks how many tables pay for themselves. +// The per-table code-length counts are rows of LENGTHS_ROW bytes in +// lengths[], and the symbols of table n are values[values_offset[n]..]. // -// - code points starting at 128 (word_start) and potentially extending -// to 255 (word_end) (but never interfering with the target -// language's used code points) stand for dictionary entries in a -// dictionary with size up to 256 code points. The dictionary entries -// are computed with a heuristic based on frequent substrings of 2 to -// 9 code points. These are called "words" but are not, grammatically -// speaking, words. They're just spans of code points that frequently -// occur together. They are ordered shortest to longest. +// - Symbol values 0..0x7F are ASCII (with 1 reserved for QSTR escapes, +// below, and 2 and 3 reserved for future escapes). Values from 0x80 up to +// 0x80 + alphabet_size - 1 are the non-ASCII characters used by the +// translation, renumbered by decreasing frequency; alphabet[] maps them +// back to Unicode code points. When a translation uses more than 127 +// distinct non-ASCII characters this is not possible, alphabet_size is 0 +// and the symbols are raw 16-bit code points instead (mchar_t is then +// uint16_t rather than uint8_t; it's very beneficial for mchar_t to be 8 +// bits!). At present, no translation requires code points outside the +// BMP, so this is adequate. // -// - If the translation uses a lot of code points or widely spaced code points, -// then the huffman table entries are UTF-16 code points. But if the translation -// uses only ASCII 7-bit code points plus a SMALL range of higher code points that -// still fit in 8 bits, translation_offset and translation_offstart are used to -// renumber the code points so that they still fit within 8 bits. (it's very beneficial -// for mchar_t to be 8 bits instead of 16!) +// - Symbol values from word_start to word_end stand for dictionary entries +// in a dictionary of up to 256 - word_start entries. The dictionary +// entries are computed with a heuristic based on frequent substrings of 2 +// to 11 symbols. These are called "words" but are not, grammatically +// speaking, words. They're just spans of symbols that frequently occur +// together. They are ordered shortest to longest. // // - dictionary entries are non-overlapping, and the _ending_ index of each // entry is stored in an array. A count of words of each length, from @@ -56,10 +57,11 @@ // - Value 1 ('\1') is used to indicate that a QSTR number follows. the // QSTR is encoded as a fixed number of bits (translation_qstr_bits), e.g., // 10 bits if the highest core qstr is from 512 to 1023 inclusive. -// (maketranslationdata uses a simple heuristic where any qstr >= 3 -// characters long is encoded in this way; this is simple but probably not -// optimal. In fact, the rule of >= 2 characters is better for SOME languages -// on SOME boards.) +// (maketranslationdata uses a simple heuristic where any qstr >= 4 +// characters long may be encoded in this way; whether a given occurrence +// is coded as a qstr or as characters is decided per occurrence by the +// parser, which picks the tokenization of each string that takes the +// fewest bits.) // // The "data" / "tail" construct is so that the struct's last member is a // "flexible array". However, the _only_ member is not permitted to be diff --git a/supervisor/shared/translate/translate.c b/supervisor/shared/translate/translate.c index b590b385a71..cc40d06e34a 100644 --- a/supervisor/shared/translate/translate.c +++ b/supervisor/shared/translate/translate.c @@ -37,14 +37,10 @@ static void get_word(int n, const mchar_t **pos, const mchar_t **end) { *end = *pos + len; } -static void put_utf8(vstr_t *vstr, int u) { - if (u >= translation_offstart) { - u += translation_offset; - } +static void put_utf8(vstr_t *vstr, unsigned u) { if (word_start <= u && u <= word_end) { - uint n = (u - word_start); const mchar_t *pos, *end; - get_word(n, &pos, &end); + get_word(u - word_start, &pos, &end); // note that at present, entries in the words table are // guaranteed not to represent words themselves, so this adds // at most 1 level of recursive call @@ -53,9 +49,39 @@ static void put_utf8(vstr_t *vstr, int u) { } return; } + // alphabet_size is a compile-time constant; the test folds away when it is 0. + if (alphabet_size > 0 && u >= 0x80 && u < 0x80 + alphabet_size) { + u = alphabet[u - 0x80]; + } vstr_add_char(vstr, u); } +// The Huffman table used for the next symbol depends on the class of the last +// byte written. Classifying the last byte is the same as classifying the last +// code point: every byte of a multi-byte UTF-8 sequence is >= 0x80. +// This must match base_class() in py/maketranslationdata.py. +static uint8_t base_class(uint8_t last) { + if (last == ' ') { + return 0; + } + if (last >= 'a' && last <= 'z') { + return 1; + } + if (last >= 'A' && last <= 'Z') { + return 2; + } + if (last >= '0' && last <= '9') { + return 3; + } + if (last == '%') { + return 5; + } + if (last >= 0x80) { + return 6; + } + return 4; +} + uint16_t decompress_length(mp_rom_error_text_t compressed) { #ifndef NO_QSTR #if (compress_max_length_bits <= 8) @@ -99,20 +125,25 @@ static void decompress_vstr(mp_rom_error_text_t compressed, vstr_t *decompressed size_t alloc = decompressed->alloc - 1; // Stop one early because the last byte is always NULL. for (; decompressed->len < alloc;) { + // The start of a string is classed like a space. + uint8_t last = decompressed->len ? (uint8_t)decompressed->buf[decompressed->len - 1] : ' '; + uint8_t cls = class_map[base_class(last)]; + const uint8_t *len_row = lengths + cls * LENGTHS_ROW; + uint32_t bits = 0; uint8_t bit_length = 0; - uint32_t max_code = lengths[0]; - uint32_t searched_length = lengths[0]; + uint32_t max_code = len_row[0]; + uint32_t searched_length = len_row[0]; while (true) { bits = (bits << 1) | next_bit(&b); bit_length += 1; if (max_code > 0 && bits < max_code) { break; } - max_code = (max_code << 1) + lengths[bit_length]; - searched_length += lengths[bit_length]; + max_code = (max_code << 1) + len_row[bit_length]; + searched_length += len_row[bit_length]; } - int v = values[searched_length + bits - max_code]; + unsigned v = values[values_offset[cls] + searched_length + bits - max_code]; if (v == 1) { qstr q = get_nbits(&b, translation_qstr_bits); vstr_add_str(decompressed, qstr_str(q)); From 54e06ec4489eb9f83cf14323fe500cbf50b925f8 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Sun, 13 Sep 2026 23:06:56 -0400 Subject: [PATCH 2/4] Name the translation classes and escape symbols Add `translation_class_t` and `translation_symbol_t` enums to `compressed_string.h` and use the names in `base_class()`, the decode loop, and the matching Python constants in `maketranslationdata.py`, instead of bare 0..6 and 1. No change to generated data or firmware. Co-Authored-By: Claude Fable 5.1 --- py/maketranslationdata.py | 38 ++++++++++++------- .../shared/translate/compressed_string.h | 18 +++++++++ supervisor/shared/translate/translate.c | 18 ++++----- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/py/maketranslationdata.py b/py/maketranslationdata.py index 19bcb0970c5..40861283885 100644 --- a/py/maketranslationdata.py +++ b/py/maketranslationdata.py @@ -73,19 +73,29 @@ '"': '\\"', } -# Reserved symbol values. 2 and 3 are unused for now but must not appear in -# translated text either. -QSTR_ESC = "\1" +# Symbol values with a special meaning, as characters. Must match +# translation_symbol_t in supervisor/shared/translate/compressed_string.h. +# 2 and 3 are unused for now but must not appear in translated text either. +SYMBOL_QSTR = "\1" RESERVED_CHARS = {"\1", "\2", "\3"} # The first non-ASCII symbol value in the dense alphabet. ALPHABET_BASE = 0x80 # Huffman codes are chosen per "class" of the previously decoded character. -# These base classes are fixed in the C decoder (translate.c, base_class()); +# These base classes are fixed in the C decoder (translate.c, base_class()) and +# must match translation_class_t in supervisor/shared/translate/compressed_string.h; # the generator collapses them with class_map[] into TRANSLATION_CLASSES tables. +CLASS_START_OR_SPACE = 0 +CLASS_LOWER = 1 +CLASS_UPPER = 2 +CLASS_DIGIT = 3 +CLASS_OTHER = 4 +CLASS_PERCENT = 5 +CLASS_NON_ASCII = 6 NUM_BASE_CLASSES = 7 -# Presets to try, as class_map. Each maps a base class to a table index. +# Presets to try, as class_map. Each maps a base class (in the order above) to +# a table index. CLASS_PRESETS = ( (0, 0, 0, 0, 0, 0, 0), # a single table: the tables cost more than they save (0, 1, 1, 2, 2, 2, 1), # space, letter, other @@ -99,19 +109,19 @@ def base_class(c): Computed on the (possibly remapped) character: remapped characters are >= 0x80 exactly when the original is, which is all the classification looks at.""" if c is None or c == " ": - return 0 + return CLASS_START_OR_SPACE o = ord(c) if 0x61 <= o <= 0x7A: - return 1 + return CLASS_LOWER if 0x41 <= o <= 0x5A: - return 2 + return CLASS_UPPER if 0x30 <= o <= 0x39: - return 3 + return CLASS_DIGIT if o == 0x25: # '%' - return 5 + return CLASS_PERCENT if o >= 0x80: - return 6 - return 4 + return CLASS_NON_ASCII + return CLASS_OTHER # this must match the equivalent function in qstr.c @@ -225,7 +235,7 @@ def token_len(token): def token_symbol(token): """The Huffman symbol a token is coded as (qstrs share one escape symbol).""" - return QSTR_ESC if is_qstr(token) else token + return SYMBOL_QSTR if is_qstr(token) else token def code_lengths(counter): @@ -386,7 +396,7 @@ def compute_huffman_coding(qstrs, translation_name, translations, f, compression for t in texts: for atom in extractor.iter(t): if atom in qstrs: - atom = QSTR_ESC + atom = SYMBOL_QSTR counter[atom] += 1 cb = huffman.codebook(counter.items()) lengths = sorted(dict((v, len(cb[k])) for k, v in counter.items()).items()) diff --git a/supervisor/shared/translate/compressed_string.h b/supervisor/shared/translate/compressed_string.h index 5d1bc260c76..c60eb50cedc 100644 --- a/supervisor/shared/translate/compressed_string.h +++ b/supervisor/shared/translate/compressed_string.h @@ -78,6 +78,24 @@ typedef struct compressed_string { const uint8_t tail[]; } const *mp_rom_error_text_t; +// Class of the previously decoded byte, which selects the Huffman table for +// the next symbol. Must match base_class() in py/maketranslationdata.py. +typedef enum { + CLASS_START_OR_SPACE = 0, + CLASS_LOWER = 1, + CLASS_UPPER = 2, + CLASS_DIGIT = 3, + CLASS_OTHER = 4, + CLASS_PERCENT = 5, + CLASS_NON_ASCII = 6, +} translation_class_t; + +// Symbol values with a special meaning; every other value is a character or a +// dictionary word. Must match py/maketranslationdata.py. +typedef enum { + SYMBOL_QSTR = 1, // followed by translation_qstr_bits bits of qstr index +} translation_symbol_t; + // Return the compressed, translated version of a source string // Usually, due to LTO, this is optimized into a load of a constant // pointer. diff --git a/supervisor/shared/translate/translate.c b/supervisor/shared/translate/translate.c index cc40d06e34a..e9982991af0 100644 --- a/supervisor/shared/translate/translate.c +++ b/supervisor/shared/translate/translate.c @@ -60,26 +60,26 @@ static void put_utf8(vstr_t *vstr, unsigned u) { // byte written. Classifying the last byte is the same as classifying the last // code point: every byte of a multi-byte UTF-8 sequence is >= 0x80. // This must match base_class() in py/maketranslationdata.py. -static uint8_t base_class(uint8_t last) { +static translation_class_t base_class(uint8_t last) { if (last == ' ') { - return 0; + return CLASS_START_OR_SPACE; } if (last >= 'a' && last <= 'z') { - return 1; + return CLASS_LOWER; } if (last >= 'A' && last <= 'Z') { - return 2; + return CLASS_UPPER; } if (last >= '0' && last <= '9') { - return 3; + return CLASS_DIGIT; } if (last == '%') { - return 5; + return CLASS_PERCENT; } if (last >= 0x80) { - return 6; + return CLASS_NON_ASCII; } - return 4; + return CLASS_OTHER; } uint16_t decompress_length(mp_rom_error_text_t compressed) { @@ -144,7 +144,7 @@ static void decompress_vstr(mp_rom_error_text_t compressed, vstr_t *decompressed searched_length += len_row[bit_length]; } unsigned v = values[values_offset[cls] + searched_length + bits - max_code]; - if (v == 1) { + if (v == SYMBOL_QSTR) { qstr q = get_nbits(&b, translation_qstr_bits); vstr_add_str(decompressed, qstr_str(q)); } else { From 8381f89ca95236c31e2f5373644db698d7f20d08 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Sun, 13 Sep 2026 23:24:31 -0400 Subject: [PATCH 3/4] maketranslationdata.py: name the classes in CLASS_PRESETS Write each preset as the list of Huffman tables to build, each naming the base classes that share it, and derive `class_map[]` with `class_map_for()` instead of spelling out index tuples. Generated output is unchanged. Co-Authored-By: Claude Fable 5.1 --- py/maketranslationdata.py | 60 ++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/py/maketranslationdata.py b/py/maketranslationdata.py index 40861283885..e4debe889ab 100644 --- a/py/maketranslationdata.py +++ b/py/maketranslationdata.py @@ -94,16 +94,61 @@ CLASS_PERCENT = 5 CLASS_NON_ASCII = 6 NUM_BASE_CLASSES = 7 -# Presets to try, as class_map. Each maps a base class (in the order above) to -# a table index. +# Presets to try. Each is the list of Huffman tables to build, each table +# naming the base classes that share it. class_map_for() turns one into the +# class_map[] the decoder uses. CLASS_PRESETS = ( - (0, 0, 0, 0, 0, 0, 0), # a single table: the tables cost more than they save - (0, 1, 1, 2, 2, 2, 1), # space, letter, other - (0, 1, 2, 3, 4, 5, 1), # non-ASCII shares the lowercase table - (0, 1, 2, 3, 4, 5, 6), # non-ASCII has its own table + # one table: the tables cost more than they save + [ + [ + CLASS_START_OR_SPACE, + CLASS_LOWER, + CLASS_UPPER, + CLASS_DIGIT, + CLASS_OTHER, + CLASS_PERCENT, + CLASS_NON_ASCII, + ] + ], + # space, letter, other + [ + [CLASS_START_OR_SPACE], + [CLASS_LOWER, CLASS_UPPER, CLASS_NON_ASCII], + [CLASS_DIGIT, CLASS_OTHER, CLASS_PERCENT], + ], + # non-ASCII shares the lowercase table + [ + [CLASS_START_OR_SPACE], + [CLASS_LOWER, CLASS_NON_ASCII], + [CLASS_UPPER], + [CLASS_DIGIT], + [CLASS_OTHER], + [CLASS_PERCENT], + ], + # non-ASCII has its own table + [ + [CLASS_START_OR_SPACE], + [CLASS_LOWER], + [CLASS_UPPER], + [CLASS_DIGIT], + [CLASS_OTHER], + [CLASS_PERCENT], + [CLASS_NON_ASCII], + ], ) +def class_map_for(preset): + """class_map[] for a preset: base class -> index of the table it uses.""" + class_map = [None] * NUM_BASE_CLASSES + for table, classes in enumerate(preset): + for cls in classes: + assert class_map[cls] is None, f"class {cls} in two tables" + class_map[cls] = table + assert None not in class_map, "every base class needs a table" + return tuple(class_map) + + def base_class(c): """Class of the character preceding the next symbol; None means start of string. Computed on the (possibly remapped) character: remapped characters are >= 0x80 @@ -507,7 +552,8 @@ def table_bytes(counts, used_words): return n best = None - for class_map in CLASS_PRESETS: + for preset in CLASS_PRESETS: + class_map = class_map_for(preset) tokens = greedy_tokens counts = tally(texts, tokens, class_map) lens = {cls: code_lengths(cn) for cls, cn in counts.items()} From 58613db35587ed2f2407302d519913f258aa974c Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Sun, 13 Sep 2026 18:08:24 -0400 Subject: [PATCH 4/4] Compressed translations: escape rare characters instead of 16-bit tables Translations with more than 127 distinct non-ASCII characters (ja, ko) used `uint16_t` for every `values[]` and `words[]` entry, doubling their tables for the sake of a few hundred kanji and hangul, most used once or twice. Keep only the most frequent non-ASCII characters in the dense 8-bit alphabet (the generator tries 32, 48, 64 and 80 and keeps the smallest result) and escape the rest: symbol 2 followed by an index into a small `uint16_t rare_chars[]` for characters used more than once, symbol 3 followed by a raw 16-bit code point for characters used once. Dictionary words cannot contain escaped characters. The 16-bit table mode and `translation_requires_uint16` are gone; `mchar_t` is replaced by `uint8_t`. Languages whose alphabet already fit are byte-identical to before, and the two decoder branches fold away for them. On metro_m4_express: ja +708 bytes free, ko +820, fr unchanged. Verified on the unix coverage port (979 tests) and on a Metro M4 Express in ja. Co-Authored-By: Claude Fable 5.1 --- py/maketranslationdata.py | 372 +++++++++++------- .../shared/translate/compressed_string.h | 26 +- supervisor/shared/translate/translate.c | 10 +- 3 files changed, 256 insertions(+), 152 deletions(-) diff --git a/py/maketranslationdata.py b/py/maketranslationdata.py index e4debe889ab..22b0ed1eeb9 100644 --- a/py/maketranslationdata.py +++ b/py/maketranslationdata.py @@ -75,9 +75,10 @@ # Symbol values with a special meaning, as characters. Must match # translation_symbol_t in supervisor/shared/translate/compressed_string.h. -# 2 and 3 are unused for now but must not appear in translated text either. SYMBOL_QSTR = "\1" -RESERVED_CHARS = {"\1", "\2", "\3"} +SYMBOL_RARE_INDEX = "\2" +SYMBOL_RARE_RAW = "\3" +RESERVED_CHARS = {SYMBOL_QSTR, SYMBOL_RARE_INDEX, SYMBOL_RARE_RAW} # The first non-ASCII symbol value in the dense alphabet. ALPHABET_BASE = 0x80 @@ -235,9 +236,16 @@ def iter_substrings(s, minlen, maxlen): yield s[begin : begin + n] -# Languages whose non-ASCII alphabet does not fit the dense 8-bit alphabet and -# therefore use 16-bit table entries. -translation_requires_uint16 = {"ja", "ko"} +# Non-ASCII characters that do not get a dense symbol are "rare". While the text +# is being processed they are remapped into two private-use ranges so that they +# stay single characters: those used more than once become an index into +# rare_chars[], those used once are coded as a raw 16-bit code point. +RARE_IDX_BASE = 0xF0000 +RARE_RAW_BASE = 0x100000 +RARE_SPLIT = re.compile("[\U000f0000-\U0010ffff]") +# Dense alphabet sizes to try when a language has more non-ASCII characters +# than fit; the rest of the code points go to the dictionary. +DENSE_ALPHABET_TRIES = (32, 48, 64, 80) @dataclass @@ -253,14 +261,16 @@ class EncodingTable: # Dictionary words, in remapped characters, sorted by length. words: list word_start: int - # Dense alphabet: index -> original character. Empty in uint16 mode. + # Dense alphabet: index -> original character. alphabet: list + # Rare characters coded by index -> original character. + rare_idx_chars: list + rare_index_bits: int # Original character -> remapped character (identity for ASCII). remap: dict translation_qstr_bits: int qstrs: dict qstrs_inv: dict - values_type: str # Remapped text -> token list used to encode it. A token is a str (character # or word) or a ("q", qstr) tuple. tokens: dict = field(default_factory=dict) @@ -274,13 +284,31 @@ def is_qstr(token): return isinstance(token, tuple) +def is_rare(token): + return len(token) == 1 and ord(token) >= RARE_IDX_BASE + + def token_len(token): return len(token[1]) if is_qstr(token) else len(token) def token_symbol(token): - """The Huffman symbol a token is coded as (qstrs share one escape symbol).""" - return SYMBOL_QSTR if is_qstr(token) else token + """The Huffman symbol a token is coded as. qstrs and rare characters are + coded as an escape symbol followed by a fixed number of bits.""" + if is_qstr(token): + return SYMBOL_QSTR + if is_rare(token): + return SYMBOL_RARE_RAW if ord(token) >= RARE_RAW_BASE else SYMBOL_RARE_INDEX + return token + + +def token_extra_bits(token, qstr_bits, rare_index_bits): + """Fixed-width bits that follow the token's Huffman symbol.""" + if is_qstr(token): + return qstr_bits + if is_rare(token): + return 16 if ord(token) >= RARE_RAW_BASE else rare_index_bits + return 0 def code_lengths(counter): @@ -314,7 +342,14 @@ def canonical_codes(lengths): def parse_optimal( - text, lens, class_map, words_by_first, qstrs_by_first, qstr_bits, unknown_len=32 + text, + lens, + class_map, + words_by_first, + qstrs_by_first, + qstr_bits, + rare_index_bits, + unknown_len=32, ): """Tokenize text with the fewest bits given per-class code lengths. Symbols missing from a class table are allowed at a penalty so a parse always exists.""" @@ -336,8 +371,7 @@ def parse_optimal( cands.append(("q", q)) for tok in cands: cost = table.get(token_symbol(tok), unknown_len) - if is_qstr(tok): - cost += qstr_bits + cost += token_extra_bits(tok, qstr_bits, rare_index_bits) j = i + token_len(tok) if best[i] + cost < best[j]: best[j] = best[i] + cost @@ -364,69 +398,21 @@ def tally(texts, tokens, class_map): return counts -def encoded_bits(text, tokens, lens, class_map, qstr_bits): +def encoded_bits(text, tokens, lens, class_map, qstr_bits, rare_index_bits): bits = 0 pos = 0 for tok in tokens: cls = class_map[base_class(text[pos - 1] if pos else None)] bits += lens[cls][token_symbol(tok)] - if is_qstr(tok): - bits += qstr_bits + bits += token_extra_bits(tok, qstr_bits, rare_index_bits) pos += token_len(tok) return bits -def compute_huffman_coding(qstrs, translation_name, translations, f, compression_level): - # possible future improvement: some languages are better when consider len(k) > 2. try both? - qstrs = dict((k, v) for k, v in qstrs.items() if len(k) > 3) - qstr_strs = list(qstrs.keys()) - original_texts = [t[1] for t in translations] +def find_words(texts, qstrs, qstr_strs, max_words): + """Greedy dictionary search: repeatedly add the 2- to 11-gram estimated to + save the most bits until the dictionary is full or nothing pays off.""" words = [] - - for text in original_texts: - bad = RESERVED_CHARS.intersection(text) - if bad: - raise ValueError(f"Translation contains reserved character {bad!r}: {text!r}") - - translation_name = translation_name.split("/")[-1].split(".")[0] - - # Dense alphabet: non-ASCII characters are renumbered from 0x80 by decreasing - # frequency so that all symbols fit in 8 bits. If there are too many of them, - # fall back to raw 16-bit code points. - hi_count = collections.Counter(c for t in original_texts for c in t if ord(c) >= 0x80) - if len(hi_count) <= 0x7F: - alphabet = [c for c, _ in hi_count.most_common()] - remap = {c: chr(ALPHABET_BASE + i) for i, c in enumerate(alphabet)} - word_start = ALPHABET_BASE + len(alphabet) - max_words = 0x100 - word_start - values_type = "uint8_t" - else: - if translation_name not in translation_requires_uint16: - raise ValueError( - f"Translation {translation_name} expected to fit in 8 bits but required 16 bits" - ) - alphabet = [] - remap = {} - # Words take the unused code points from 0x80 up to the lowest one in use. - end_unused = min([0xFF] + [o for o in map(ord, hi_count) if o < 0xFF]) - word_start = ALPHABET_BASE - max_words = end_unused - word_start - values_type = "uint16_t" - if compression_level < 5: - max_words = 0 - bits_per_codepoint = 16 if values_type == "uint16_t" else 8 - - texts = [remap_text(t, remap) for t in original_texts] - - # Prune the qstrs to only those that appear in the texts - qstr_counters = collections.Counter() - qstr_extractor = TextSplitter(qstr_strs) - for t in texts: - for qstr in qstr_extractor.iter(t): - if qstr in qstr_strs: - qstr_counters[qstr] += 1 - qstr_strs = list(qstr_counters.keys()) - while len(words) < max_words: # Until the dictionary is filled to capacity, use a heuristic to find # the best "word" (2- to 11-gram) to add to it. @@ -477,15 +463,18 @@ def est_len(occ): # The difference between the two is the estimated net savings, in bits. def est_net_savings(s, occ): savings = occ * (bit_length(s) - est_len(occ)) - cost = len(s) * bits_per_codepoint + 24 + cost = len(s) * 8 + 24 return savings - cost counter = collections.Counter() for t in texts: for found, word in extractor.iter_words(t): if not found: - for substr in iter_substrings(word, minlen=2, maxlen=11): - counter[substr] += 1 + # Words are stored as 8-bit symbols, so they cannot contain + # rare (escaped) characters: split around them. + for piece in RARE_SPLIT.split(word): + for substr in iter_substrings(piece, minlen=2, maxlen=11): + counter[substr] += 1 # Score the candidates we found. This is a semi-empirical formula that # attempts to model the number of bits saved as closely as possible. @@ -512,68 +501,155 @@ def est_net_savings(s, occ): word = scores[0][0] words.append(word) + return words - # Now that the dictionary is fixed, find the tokenization of each string that - # takes the fewest bits, with Huffman codes chosen per class of the previous - # character. Code lengths and tokenization depend on each other, so iterate - # a few rounds from the greedy tokenization; the final tokenization and the - # codes built from it are what get emitted, so they are always consistent. - use_qstrs = compression_level > 3 - dp_qstrs = qstr_strs if use_qstrs else [] - # Upper bound on the qstr index width while parsing; the real width is computed below. - est_qstr_bits = max([0] + [qstrs[q] for q in dp_qstrs]).bit_length() - words_by_first = collections.defaultdict(list) - for w in words: - words_by_first[w[0]].append(w) - qstrs_by_first = collections.defaultdict(list) - for q in dp_qstrs: - qstrs_by_first[q[0]].append(q) - - greedy = TextSplitter(words + dp_qstrs) - greedy_tokens = {} - for t in texts: - greedy_tokens[t] = [("q", a) if a in qstrs else a for a in greedy.iter(t)] + +def compute_huffman_coding(qstrs, translation_name, translations, f, compression_level): + # possible future improvement: some languages are better when consider len(k) > 2. try both? + qstrs = dict((k, v) for k, v in qstrs.items() if len(k) > 3) + all_qstr_strs = list(qstrs.keys()) + original_texts = [t[1] for t in translations] + + for text in original_texts: + bad = RESERVED_CHARS.intersection(text) + if bad: + raise ValueError(f"Translation contains reserved character {bad!r}: {text!r}") + if any(ord(c) >= RARE_IDX_BASE for c in text): + raise ValueError(f"Translation contains private-use character: {text!r}") max_translation_encoded_length = max(len(t.encode("utf-8")) for t in original_texts) encoded_length_bits = max_translation_encoded_length.bit_length() + use_qstrs = compression_level > 3 - def table_bytes(counts, used_words): - nclasses = max(class_map) + 1 - row = max(max(code_lengths(cn).values()) for cn in counts.values()) + 1 - mchar = bits_per_codepoint // 8 - n = sum(len(cn) for cn in counts.values()) * mchar # values - n += nclasses * row # lengths - n += 2 * (nclasses + 1) # values_offset - n += NUM_BASE_CLASSES # class_map - n += sum(len(w) for w in used_words) * mchar # words - if used_words: - n += len(used_words[-1]) - len(used_words[0]) + 1 # wlencount - n += 2 * len(alphabet) - return n - - best = None - for preset in CLASS_PRESETS: - class_map = class_map_for(preset) - tokens = greedy_tokens - counts = tally(texts, tokens, class_map) - lens = {cls: code_lengths(cn) for cls, cn in counts.items()} - for _ in range(3): - tokens = { - t: parse_optimal(t, lens, class_map, words_by_first, qstrs_by_first, est_qstr_bits) - for t in texts - } + # Non-ASCII characters by decreasing frequency. The most frequent ones are + # renumbered from 0x80 (the dense alphabet), so that all symbols fit in 8 + # bits; the rest are escaped. + hi_count = collections.Counter(c for t in original_texts for c in t if ord(c) >= 0x80) + hi_chars = [c for c, _ in hi_count.most_common()] + if len(hi_chars) <= 0x7F: + dense_tries = (len(hi_chars),) + else: + dense_tries = DENSE_ALPHABET_TRIES + + def encode(dense_count): + alphabet = hi_chars[:dense_count] + rare = hi_chars[dense_count:] + rare_idx_chars = [c for c in rare if hi_count[c] > 1] + rare_raw_chars = [c for c in rare if hi_count[c] == 1] + rare_index_bits = (len(rare_idx_chars) - 1).bit_length() if rare_idx_chars else 0 + remap = {c: chr(ALPHABET_BASE + i) for i, c in enumerate(alphabet)} + remap.update({c: chr(RARE_IDX_BASE + i) for i, c in enumerate(rare_idx_chars)}) + remap.update({c: chr(RARE_RAW_BASE + i) for i, c in enumerate(rare_raw_chars)}) + word_start = ALPHABET_BASE + len(alphabet) + max_words = 0x100 - word_start if compression_level >= 5 else 0 + + texts = [remap_text(t, remap) for t in original_texts] + + # Prune the qstrs to only those that appear in the texts + qstr_counters = collections.Counter() + qstr_extractor = TextSplitter(all_qstr_strs) + for t in texts: + for qstr in qstr_extractor.iter(t): + if qstr in qstrs: + qstr_counters[qstr] += 1 + qstr_strs = list(qstr_counters.keys()) + + words = find_words(texts, qstrs, qstr_strs, max_words) + + # Now that the dictionary is fixed, find the tokenization of each string + # that takes the fewest bits, with Huffman codes chosen per class of the + # previous character. Code lengths and tokenization depend on each other, + # so iterate a few rounds from the greedy tokenization; the final + # tokenization and the codes built from it are what get emitted, so they + # are always consistent. + dp_qstrs = qstr_strs if use_qstrs else [] + # Upper bound on the qstr index width while parsing; the real width is computed below. + est_qstr_bits = max([0] + [qstrs[q] for q in dp_qstrs]).bit_length() + words_by_first = collections.defaultdict(list) + for w in words: + words_by_first[w[0]].append(w) + qstrs_by_first = collections.defaultdict(list) + for q in dp_qstrs: + qstrs_by_first[q[0]].append(q) + + greedy = TextSplitter(words + dp_qstrs) + greedy_tokens = {} + for t in texts: + greedy_tokens[t] = [("q", a) if a in qstrs else a for a in greedy.iter(t)] + + def table_bytes(class_map, counts, used_words): + nclasses = max(class_map) + 1 + row = max(max(code_lengths(cn).values()) for cn in counts.values()) + 1 + n = sum(len(cn) for cn in counts.values()) # values + n += nclasses * row # lengths + n += 2 * (nclasses + 1) # values_offset + n += NUM_BASE_CLASSES # class_map + n += sum(len(w) for w in used_words) # words + if used_words: + n += len(used_words[-1]) - len(used_words[0]) + 1 # wlencount + n += 2 * len(alphabet) + n += 2 * len(rare_idx_chars) + return n + + best = None + for preset in CLASS_PRESETS: + class_map = class_map_for(preset) + tokens = greedy_tokens counts = tally(texts, tokens, class_map) lens = {cls: code_lengths(cn) for cls, cn in counts.items()} - used_symbols = set().union(*counts.values()) - used_words = sorted((w for w in words if w in used_symbols), key=len) - size = table_bytes(counts, used_words) + sum( - (encoded_length_bits + encoded_bits(t, tokens[t], lens, class_map, est_qstr_bits) + 7) - // 8 - for t in texts + for _ in range(3): + tokens = { + t: parse_optimal( + t, + lens, + class_map, + words_by_first, + qstrs_by_first, + est_qstr_bits, + rare_index_bits, + ) + for t in texts + } + counts = tally(texts, tokens, class_map) + lens = {cls: code_lengths(cn) for cls, cn in counts.items()} + used_symbols = set().union(*counts.values()) + used_words = sorted((w for w in words if w in used_symbols), key=len) + size = table_bytes(class_map, counts, used_words) + sum( + ( + encoded_length_bits + + encoded_bits(t, tokens[t], lens, class_map, est_qstr_bits, rare_index_bits) + + 7 + ) + // 8 + for t in texts + ) + if best is None or size < best[0]: + best = (size, class_map, tokens, lens, used_words) + size, class_map, tokens, lens, words = best + return size, dict( + class_map=class_map, + tokens=tokens, + lens=lens, + words=words, + alphabet=alphabet, + rare_idx_chars=rare_idx_chars, + rare_raw_chars=rare_raw_chars, + rare_index_bits=rare_index_bits, + remap=remap, + word_start=word_start, ) - if best is None or size < best[0]: - best = (size, class_map, tokens, counts, lens, used_words) - size, class_map, tokens, counts, lens, words = best + + size, r = min((encode(k) for k in dense_tries), key=lambda x: x[0]) + class_map = r["class_map"] + tokens = r["tokens"] + lens = r["lens"] + words = r["words"] + alphabet = r["alphabet"] + rare_idx_chars = r["rare_idx_chars"] + rare_raw_chars = r["rare_raw_chars"] + rare_index_bits = r["rare_index_bits"] + remap = r["remap"] + word_start = r["word_start"] used_qstr = 0 for toks in tokens.values(): @@ -604,12 +680,13 @@ def symbol_value(sym): lengths_row = max(1, max(len(r) for r in lengths_rows)) lengths_rows = [r + [0] * (lengths_row - len(r)) for r in lengths_rows] assert all(len(code) >= 1 for canon in canonical for code in canon.values()) + assert all(v < 0x100 for vs in values for v in vs) + unmap = {v: k for k, v in remap.items()} f.write(f"// # words {len(words)}\n") - f.write( - "// words {}\n".format([remap_text(w, {v: k for k, v in remap.items()}) for w in words]) - ) + f.write("// words {}\n".format([remap_text(w, unmap) for w in words])) f.write(f"// # alphabet {len(alphabet)}\n") + f.write(f"// # rare chars by index {len(rare_idx_chars)}, raw {len(rare_raw_chars)}\n") f.write(f"// class_map {class_map} tables {nclasses}\n") for cls in range(nclasses): f.write(f"// class {cls}: {len(values[cls])} symbols, lengths {lengths_rows[cls]}\n") @@ -618,7 +695,6 @@ def symbol_value(sym): minlen = len(words[0]) if words else 0 wlencount = [len([None for w in words if len(w) == l]) for l in range(minlen, maxlen + 1)] - f.write("typedef {} mchar_t;\n".format(values_type)) f.write("#define compress_max_length_bits ({})\n".format(encoded_length_bits)) f.write("#define TRANSLATION_CLASSES {}\n".format(nclasses)) f.write("#define LENGTHS_ROW {}\n".format(lengths_row)) @@ -641,14 +717,22 @@ def symbol_value(sym): ) ) f.write( - "const mchar_t values[] = {{ {} }};\n".format(", ".join(str(x) for v in values for x in v)) + "const uint8_t values[] = {{ {} }};\n".format(", ".join(str(x) for v in values for x in v)) ) f.write("#define alphabet_size {}\n".format(len(alphabet))) f.write( "const uint16_t alphabet[] = {{ {} }};\n".format(", ".join(str(ord(c)) for c in alphabet)) ) + f.write("#define rare_index_count {}\n".format(len(rare_idx_chars))) + f.write("#define rare_index_bits {}\n".format(rare_index_bits)) + f.write("#define rare_raw_count {}\n".format(len(rare_raw_chars))) f.write( - "const mchar_t words[] = {{ {} }};\n".format( + "const uint16_t rare_chars[] = {{ {} }};\n".format( + ", ".join(str(ord(c)) for c in rare_idx_chars) + ) + ) + f.write( + "const uint8_t words[] = {{ {} }};\n".format( ", ".join(str(ord(c)) for w in words for c in w) ) ) @@ -669,11 +753,12 @@ def symbol_value(sym): words, word_start, alphabet, + rare_idx_chars, + rare_index_bits, remap, translation_qstr_bits, qstrs, qstrs_inv, - values_type, tokens, ) @@ -702,15 +787,17 @@ def getnbits(n): length = getnbits(encoded_length_bits) decoded = 0 - def emit(u): + def emit_char(c): nonlocal last, decoded - if ALPHABET_BASE <= u < ALPHABET_BASE + alphabet_size: - c = et.alphabet[u - ALPHABET_BASE] - else: - c = chr(u) dec.append(c) decoded += len(c.encode("utf-8")) - last = c + last = c[-1] + + def emit(u): + if ALPHABET_BASE <= u < ALPHABET_BASE + alphabet_size: + emit_char(et.alphabet[u - ALPHABET_BASE]) + else: + emit_char(chr(u)) while decoded < length: cls = et.class_map[base_class(last)] @@ -729,10 +816,11 @@ def emit(u): v = et.values[cls][searched_length + bits - max_code] if v == 1: qstr_idx = getnbits(et.translation_qstr_bits) - s = et.qstrs_inv[qstr_idx] - dec.append(s) - decoded += len(s.encode("utf-8")) - last = s[-1] + emit_char(et.qstrs_inv[qstr_idx]) + elif v == 2: + emit_char(et.rare_idx_chars[getnbits(et.rare_index_bits)]) + elif v == 3: + emit_char(chr(getnbits(16))) elif et.word_start <= v <= word_end: for c in et.words[v - et.word_start]: emit(ord(c)) @@ -748,6 +836,7 @@ def compress(encoding_table, decompressed, encoded_length_bits, len_translation_ et = encoding_table text = remap_text(decompressed, et.remap) tokens = et.tokens[text] + unmap = {v: k for k, v in et.remap.items()} enc = 1 @@ -772,6 +861,11 @@ def put_code(enc, cls, sym): enc = put_code(enc, cls, token_symbol(tok)) if is_qstr(tok): enc = put_bits(enc, et.qstrs[tok[1]], et.translation_qstr_bits) + elif is_rare(tok): + if ord(tok) >= RARE_RAW_BASE: + enc = put_bits(enc, ord(unmap[tok]), 16) + else: + enc = put_bits(enc, ord(tok) - RARE_IDX_BASE, et.rare_index_bits) pos += token_len(tok) while enc.bit_length() % 8 != 1: diff --git a/supervisor/shared/translate/compressed_string.h b/supervisor/shared/translate/compressed_string.h index c60eb50cedc..7bfe2000b73 100644 --- a/supervisor/shared/translate/compressed_string.h +++ b/supervisor/shared/translate/compressed_string.h @@ -29,16 +29,18 @@ // The per-table code-length counts are rows of LENGTHS_ROW bytes in // lengths[], and the symbols of table n are values[values_offset[n]..]. // -// - Symbol values 0..0x7F are ASCII (with 1 reserved for QSTR escapes, -// below, and 2 and 3 reserved for future escapes). Values from 0x80 up to -// 0x80 + alphabet_size - 1 are the non-ASCII characters used by the -// translation, renumbered by decreasing frequency; alphabet[] maps them -// back to Unicode code points. When a translation uses more than 127 -// distinct non-ASCII characters this is not possible, alphabet_size is 0 -// and the symbols are raw 16-bit code points instead (mchar_t is then -// uint16_t rather than uint8_t; it's very beneficial for mchar_t to be 8 -// bits!). At present, no translation requires code points outside the -// BMP, so this is adequate. +// - Symbol values 0..0x7F are ASCII (with 1, 2 and 3 reserved for the +// escapes below). Values from 0x80 up to 0x80 + alphabet_size - 1 are the +// most frequent non-ASCII characters used by the translation, renumbered +// by decreasing frequency; alphabet[] maps them back to Unicode code +// points. Most translations fit their whole non-ASCII repertoire in the +// alphabet. Those that don't (ja, ko) keep the 32 to 80 most frequent +// characters there and escape the rest: value 2 is followed by a +// rare_index_bits-bit index into rare_chars[] (characters used more than +// once), value 3 by a raw 16-bit code point (characters used once, which +// are cheaper without a table entry). All symbols are therefore 8 bits. +// At present, no translation requires code points outside the BMP, so +// this is adequate. // // - Symbol values from word_start to word_end stand for dictionary entries // in a dictionary of up to 256 - word_start entries. The dictionary @@ -93,7 +95,9 @@ typedef enum { // Symbol values with a special meaning; every other value is a character or a // dictionary word. Must match py/maketranslationdata.py. typedef enum { - SYMBOL_QSTR = 1, // followed by translation_qstr_bits bits of qstr index + SYMBOL_QSTR = 1, // followed by translation_qstr_bits bits of qstr index + SYMBOL_RARE_INDEX = 2, // followed by rare_index_bits bits of rare_chars[] index + SYMBOL_RARE_RAW = 3, // followed by a 16-bit code point } translation_symbol_t; // Return the compressed, translated version of a source string diff --git a/supervisor/shared/translate/translate.c b/supervisor/shared/translate/translate.c index e9982991af0..d0c05dda6e5 100644 --- a/supervisor/shared/translate/translate.c +++ b/supervisor/shared/translate/translate.c @@ -23,7 +23,7 @@ void serial_write_compressed(mp_rom_error_text_t compressed) { mp_printf(MP_PYTHON_PRINTER, "%S", compressed); } -static void get_word(int n, const mchar_t **pos, const mchar_t **end) { +static void get_word(int n, const uint8_t **pos, const uint8_t **end) { int len = minlen; int i = 0; *pos = words; @@ -39,7 +39,7 @@ static void get_word(int n, const mchar_t **pos, const mchar_t **end) { static void put_utf8(vstr_t *vstr, unsigned u) { if (word_start <= u && u <= word_end) { - const mchar_t *pos, *end; + const uint8_t *pos, *end; get_word(u - word_start, &pos, &end); // note that at present, entries in the words table are // guaranteed not to represent words themselves, so this adds @@ -147,6 +147,12 @@ static void decompress_vstr(mp_rom_error_text_t compressed, vstr_t *decompressed if (v == SYMBOL_QSTR) { qstr q = get_nbits(&b, translation_qstr_bits); vstr_add_str(decompressed, qstr_str(q)); + } else if (rare_index_count > 0 && v == SYMBOL_RARE_INDEX) { + // A non-ASCII character outside the dense alphabet, by table index. + vstr_add_char(decompressed, rare_chars[get_nbits(&b, rare_index_bits)]); + } else if (rare_raw_count > 0 && v == SYMBOL_RARE_RAW) { + // A non-ASCII character used only once, as a raw code point. + vstr_add_char(decompressed, get_nbits(&b, 16)); } else { put_utf8(decompressed, v); }