diff --git a/py/maketranslationdata.py b/py/maketranslationdata.py index c3c3442e311..22b0ed1eeb9 100644 --- a/py/maketranslationdata.py +++ b/py/maketranslationdata.py @@ -6,13 +6,13 @@ supported CPython version are unintended. For documentation about the format of compressed translated strings, see -supervisor/shared/translate/translate.h +supervisor/shared/translate/compressed_string.h """ from __future__ import print_function import bisect -from dataclasses import dataclass +from dataclasses import dataclass, field import re import sys @@ -73,6 +73,102 @@ '"': '\\"', } +# Symbol values with a special meaning, as characters. Must match +# translation_symbol_t in supervisor/shared/translate/compressed_string.h. +SYMBOL_QSTR = "\1" +SYMBOL_RARE_INDEX = "\2" +SYMBOL_RARE_RAW = "\3" +RESERVED_CHARS = {SYMBOL_QSTR, SYMBOL_RARE_INDEX, SYMBOL_RARE_RAW} + +# The first non-ASCII symbol value in the dense alphabet. +ALPHABET_BASE = 0x80 + +# Huffman codes are chosen per "class" of the previously decoded character. +# These base classes are fixed in the C decoder (translate.c, base_class()) and +# must match translation_class_t in supervisor/shared/translate/compressed_string.h; +# the generator collapses them with class_map[] into TRANSLATION_CLASSES tables. +CLASS_START_OR_SPACE = 0 +CLASS_LOWER = 1 +CLASS_UPPER = 2 +CLASS_DIGIT = 3 +CLASS_OTHER = 4 +CLASS_PERCENT = 5 +CLASS_NON_ASCII = 6 +NUM_BASE_CLASSES = 7 +# Presets to try. Each is the list of Huffman tables to build, each table +# naming the base classes that share it. class_map_for() turns one into the +# class_map[] the decoder uses. +CLASS_PRESETS = ( + # one table: the tables cost more than they save + [ + [ + CLASS_START_OR_SPACE, + CLASS_LOWER, + CLASS_UPPER, + CLASS_DIGIT, + CLASS_OTHER, + CLASS_PERCENT, + CLASS_NON_ASCII, + ] + ], + # space, letter, other + [ + [CLASS_START_OR_SPACE], + [CLASS_LOWER, CLASS_UPPER, CLASS_NON_ASCII], + [CLASS_DIGIT, CLASS_OTHER, CLASS_PERCENT], + ], + # non-ASCII shares the lowercase table + [ + [CLASS_START_OR_SPACE], + [CLASS_LOWER, CLASS_NON_ASCII], + [CLASS_UPPER], + [CLASS_DIGIT], + [CLASS_OTHER], + [CLASS_PERCENT], + ], + # non-ASCII has its own table + [ + [CLASS_START_OR_SPACE], + [CLASS_LOWER], + [CLASS_UPPER], + [CLASS_DIGIT], + [CLASS_OTHER], + [CLASS_PERCENT], + [CLASS_NON_ASCII], + ], +) + + +def class_map_for(preset): + """class_map[] for a preset: base class -> index of the table it uses.""" + class_map = [None] * NUM_BASE_CLASSES + for table, classes in enumerate(preset): + for cls in classes: + assert class_map[cls] is None, f"class {cls} in two tables" + class_map[cls] = table + assert None not in class_map, "every base class needs a table" + return tuple(class_map) + + +def base_class(c): + """Class of the character preceding the next symbol; None means start of string. + Computed on the (possibly remapped) character: remapped characters are >= 0x80 + exactly when the original is, which is all the classification looks at.""" + if c is None or c == " ": + return CLASS_START_OR_SPACE + o = ord(c) + if 0x61 <= o <= 0x7A: + return CLASS_LOWER + if 0x41 <= o <= 0x5A: + return CLASS_UPPER + if 0x30 <= o <= 0x39: + return CLASS_DIGIT + if o == 0x25: # '%' + return CLASS_PERCENT + if o >= 0x80: + return CLASS_NON_ASCII + return CLASS_OTHER + # this must match the equivalent function in qstr.c def compute_hash(qstr, bytes_hash): @@ -140,100 +236,183 @@ def iter_substrings(s, minlen, maxlen): yield s[begin : begin + n] -translation_requires_uint16 = {"cs", "ja", "ko", "pl", "tr", "zh_Latn_pinyin"} +# Non-ASCII characters that do not get a dense symbol are "rare". While the text +# is being processed they are remapped into two private-use ranges so that they +# stay single characters: those used more than once become an index into +# rare_chars[], those used once are coded as a raw 16-bit code point. +RARE_IDX_BASE = 0xF0000 +RARE_RAW_BASE = 0x100000 +RARE_SPLIT = re.compile("[\U000f0000-\U0010ffff]") +# Dense alphabet sizes to try when a language has more non-ASCII characters +# than fit; the rest of the code points go to the dictionary. +DENSE_ALPHABET_TRIES = (32, 48, 64, 80) -def compute_unicode_offset(texts): - all_ch = set(" ".join(texts)) - ch_160 = sorted(c for c in all_ch if 160 <= ord(c) < 255) - ch_256 = sorted(c for c in all_ch if 255 < ord(c)) - if not ch_256: - return 0, 0 - min_256 = ord(min(ch_256)) - span = ord(max(ch_256)) - ord(min(ch_256)) + 1 +@dataclass +class EncodingTable: + # One entry per class table: list of symbols in canonical order. + values: list + # One entry per class table: list of code-length counts, padded to lengths_row. + lengths: list + lengths_row: int + class_map: tuple + # Per class table: symbol -> canonical code (string of '0'/'1'). + canonical: list + # Dictionary words, in remapped characters, sorted by length. + words: list + word_start: int + # Dense alphabet: index -> original character. + alphabet: list + # Rare characters coded by index -> original character. + rare_idx_chars: list + rare_index_bits: int + # Original character -> remapped character (identity for ASCII). + remap: dict + translation_qstr_bits: int + qstrs: dict + qstrs_inv: dict + # Remapped text -> token list used to encode it. A token is a str (character + # or word) or a ("q", qstr) tuple. + tokens: dict = field(default_factory=dict) - if ch_160: - max_160 = ord(max(ch_160)) + 1 - else: - max_160 = max(160, 255 - span) - if max_160 + span > 256: - return 0, 0 +def remap_text(text, remap): + return "".join(remap.get(c, c) for c in text) - offstart = max_160 - offset = min_256 - max_160 - return offstart, offset +def is_qstr(token): + return isinstance(token, tuple) -@dataclass -class EncodingTable: - values: object - lengths: object - words: object - canonical: object - extractor: object - apply_offset: object - remove_offset: object - translation_qstr_bits: int - qstrs: object - qstrs_inv: object +def is_rare(token): + return len(token) == 1 and ord(token) >= RARE_IDX_BASE -def compute_huffman_coding(qstrs, translation_name, translations, f, compression_level): - # possible future improvement: some languages are better when consider len(k) > 2. try both? - qstrs = dict((k, v) for k, v in qstrs.items() if len(k) > 3) - qstr_strs = list(qstrs.keys()) - texts = [t[1] for t in translations] - words = [] - start_unused = 0x80 - end_unused = 0xFF - max_ord = 0 - offstart, offset = compute_unicode_offset(texts) - - def apply_offset(c): - oc = ord(c) - if oc >= offstart: - oc += offset - return chr(oc) - - def remove_offset(c): - oc = ord(c) - if oc >= offstart: - oc = oc - offset - try: - return chr(oc) - except Exception as e: - raise ValueError(f"remove_offset {offstart=} {oc=}") from e +def token_len(token): + return len(token[1]) if is_qstr(token) else len(token) + + +def token_symbol(token): + """The Huffman symbol a token is coded as. qstrs and rare characters are + coded as an escape symbol followed by a fixed number of bits.""" + if is_qstr(token): + return SYMBOL_QSTR + if is_rare(token): + return SYMBOL_RARE_RAW if ord(token) >= RARE_RAW_BASE else SYMBOL_RARE_INDEX + return token - for text in texts: - for c in text: - c = remove_offset(c) - ord_c = ord(c) - max_ord = max(ord_c, max_ord) - if 0x80 <= ord_c < 0xFF: - end_unused = min(ord_c, end_unused) - max_words = end_unused - 0x80 - if compression_level < 5: - max_words = 0 - - bits_per_codepoint = 16 if max_ord > 255 else 8 - values_type = "uint16_t" if max_ord > 255 else "uint8_t" - translation_name = translation_name.split("/")[-1].split(".")[0] - if max_ord > 255 and translation_name not in translation_requires_uint16: - raise ValueError( - f"Translation {translation_name} expected to fit in 8 bits but required 16 bits" - ) - # Prune the qstrs to only those that appear in the texts - qstr_counters = collections.Counter() - qstr_extractor = TextSplitter(qstr_strs) - for t in texts: - for qstr in qstr_extractor.iter(t): - if qstr in qstr_strs: - qstr_counters[qstr] += 1 - qstr_strs = list(qstr_counters.keys()) +def token_extra_bits(token, qstr_bits, rare_index_bits): + """Fixed-width bits that follow the token's Huffman symbol.""" + if is_qstr(token): + return qstr_bits + if is_rare(token): + return 16 if ord(token) >= RARE_RAW_BASE else rare_index_bits + return 0 + +def code_lengths(counter): + """Huffman code lengths for a symbol counter. A lone symbol gets a 1-bit code.""" + if len(counter) == 1: + return {next(iter(counter)): 1} + cb = huffman.codebook(counter.items()) + return {k: len(v) for k, v in cb.items()} + + +def canonical_codes(lengths): + """Canonical Huffman codes from a symbol -> length dict. + Returns (values in canonical order, per-length counts, symbol -> code).""" + if not lengths: + return [], [], {} + values = [] + length_count = collections.Counter() + canonical = {} + renumbered = 0 + last_length = None + for atom, length in sorted(lengths.items(), key=lambda x: (x[1], x[0])): + values.append(atom) + length_count[length] += 1 + if last_length: + renumbered <<= length - last_length + canonical[atom] = "{0:0{width}b}".format(renumbered, width=length) + renumbered += 1 + last_length = length + counts = [length_count.get(i, 0) for i in range(1, max(length_count) + 2)] + return values, counts, canonical + + +def parse_optimal( + text, + lens, + class_map, + words_by_first, + qstrs_by_first, + qstr_bits, + rare_index_bits, + unknown_len=32, +): + """Tokenize text with the fewest bits given per-class code lengths. + Symbols missing from a class table are allowed at a penalty so a parse always exists.""" + n = len(text) + best = [math.inf] * (n + 1) + back = [None] * (n + 1) + best[0] = 0 + for i in range(n): + if best[i] == math.inf: + continue + table = lens[class_map[base_class(text[i - 1] if i else None)]] + c = text[i] + cands = [c] + for w in words_by_first.get(c, ()): + if text.startswith(w, i): + cands.append(w) + for q in qstrs_by_first.get(c, ()): + if text.startswith(q, i): + cands.append(("q", q)) + for tok in cands: + cost = table.get(token_symbol(tok), unknown_len) + cost += token_extra_bits(tok, qstr_bits, rare_index_bits) + j = i + token_len(tok) + if best[i] + cost < best[j]: + best[j] = best[i] + cost + back[j] = tok + out = [] + i = n + while i > 0: + tok = back[i] + out.append(tok) + i -= token_len(tok) + out.reverse() + return out + + +def tally(texts, tokens, class_map): + """Symbol counts per class table for the given tokenizations.""" + counts = collections.defaultdict(collections.Counter) + for text in texts: + pos = 0 + for tok in tokens[text]: + cls = class_map[base_class(text[pos - 1] if pos else None)] + counts[cls][token_symbol(tok)] += 1 + pos += token_len(tok) + return counts + + +def encoded_bits(text, tokens, lens, class_map, qstr_bits, rare_index_bits): + bits = 0 + pos = 0 + for tok in tokens: + cls = class_map[base_class(text[pos - 1] if pos else None)] + bits += lens[cls][token_symbol(tok)] + bits += token_extra_bits(tok, qstr_bits, rare_index_bits) + pos += token_len(tok) + return bits + + +def find_words(texts, qstrs, qstr_strs, max_words): + """Greedy dictionary search: repeatedly add the 2- to 11-gram estimated to + save the most bits until the dictionary is full or nothing pays off.""" + words = [] while len(words) < max_words: # Until the dictionary is filled to capacity, use a heuristic to find # the best "word" (2- to 11-gram) to add to it. @@ -248,7 +427,7 @@ def remove_offset(c): for t in texts: for atom in extractor.iter(t): if atom in qstrs: - atom = "\1" + atom = SYMBOL_QSTR counter[atom] += 1 cb = huffman.codebook(counter.items()) lengths = sorted(dict((v, len(cb[k])) for k, v in counter.items()).items()) @@ -284,15 +463,18 @@ def est_len(occ): # The difference between the two is the estimated net savings, in bits. def est_net_savings(s, occ): savings = occ * (bit_length(s) - est_len(occ)) - cost = len(s) * bits_per_codepoint + 24 + cost = len(s) * 8 + 24 return savings - cost counter = collections.Counter() for t in texts: for found, word in extractor.iter_words(t): if not found: - for substr in iter_substrings(word, minlen=2, maxlen=11): - counter[substr] += 1 + # Words are stored as 8-bit symbols, so they cannot contain + # rare (escaped) characters: split around them. + for piece in RARE_SPLIT.split(word): + for substr in iter_substrings(piece, minlen=2, maxlen=11): + counter[substr] += 1 # Score the candidates we found. This is a semi-empirical formula that # attempts to model the number of bits saved as closely as possible. @@ -319,90 +501,239 @@ def est_net_savings(s, occ): word = scores[0][0] words.append(word) + return words + - splitters = words[:] - if compression_level > 3: - splitters.extend(qstr_strs) +def compute_huffman_coding(qstrs, translation_name, translations, f, compression_level): + # possible future improvement: some languages are better when consider len(k) > 2. try both? + qstrs = dict((k, v) for k, v in qstrs.items() if len(k) > 3) + all_qstr_strs = list(qstrs.keys()) + original_texts = [t[1] for t in translations] + + for text in original_texts: + bad = RESERVED_CHARS.intersection(text) + if bad: + raise ValueError(f"Translation contains reserved character {bad!r}: {text!r}") + if any(ord(c) >= RARE_IDX_BASE for c in text): + raise ValueError(f"Translation contains private-use character: {text!r}") + + max_translation_encoded_length = max(len(t.encode("utf-8")) for t in original_texts) + encoded_length_bits = max_translation_encoded_length.bit_length() + use_qstrs = compression_level > 3 + + # Non-ASCII characters by decreasing frequency. The most frequent ones are + # renumbered from 0x80 (the dense alphabet), so that all symbols fit in 8 + # bits; the rest are escaped. + hi_count = collections.Counter(c for t in original_texts for c in t if ord(c) >= 0x80) + hi_chars = [c for c, _ in hi_count.most_common()] + if len(hi_chars) <= 0x7F: + dense_tries = (len(hi_chars),) + else: + dense_tries = DENSE_ALPHABET_TRIES + + def encode(dense_count): + alphabet = hi_chars[:dense_count] + rare = hi_chars[dense_count:] + rare_idx_chars = [c for c in rare if hi_count[c] > 1] + rare_raw_chars = [c for c in rare if hi_count[c] == 1] + rare_index_bits = (len(rare_idx_chars) - 1).bit_length() if rare_idx_chars else 0 + remap = {c: chr(ALPHABET_BASE + i) for i, c in enumerate(alphabet)} + remap.update({c: chr(RARE_IDX_BASE + i) for i, c in enumerate(rare_idx_chars)}) + remap.update({c: chr(RARE_RAW_BASE + i) for i, c in enumerate(rare_raw_chars)}) + word_start = ALPHABET_BASE + len(alphabet) + max_words = 0x100 - word_start if compression_level >= 5 else 0 + + texts = [remap_text(t, remap) for t in original_texts] + + # Prune the qstrs to only those that appear in the texts + qstr_counters = collections.Counter() + qstr_extractor = TextSplitter(all_qstr_strs) + for t in texts: + for qstr in qstr_extractor.iter(t): + if qstr in qstrs: + qstr_counters[qstr] += 1 + qstr_strs = list(qstr_counters.keys()) + + words = find_words(texts, qstrs, qstr_strs, max_words) + + # Now that the dictionary is fixed, find the tokenization of each string + # that takes the fewest bits, with Huffman codes chosen per class of the + # previous character. Code lengths and tokenization depend on each other, + # so iterate a few rounds from the greedy tokenization; the final + # tokenization and the codes built from it are what get emitted, so they + # are always consistent. + dp_qstrs = qstr_strs if use_qstrs else [] + # Upper bound on the qstr index width while parsing; the real width is computed below. + est_qstr_bits = max([0] + [qstrs[q] for q in dp_qstrs]).bit_length() + words_by_first = collections.defaultdict(list) + for w in words: + words_by_first[w[0]].append(w) + qstrs_by_first = collections.defaultdict(list) + for q in dp_qstrs: + qstrs_by_first[q[0]].append(q) + + greedy = TextSplitter(words + dp_qstrs) + greedy_tokens = {} + for t in texts: + greedy_tokens[t] = [("q", a) if a in qstrs else a for a in greedy.iter(t)] + + def table_bytes(class_map, counts, used_words): + nclasses = max(class_map) + 1 + row = max(max(code_lengths(cn).values()) for cn in counts.values()) + 1 + n = sum(len(cn) for cn in counts.values()) # values + n += nclasses * row # lengths + n += 2 * (nclasses + 1) # values_offset + n += NUM_BASE_CLASSES # class_map + n += sum(len(w) for w in used_words) # words + if used_words: + n += len(used_words[-1]) - len(used_words[0]) + 1 # wlencount + n += 2 * len(alphabet) + n += 2 * len(rare_idx_chars) + return n + + best = None + for preset in CLASS_PRESETS: + class_map = class_map_for(preset) + tokens = greedy_tokens + counts = tally(texts, tokens, class_map) + lens = {cls: code_lengths(cn) for cls, cn in counts.items()} + for _ in range(3): + tokens = { + t: parse_optimal( + t, + lens, + class_map, + words_by_first, + qstrs_by_first, + est_qstr_bits, + rare_index_bits, + ) + for t in texts + } + counts = tally(texts, tokens, class_map) + lens = {cls: code_lengths(cn) for cls, cn in counts.items()} + used_symbols = set().union(*counts.values()) + used_words = sorted((w for w in words if w in used_symbols), key=len) + size = table_bytes(class_map, counts, used_words) + sum( + ( + encoded_length_bits + + encoded_bits(t, tokens[t], lens, class_map, est_qstr_bits, rare_index_bits) + + 7 + ) + // 8 + for t in texts + ) + if best is None or size < best[0]: + best = (size, class_map, tokens, lens, used_words) + size, class_map, tokens, lens, words = best + return size, dict( + class_map=class_map, + tokens=tokens, + lens=lens, + words=words, + alphabet=alphabet, + rare_idx_chars=rare_idx_chars, + rare_raw_chars=rare_raw_chars, + rare_index_bits=rare_index_bits, + remap=remap, + word_start=word_start, + ) + + size, r = min((encode(k) for k in dense_tries), key=lambda x: x[0]) + class_map = r["class_map"] + tokens = r["tokens"] + lens = r["lens"] + words = r["words"] + alphabet = r["alphabet"] + rare_idx_chars = r["rare_idx_chars"] + rare_raw_chars = r["rare_raw_chars"] + rare_index_bits = r["rare_index_bits"] + remap = r["remap"] + word_start = r["word_start"] - words.sort(key=len) - extractor = TextSplitter(splitters) - counter = collections.Counter() used_qstr = 0 - for t in texts: - for atom in extractor.iter(t): - if atom in qstrs: - used_qstr = max(used_qstr, qstrs[atom]) - atom = "\1" - counter[atom] += 1 - cb = huffman.codebook(counter.items()) + for toks in tokens.values(): + for tok in toks: + if is_qstr(tok): + used_qstr = max(used_qstr, qstrs[tok[1]]) + translation_qstr_bits = used_qstr.bit_length() - word_start = start_unused word_end = word_start + len(words) - 1 - f.write(f"// # words {len(words)}\n") - f.write(f"// words {words}\n") + word_code = {w: chr(word_start + i) for i, w in enumerate(words)} - values = [] - length_count = {} - renumbered = 0 - last_length = None - canonical = {} - for atom, code in sorted(cb.items(), key=lambda x: (len(x[1]), x[0])): - if atom in qstr_strs: - atom = "\1" - values.append(atom) - length = len(code) - if length not in length_count: - length_count[length] = 0 - length_count[length] += 1 - if last_length: - renumbered <<= length - last_length - # print(f"atom={repr(atom)} code={code}", file=sys.stderr) - canonical[atom] = "{0:0{width}b}".format(renumbered, width=length) - if len(atom) > 1: - o = words.index(atom) + 0x80 - s = "".join(C_ESCAPES.get(ch1, ch1) for ch1 in atom) - f.write(f"// {o} {s} {counter[atom]} {canonical[atom]} {renumbered}\n") - else: - s = C_ESCAPES.get(atom, atom) - canonical[atom] = "{0:0{width}b}".format(renumbered, width=length) - o = ord(atom) - f.write(f"// {o} {s} {counter[atom]} {canonical[atom]} {renumbered}\n") - renumbered += 1 - last_length = length - lengths = bytearray() - f.write(f"// length count {length_count}\n") - - for i in range(1, max(length_count) + 2): - lengths.append(length_count.get(i, 0)) - f.write(f"// values {values} lengths {len(lengths)} {lengths}\n") + def symbol_value(sym): + return ord(word_code.get(sym, sym)) - f.write(f"// {values} {lengths}\n") - values = [(atom if len(atom) == 1 else chr(0x80 + words.index(atom))) for atom in values] - max_translation_encoded_length = max( - len(translation.encode("utf-8")) for (original, translation) in translations - ) + nclasses = max(class_map) + 1 + values = [] + lengths_rows = [] + canonical = [] + for cls in range(nclasses): + # Symbols are ordered by (length, value) so that the C decoder's canonical + # walk lands on the same index. + table = lens.get(cls, {}) + by_value = {chr(symbol_value(s)): l for s, l in table.items()} + v, counts_row, canon = canonical_codes(by_value) + values.append([ord(x) for x in v]) + lengths_rows.append(counts_row) + canonical.append({s: canon[chr(symbol_value(s))] for s in table}) + lengths_row = max(1, max(len(r) for r in lengths_rows)) + lengths_rows = [r + [0] * (lengths_row - len(r)) for r in lengths_rows] + assert all(len(code) >= 1 for canon in canonical for code in canon.values()) + assert all(v < 0x100 for vs in values for v in vs) + + unmap = {v: k for k, v in remap.items()} + f.write(f"// # words {len(words)}\n") + f.write("// words {}\n".format([remap_text(w, unmap) for w in words])) + f.write(f"// # alphabet {len(alphabet)}\n") + f.write(f"// # rare chars by index {len(rare_idx_chars)}, raw {len(rare_raw_chars)}\n") + f.write(f"// class_map {class_map} tables {nclasses}\n") + for cls in range(nclasses): + f.write(f"// class {cls}: {len(values[cls])} symbols, lengths {lengths_rows[cls]}\n") maxlen = len(words[-1]) if words else 0 minlen = len(words[0]) if words else 0 wlencount = [len([None for w in words if len(w) == l]) for l in range(minlen, maxlen + 1)] - translation_qstr_bits = used_qstr.bit_length() - - f.write("typedef {} mchar_t;\n".format(values_type)) - f.write("const uint8_t lengths[] = {{ {} }};\n".format(", ".join(map(str, lengths)))) + f.write("#define compress_max_length_bits ({})\n".format(encoded_length_bits)) + f.write("#define TRANSLATION_CLASSES {}\n".format(nclasses)) + f.write("#define LENGTHS_ROW {}\n".format(lengths_row)) f.write( - "const mchar_t values[] = {{ {} }};\n".format( - ", ".join(str(ord(remove_offset(u))) for u in values) + "const uint8_t class_map[{}] = {{ {} }};\n".format( + NUM_BASE_CLASSES, ", ".join(map(str, class_map)) ) ) f.write( - "#define compress_max_length_bits ({})\n".format( - max_translation_encoded_length.bit_length() + "const uint8_t lengths[] = {{ {} }};\n".format( + ", ".join(str(x) for row in lengths_rows for x in row) ) ) + offsets = [0] + for v in values: + offsets.append(offsets[-1] + len(v)) f.write( - "const mchar_t words[] = {{ {} }};\n".format( - ", ".join(str(ord(remove_offset(c))) for w in words for c in w) + "const uint16_t values_offset[{}] = {{ {} }};\n".format( + nclasses + 1, ", ".join(map(str, offsets)) + ) + ) + f.write( + "const uint8_t values[] = {{ {} }};\n".format(", ".join(str(x) for v in values for x in v)) + ) + f.write("#define alphabet_size {}\n".format(len(alphabet))) + f.write( + "const uint16_t alphabet[] = {{ {} }};\n".format(", ".join(str(ord(c)) for c in alphabet)) + ) + f.write("#define rare_index_count {}\n".format(len(rare_idx_chars))) + f.write("#define rare_index_bits {}\n".format(rare_index_bits)) + f.write("#define rare_raw_count {}\n".format(len(rare_raw_chars))) + f.write( + "const uint16_t rare_chars[] = {{ {} }};\n".format( + ", ".join(str(ord(c)) for c in rare_idx_chars) + ) + ) + f.write( + "const uint8_t words[] = {{ {} }};\n".format( + ", ".join(str(ord(c)) for w in words for c in w) ) ) f.write("const uint8_t wlencount[] = {{ {} }};\n".format(", ".join(str(p) for p in wlencount))) @@ -410,30 +741,33 @@ def est_net_savings(s, occ): f.write("#define word_end {}\n".format(word_end)) f.write("#define minlen {}\n".format(minlen)) f.write("#define maxlen {}\n".format(maxlen)) - f.write("#define translation_offstart {}\n".format(offstart)) - f.write("#define translation_offset {}\n".format(offset)) f.write("#define translation_qstr_bits {}\n".format(translation_qstr_bits)) qstrs_inv = dict((v, k) for k, v in qstrs.items()) return EncodingTable( values, - lengths, - words, + lengths_rows, + lengths_row, + class_map, canonical, - extractor, - apply_offset, - remove_offset, + words, + word_start, + alphabet, + rare_idx_chars, + rare_index_bits, + remap, translation_qstr_bits, qstrs, qstrs_inv, + tokens, ) def decompress(encoding_table, encoded, encoded_length_bits): - qstrs_inv = encoding_table.qstrs_inv - values = encoding_table.values - lengths = encoding_table.lengths - words = encoding_table.words + """Decode as the C decoder does (translate.c). Returns the original text.""" + et = encoding_table + alphabet_size = len(et.alphabet) + word_end = et.word_start + len(et.words) - 1 def bititer(): for byte in encoded: @@ -449,10 +783,25 @@ def getnbits(n): return bits dec = [] + last = None length = getnbits(encoded_length_bits) + decoded = 0 - i = 0 - while i < length: + def emit_char(c): + nonlocal last, decoded + dec.append(c) + decoded += len(c.encode("utf-8")) + last = c[-1] + + def emit(u): + if ALPHABET_BASE <= u < ALPHABET_BASE + alphabet_size: + emit_char(et.alphabet[u - ALPHABET_BASE]) + else: + emit_char(chr(u)) + + while decoded < length: + cls = et.class_map[base_class(last)] + lengths = et.lengths[cls] bits = 0 bit_length = 0 max_code = lengths[0] @@ -461,28 +810,33 @@ def getnbits(n): bits = (bits << 1) | nextbit() bit_length += 1 if max_code > 0 and bits < max_code: - # print('{0:0{width}b}'.format(bits, width=bit_length)) break max_code = (max_code << 1) + lengths[bit_length] searched_length += lengths[bit_length] - - v = values[searched_length + bits - max_code] - if v == chr(1): - qstr_idx = getnbits(encoding_table.translation_qstr_bits) - v = qstrs_inv[qstr_idx] - elif v >= chr(0x80) and v < chr(0x80 + len(words)): - v = words[ord(v) - 0x80] - i += len(v.encode("utf-8")) - dec.append(v) + v = et.values[cls][searched_length + bits - max_code] + if v == 1: + qstr_idx = getnbits(et.translation_qstr_bits) + emit_char(et.qstrs_inv[qstr_idx]) + elif v == 2: + emit_char(et.rare_idx_chars[getnbits(et.rare_index_bits)]) + elif v == 3: + emit_char(chr(getnbits(16))) + elif et.word_start <= v <= word_end: + for c in et.words[v - et.word_start]: + emit(ord(c)) + else: + emit(v) return "".join(dec) def compress(encoding_table, decompressed, encoded_length_bits, len_translation_encoded): + """Encode a translation using the tokenization chosen in compute_huffman_coding().""" if not isinstance(decompressed, str): raise TypeError() - qstrs = encoding_table.qstrs - canonical = encoding_table.canonical - extractor = encoding_table.extractor + et = encoding_table + text = remap_text(decompressed, et.remap) + tokens = et.tokens[text] + unmap = {v: k for k, v in et.remap.items()} enc = 1 @@ -494,17 +848,25 @@ def put_bits(enc, b, n): enc = put_bit(enc, b & (1 << i)) return enc + def put_code(enc, cls, sym): + for b in et.canonical[cls][sym]: + enc = put_bit(enc, b == "1") + return enc + enc = put_bits(enc, len_translation_encoded, encoded_length_bits) - for atom in extractor.iter(decompressed): - if atom in qstrs: - can = canonical["\1"] - else: - can = canonical[atom] - for b in can: - enc = put_bit(enc, b == "1") - if atom in qstrs: - enc = put_bits(enc, qstrs[atom], encoding_table.translation_qstr_bits) + pos = 0 + for tok in tokens: + cls = et.class_map[base_class(text[pos - 1] if pos else None)] + enc = put_code(enc, cls, token_symbol(tok)) + if is_qstr(tok): + enc = put_bits(enc, et.qstrs[tok[1]], et.translation_qstr_bits) + elif is_rare(tok): + if ord(tok) >= RARE_RAW_BASE: + enc = put_bits(enc, ord(unmap[tok]), 16) + else: + enc = put_bits(enc, ord(tok) - RARE_IDX_BASE, et.rare_index_bits) + pos += token_len(tok) while enc.bit_length() % 8 != 1: enc = put_bit(enc, 0) diff --git a/supervisor/shared/translate/compressed_string.h b/supervisor/shared/translate/compressed_string.h index 89dd75c22ae..7bfe2000b73 100644 --- a/supervisor/shared/translate/compressed_string.h +++ b/supervisor/shared/translate/compressed_string.h @@ -13,38 +13,41 @@ // The format of the compressed data is: // - the size of the uncompressed string in UTF-8 bytes, encoded as a // (compress_max_length_bits)-bit number. compress_max_length_bits is -// computed during dictionary generation time, and happens to be 8 -// for all current platforms. However, it'll probably end up being -// 9 in some translations sometime in the future. This length excludes +// computed during dictionary generation time, and is 8 for all +// current translations except ru, which needs 9. This length excludes // the trailing NUL, though notably decompress_length includes it. // -// - followed by the huffman encoding of the individual code -// points that make up the string. The trailing "\0" is not -// represented by a huffman code, but is implied by the length. -// (building the huffman encoding on UTF-16 code points gave better -// compression than building it on UTF-8 bytes) +// - followed by the huffman encoding of the symbols that make up the +// string. The trailing "\0" is not represented by a huffman code, but +// is implied by the length. // -// - If possible, the code points are represented as uint8_t values, with -// 0..127 representing themselves and 160..255 representing another range -// of Unicode, controlled by translation_offset and translation_offstart. -// If this is not possible, uint16_t values are used. At present, no translation -// requires code points not in the BMP, so this is adequate. +// - Each symbol is coded with one of TRANSLATION_CLASSES canonical Huffman +// tables, chosen by the class of the previously decoded byte: start of +// string or space, lowercase, uppercase, digit, '%', other, or >= 0x80. +// base_class() computes that seven-way class and class_map[] collapses it +// to a table index; the generator picks how many tables pay for themselves. +// The per-table code-length counts are rows of LENGTHS_ROW bytes in +// lengths[], and the symbols of table n are values[values_offset[n]..]. // -// - code points starting at 128 (word_start) and potentially extending -// to 255 (word_end) (but never interfering with the target -// language's used code points) stand for dictionary entries in a -// dictionary with size up to 256 code points. The dictionary entries -// are computed with a heuristic based on frequent substrings of 2 to -// 9 code points. These are called "words" but are not, grammatically -// speaking, words. They're just spans of code points that frequently -// occur together. They are ordered shortest to longest. +// - Symbol values 0..0x7F are ASCII (with 1, 2 and 3 reserved for the +// escapes below). Values from 0x80 up to 0x80 + alphabet_size - 1 are the +// most frequent non-ASCII characters used by the translation, renumbered +// by decreasing frequency; alphabet[] maps them back to Unicode code +// points. Most translations fit their whole non-ASCII repertoire in the +// alphabet. Those that don't (ja, ko) keep the 32 to 80 most frequent +// characters there and escape the rest: value 2 is followed by a +// rare_index_bits-bit index into rare_chars[] (characters used more than +// once), value 3 by a raw 16-bit code point (characters used once, which +// are cheaper without a table entry). All symbols are therefore 8 bits. +// At present, no translation requires code points outside the BMP, so +// this is adequate. // -// - If the translation uses a lot of code points or widely spaced code points, -// then the huffman table entries are UTF-16 code points. But if the translation -// uses only ASCII 7-bit code points plus a SMALL range of higher code points that -// still fit in 8 bits, translation_offset and translation_offstart are used to -// renumber the code points so that they still fit within 8 bits. (it's very beneficial -// for mchar_t to be 8 bits instead of 16!) +// - Symbol values from word_start to word_end stand for dictionary entries +// in a dictionary of up to 256 - word_start entries. The dictionary +// entries are computed with a heuristic based on frequent substrings of 2 +// to 11 symbols. These are called "words" but are not, grammatically +// speaking, words. They're just spans of symbols that frequently occur +// together. They are ordered shortest to longest. // // - dictionary entries are non-overlapping, and the _ending_ index of each // entry is stored in an array. A count of words of each length, from @@ -56,10 +59,11 @@ // - Value 1 ('\1') is used to indicate that a QSTR number follows. the // QSTR is encoded as a fixed number of bits (translation_qstr_bits), e.g., // 10 bits if the highest core qstr is from 512 to 1023 inclusive. -// (maketranslationdata uses a simple heuristic where any qstr >= 3 -// characters long is encoded in this way; this is simple but probably not -// optimal. In fact, the rule of >= 2 characters is better for SOME languages -// on SOME boards.) +// (maketranslationdata uses a simple heuristic where any qstr >= 4 +// characters long may be encoded in this way; whether a given occurrence +// is coded as a qstr or as characters is decided per occurrence by the +// parser, which picks the tokenization of each string that takes the +// fewest bits.) // // The "data" / "tail" construct is so that the struct's last member is a // "flexible array". However, the _only_ member is not permitted to be @@ -76,6 +80,26 @@ typedef struct compressed_string { const uint8_t tail[]; } const *mp_rom_error_text_t; +// Class of the previously decoded byte, which selects the Huffman table for +// the next symbol. Must match base_class() in py/maketranslationdata.py. +typedef enum { + CLASS_START_OR_SPACE = 0, + CLASS_LOWER = 1, + CLASS_UPPER = 2, + CLASS_DIGIT = 3, + CLASS_OTHER = 4, + CLASS_PERCENT = 5, + CLASS_NON_ASCII = 6, +} translation_class_t; + +// Symbol values with a special meaning; every other value is a character or a +// dictionary word. Must match py/maketranslationdata.py. +typedef enum { + SYMBOL_QSTR = 1, // followed by translation_qstr_bits bits of qstr index + SYMBOL_RARE_INDEX = 2, // followed by rare_index_bits bits of rare_chars[] index + SYMBOL_RARE_RAW = 3, // followed by a 16-bit code point +} translation_symbol_t; + // Return the compressed, translated version of a source string // Usually, due to LTO, this is optimized into a load of a constant // pointer. diff --git a/supervisor/shared/translate/translate.c b/supervisor/shared/translate/translate.c index b590b385a71..d0c05dda6e5 100644 --- a/supervisor/shared/translate/translate.c +++ b/supervisor/shared/translate/translate.c @@ -23,7 +23,7 @@ void serial_write_compressed(mp_rom_error_text_t compressed) { mp_printf(MP_PYTHON_PRINTER, "%S", compressed); } -static void get_word(int n, const mchar_t **pos, const mchar_t **end) { +static void get_word(int n, const uint8_t **pos, const uint8_t **end) { int len = minlen; int i = 0; *pos = words; @@ -37,14 +37,10 @@ static void get_word(int n, const mchar_t **pos, const mchar_t **end) { *end = *pos + len; } -static void put_utf8(vstr_t *vstr, int u) { - if (u >= translation_offstart) { - u += translation_offset; - } +static void put_utf8(vstr_t *vstr, unsigned u) { if (word_start <= u && u <= word_end) { - uint n = (u - word_start); - const mchar_t *pos, *end; - get_word(n, &pos, &end); + const uint8_t *pos, *end; + get_word(u - word_start, &pos, &end); // note that at present, entries in the words table are // guaranteed not to represent words themselves, so this adds // at most 1 level of recursive call @@ -53,9 +49,39 @@ static void put_utf8(vstr_t *vstr, int u) { } return; } + // alphabet_size is a compile-time constant; the test folds away when it is 0. + if (alphabet_size > 0 && u >= 0x80 && u < 0x80 + alphabet_size) { + u = alphabet[u - 0x80]; + } vstr_add_char(vstr, u); } +// The Huffman table used for the next symbol depends on the class of the last +// byte written. Classifying the last byte is the same as classifying the last +// code point: every byte of a multi-byte UTF-8 sequence is >= 0x80. +// This must match base_class() in py/maketranslationdata.py. +static translation_class_t base_class(uint8_t last) { + if (last == ' ') { + return CLASS_START_OR_SPACE; + } + if (last >= 'a' && last <= 'z') { + return CLASS_LOWER; + } + if (last >= 'A' && last <= 'Z') { + return CLASS_UPPER; + } + if (last >= '0' && last <= '9') { + return CLASS_DIGIT; + } + if (last == '%') { + return CLASS_PERCENT; + } + if (last >= 0x80) { + return CLASS_NON_ASCII; + } + return CLASS_OTHER; +} + uint16_t decompress_length(mp_rom_error_text_t compressed) { #ifndef NO_QSTR #if (compress_max_length_bits <= 8) @@ -99,23 +125,34 @@ static void decompress_vstr(mp_rom_error_text_t compressed, vstr_t *decompressed size_t alloc = decompressed->alloc - 1; // Stop one early because the last byte is always NULL. for (; decompressed->len < alloc;) { + // The start of a string is classed like a space. + uint8_t last = decompressed->len ? (uint8_t)decompressed->buf[decompressed->len - 1] : ' '; + uint8_t cls = class_map[base_class(last)]; + const uint8_t *len_row = lengths + cls * LENGTHS_ROW; + uint32_t bits = 0; uint8_t bit_length = 0; - uint32_t max_code = lengths[0]; - uint32_t searched_length = lengths[0]; + uint32_t max_code = len_row[0]; + uint32_t searched_length = len_row[0]; while (true) { bits = (bits << 1) | next_bit(&b); bit_length += 1; if (max_code > 0 && bits < max_code) { break; } - max_code = (max_code << 1) + lengths[bit_length]; - searched_length += lengths[bit_length]; + max_code = (max_code << 1) + len_row[bit_length]; + searched_length += len_row[bit_length]; } - int v = values[searched_length + bits - max_code]; - if (v == 1) { + unsigned v = values[values_offset[cls] + searched_length + bits - max_code]; + if (v == SYMBOL_QSTR) { qstr q = get_nbits(&b, translation_qstr_bits); vstr_add_str(decompressed, qstr_str(q)); + } else if (rare_index_count > 0 && v == SYMBOL_RARE_INDEX) { + // A non-ASCII character outside the dense alphabet, by table index. + vstr_add_char(decompressed, rare_chars[get_nbits(&b, rare_index_bits)]); + } else if (rare_raw_count > 0 && v == SYMBOL_RARE_RAW) { + // A non-ASCII character used only once, as a raw code point. + vstr_add_char(decompressed, get_nbits(&b, 16)); } else { put_utf8(decompressed, v); }