fix(language): make the parser setting real, and Chinese use jieba (#278)

CHANGELOG.md

@@ -86,6 +86,22 @@ ones are marked like "v1.0.0-fork".
   unchanged; the work behind them no longer grows with the size of your
   vocabulary.
 
+* **Chinese texts could not be read** (#278). A Chinese language created from
+  the built-in preset produced a text with no clickable words at all — nothing
+  to look up, nothing to track. Chinese and Japanese now come out of the preset
+  already pointed at a real tokenizer: **jieba** segments Chinese into words
+  rather than characters, and Japanese asks for MeCab as it always claimed to.
+  Where that tokenizer is not installed the language falls back to
+  character-by-character parsing, so it stays readable either way.
+
+* **Choosing a parser in the language form did nothing** (#278). The Parser
+  Type menu wrote its value to the database and no part of the parsing pipeline
+  ever read it — picking "Jieba (Chinese)" or "MeCab Python" parsed exactly as
+  before. The setting is now honoured. It also lists the parsers from
+  `config/parsers.php`, which it never did, so jieba and MeCab Python appear on
+  an install that has them. A language that names no parser — every language
+  that exists today — parses exactly as it did.
+
 ## [3.4.2-fork] - 2026-08-16
 
 ### Fixed

locale/de/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "Der Text wird diese Größe haben",
     "form.section_text_processing": "Textverarbeitung",
     "form.parser_type": "Parser-Typ",
+    "form.parser_automatic": "Automatisch",
     "form.parser_unavailable": " (nicht verfügbar)",
     "form.character_substitutions": "Zeichenersetzungen",
     "form.character_substitutions_help": "Zeichen vor dem Parsen ersetzen (Format: von=zu, getrennt durch |)",

locale/en/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "Text will be this size",
     "form.section_text_processing": "Text Processing",
     "form.parser_type": "Parser Type",
+    "form.parser_automatic": "Automatic",
     "form.parser_unavailable": " (unavailable)",
     "form.character_substitutions": "Character Substitutions",
     "form.character_substitutions_help": "Replace characters before parsing (format: from=to, separated by |)",

locale/es/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "El texto tendrá este tamaño",
     "form.section_text_processing": "Procesamiento de texto",
     "form.parser_type": "Tipo de analizador",
+    "form.parser_automatic": "Automático",
     "form.parser_unavailable": " (no disponible)",
     "form.character_substitutions": "Sustituciones de caracteres",
     "form.character_substitutions_help": "Reemplaza caracteres antes del análisis (formato: de=a, separados por |)",

locale/fr/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "Le texte aura cette taille",
     "form.section_text_processing": "Traitement du texte",
     "form.parser_type": "Type d'analyseur",
+    "form.parser_automatic": "Automatique",
     "form.parser_unavailable": " (indisponible)",
     "form.character_substitutions": "Substitutions de caractères",
     "form.character_substitutions_help": "Remplacer les caractères avant l'analyse (format : de=à, séparés par |)",

locale/it/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "Il testo avrà questa dimensione",
     "form.section_text_processing": "Elaborazione del testo",
     "form.parser_type": "Tipo di parser",
+    "form.parser_automatic": "Automatico",
     "form.parser_unavailable": " (non disponibile)",
     "form.character_substitutions": "Sostituzioni di caratteri",
     "form.character_substitutions_help": "Sostituisci i caratteri prima dell'analisi (formato: da=a, separati da |)",

locale/ja/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "テキストはこのサイズになります",
     "form.section_text_processing": "テキスト処理",
     "form.parser_type": "パーサータイプ",
+    "form.parser_automatic": "自動",
     "form.parser_unavailable": "（利用不可）",
     "form.character_substitutions": "文字置換",
     "form.character_substitutions_help": "解析前に文字を置換します（形式：from=to、|で区切り）",

locale/pt/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "O texto terá este tamanho",
     "form.section_text_processing": "Processamento de Texto",
     "form.parser_type": "Tipo de Analisador",
+    "form.parser_automatic": "Automático",
     "form.parser_unavailable": " (indisponível)",
     "form.character_substitutions": "Substituições de Caracteres",
     "form.character_substitutions_help": "Substituir caracteres antes da análise (formato: de=para, separados por |)",

locale/ru/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "Текст будет такого размера",
     "form.section_text_processing": "Обработка текста",
     "form.parser_type": "Тип парсера",
+    "form.parser_automatic": "Автоматически",
     "form.parser_unavailable": " (недоступно)",
     "form.character_substitutions": "Замены символов",
     "form.character_substitutions_help": "Заменять символы перед парсингом (формат: from=to, через |)",

locale/zh/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "文本将以此大小显示",
     "form.section_text_processing": "文本处理",
     "form.parser_type": "解析器类型",
+    "form.parser_automatic": "自动",
     "form.parser_unavailable": "（不可用）",
     "form.character_substitutions": "字符替换",
     "form.character_substitutions_help": "解析前替换字符（格式：from=to，用 | 分隔）",

src/Modules/Admin/Application/Services/TtsService.php

@@ -57,7 +57,7 @@ public function __construct(?LanguageFacade $languageService = null)
      *
      * @param int   $lgId       Language ID
      * @param array<string, array{
-     *     0: string, 1: string, 2: bool, 3: string, 4: string, 5: bool, 6: bool, 7: bool
+     *     0: string, 1: string, 2: bool, 3: string, 4: string, 5: bool, 6: bool, 7: bool, 8: string
      * }> $langArray Languages array from langdefs
      *
      * @return string Two-letter language code
@@ -94,7 +94,7 @@ public function getLanguageIdFromCode(string $code, array $langArray): int
      * Get language options for TTS form.
      *
      * @param array<string, array{
-     *     0: string, 1: string, 2: bool, 3: string, 4: string, 5: bool, 6: bool, 7: bool
+     *     0: string, 1: string, 2: bool, 3: string, 4: string, 5: bool, 6: bool, 7: bool, 8: string
      * }> $langArray Languages array from langdefs
      *
      * @return string HTML-formatted options string
@@ -117,7 +117,7 @@ public function getLanguageOptions(array $langArray): string
      * Get current language code for TTS settings.
      *
      * @param array<string, array{
-     *     0: string, 1: string, 2: bool, 3: string, 4: string, 5: bool, 6: bool, 7: bool
+     *     0: string, 1: string, 2: bool, 3: string, 4: string, 5: bool, 6: bool, 7: bool, 8: string
      * }> $langArray Languages array from langdefs
      *
      * @return string Current language code

src/Modules/Language/Application/LanguageFacade.php

@@ -354,7 +354,7 @@ public function getLanguageName($id): string
      *
      * @param int   $id             Language ID
      * @param array<string, array{0: string, 1: string, 2: bool, 3: string,
-     *               4: string, 5: bool, 6: bool, 7: bool}> $languagesTable
+     *               4: string, 5: bool, 6: bool, 7: bool, 8: string}> $languagesTable
      *               Table of languages, usually from LanguagePresets::getAll()
      *
      * @return string Two-letter code (e.g., BCP 47) or empty string

src/Modules/Language/Application/UseCases/GetLanguageCode.php

@@ -63,7 +63,7 @@ public function getLanguageName($id): string
      *
      * @param int   $id             Language ID
      * @param array<string, array{0: string, 1: string, 2: bool, 3: string,
-     *               4: string, 5: bool, 6: bool, 7: bool}> $languagesTable
+     *               4: string, 5: bool, 6: bool, 7: bool, 8: string}> $languagesTable
      *               Table of languages, usually from LanguagePresets::getAll()
      *
      * @return string Two-letter code (e.g., BCP 47) or empty string

src/Modules/Language/Http/LanguageApiHandler.php

@@ -482,7 +482,8 @@ public function formatGetDefinitions(): array
                 'sentSplRegExp' => $def[4],
                 'makeCharacterWord' => $def[5],
                 'removeSpaces' => $def[6],
-                'rightToLeft' => $def[7]
+                'rightToLeft' => $def[7],
+                'parserType' => $def[8]
             ];
         }
 

src/Modules/Language/Infrastructure/Data/langdefs.json

@@ -58,7 +58,7 @@
         "makeCharacterWord": true,
         "removeSpaces": true,
         "rightToLeft": false,
-        "parserType": "character"
+        "parserType": "jieba"
     },
     "Chinese (Traditional)": {
         "glosbeIso": "zh",
@@ -69,7 +69,7 @@
         "makeCharacterWord": true,
         "removeSpaces": true,
         "rightToLeft": false,
-        "parserType": "character"
+        "parserType": "jieba"
     },
     "Croatian": {
         "glosbeIso": "hr",

src/Modules/Language/Infrastructure/Parser/ParserRegistry.php

@@ -42,11 +42,16 @@ class ParserRegistry
     /**
      * Create a new parser registry with default parsers.
      *
-     * @param ExternalParserLoader|null $externalLoader Optional loader for external parsers
+     * Builds its own loader when none is given, so that a registry constructed
+     * directly still sees the parsers in config/parsers.php. It did not before,
+     * which is why jieba never reached the language form's parser list even on
+     * an install where it was installed and working.
+     *
+     * @param ExternalParserLoader|null $externalLoader Loader for external parsers
      */
     public function __construct(?ExternalParserLoader $externalLoader = null)
     {
-        $this->externalLoader = $externalLoader;
+        $this->externalLoader = $externalLoader ?? new ExternalParserLoader();
         $this->registerDefaultParsers();
         $this->registerExternalParsers();
     }
@@ -176,6 +181,85 @@ public function getDefaultType(): string
         return self::DEFAULT_PARSER;
     }
 
+    /**
+     * The parser a language deliberately asked for, if any.
+     *
+     * Only an explicit, non-default `LgParserType` counts. The legacy signals
+     * resolveParserTypeFromRow() also understands — the MECAB magic word and
+     * the split-each-character flag — are deliberately ignored here: the
+     * pipeline has always handled those itself, so returning null for them
+     * keeps their parsing byte-identical.
+     *
+     * A stored parser type is not by itself evidence of a choice, which is the
+     * subtle half. 20251223_120000_add_parser_type.sql *backfilled* the column
+     * from those same legacy signals — 'mecab' for the magic word, 'character'
+     * for LgSplitEachChar — so on every upgraded install the CJK languages
+     * already carry a type nobody picked. Routing those to the registry
+     * retokenizes them: measured on a real database, a character-split Chinese
+     * text goes from 103 words to 122 and Japanese from 46 to 60, which would
+     * silently desynchronise saved terms from new text occurrences. A value
+     * that merely restates the legacy signal beside it is therefore read as the
+     * legacy signal, not as intent.
+     *
+     * Deriving intent this way is a workaround for the magic word overloading
+     * LgRegexpWordCharacters; it goes away once that is retired and the column
+     * means only what a user chose.
+     *
+     * @param array<string, mixed> $row Database row with Lg* prefixed columns
+     *
+     * @return ParserInterface|null The chosen parser, or null to leave the
+     *         language on the built-in pipeline
+     */
+    public function getOptedInParserFromRow(array $row): ?ParserInterface
+    {
+        $type = trim((string) ($row['LgParserType'] ?? ''));
+        if ($type === '' || $type === self::DEFAULT_PARSER) {
+            return null;
+        }
+
+        if (self::restatesALegacySignal($type, $row)) {
+            return null;
+        }
+
+        $parser = $this->get($type);
+        if ($parser === null || !$parser->isAvailable()) {
+            // An unavailable parser must not drop the language onto the regex
+            // parser: for the CJK languages that ask for jieba or mecab, that
+            // yields a text with no words at all. The built-in pipeline still
+            // honours their split-each-character setting, so fall back to it.
+            return null;
+        }
+
+        return $parser;
+    }
+
+    /**
+     * Whether a stored parser type only repeats the legacy flag beside it.
+     *
+     * The backfill wrote 'mecab' where the magic word was and 'character' where
+     * LgSplitEachChar was set, so those two combinations carry no more
+     * information than the flags do, and the built-in pipeline already acts on
+     * the flags. Anything else — jieba, an external tokenizer, or 'character'
+     * on a language whose split flag is off — could only have been chosen.
+     *
+     * @param string               $type Trimmed, non-empty LgParserType
+     * @param array<string, mixed> $row  Database row with Lg* prefixed columns
+     *
+     * @return bool True when the value is derived rather than chosen
+     */
+    private static function restatesALegacySignal(string $type, array $row): bool
+    {
+        if ($type === 'mecab') {
+            return strtoupper(trim((string) ($row['LgRegexpWordCharacters'] ?? ''))) === 'MECAB';
+        }
+
+        if ($type === 'character') {
+            return (int) ($row['LgSplitEachChar'] ?? 0) === 1;
+        }
+
+        return false;
+    }
+
     /**
      * Resolve the parser type for a language.
      *

src/Modules/Language/Views/form.php

@@ -432,6 +432,7 @@ class="input"
                     <div class="control">
                         <div class="select is-fullwidth">
                             <select name="LgParserType" id="LgParserType" x-model="lang.parserType">
+                                <option value=""><?php echo __('language.form.parser_automatic'); ?></option>
                                 <?php foreach ($parserInfo as $type => $info) :
                                     $infoAvailable = isset($info['available']) && $info['available'];
                                     $infoName = isset($info['name']) && is_string($info['name']) ? $info['name'] : '';