default itn on for sensevoice

bindings/python/tests/test_text_controls.py

@@ -22,7 +22,12 @@ def test_itn_changes_sensevoice_text_normalization(itn_model_path, audio_pcm):
         disabled = session.run(audio_pcm, language="en", itn="off")
         enabled = session.run(audio_pcm, language="en", itn="on")
 
-    assert (default.text, default.raw_text) == (disabled.text, disabled.raw_text)
+    # DEFAULT resolves to ON for sensevoice: the textnorm prefix is the
+    # family's only source of casing and punctuation, so the out-of-the-box
+    # transcript is the readable one. --no-itn / itn="off" recovers upstream's
+    # spoken-form output.
+    assert (default.text, default.raw_text) == (enabled.text, enabled.raw_text)
     assert enabled.text != disabled.text
+    assert disabled.text == disabled.text.lower()
     assert "<|woitn|>" in disabled.raw_text
     assert "<|withitn|>" in enabled.raw_text

bindings/rust/transcribe-cpp/tests/transcribe.rs

@@ -128,9 +128,10 @@ fn itn_changes_sensevoice_text_normalization() {
     let enabled = run(&mut session, Itn::On);
     assert_eq!(
         (default.text, default.raw_text),
-        (disabled.text.clone(), disabled.raw_text.clone())
+        (enabled.text.clone(), enabled.raw_text.clone())
     );
     assert_ne!(enabled.text, disabled.text);
+    assert_eq!(disabled.text, disabled.text.to_lowercase());
     assert!(disabled.raw_text.contains("<|woitn|>"));
     assert!(enabled.raw_text.contains("<|withitn|>"));
 }

bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift

@@ -97,9 +97,10 @@ final class TranscribeTests: XCTestCase {
         let defaultResult = try run(.default)
         let disabled = try run(.off)
         let enabled = try run(.on)
-        XCTAssertEqual(defaultResult.text, disabled.text)
-        XCTAssertEqual(defaultResult.rawText, disabled.rawText)
+        XCTAssertEqual(defaultResult.text, enabled.text)
+        XCTAssertEqual(defaultResult.rawText, enabled.rawText)
         XCTAssertNotEqual(enabled.text, disabled.text)
+        XCTAssertEqual(disabled.text, disabled.text.lowercased())
         XCTAssertTrue(disabled.rawText.contains("<|woitn|>"))
         XCTAssertTrue(enabled.rawText.contains("<|withitn|>"))
     }

bindings/typescript/test/transcribe.test.mjs

@@ -93,8 +93,12 @@ modelTest("ITN changes SenseVoice text normalization", ITN_MODEL, async () => {
       const base = await s.run(jfk(), { language: "en", itn: "default" });
       const off = await s.run(jfk(), { language: "en", itn: "off" });
       const on = await s.run(jfk(), { language: "en", itn: "on" });
-      assert.deepEqual([base.text, base.rawText], [off.text, off.rawText]);
+      // "default" resolves to ON for sensevoice: the textnorm prefix is the
+      // family's only source of casing and punctuation, so the unconfigured
+      // transcript is the readable one. "off" recovers upstream spoken form.
+      assert.deepEqual([base.text, base.rawText], [on.text, on.rawText]);
       assert.notEqual(on.text, off.text);
+      assert.equal(off.text, off.text.toLowerCase());
       assert.match(off.rawText, /<\|woitn\|>/);
       assert.match(on.rawText, /<\|withitn\|>/);
     } finally {

docs/models/sensevoice-small.md

@@ -16,7 +16,32 @@ The same CTC head also emits language ID, simple emotion labels (`<|HAPPY|>`,
 `<|NEUTRAL|>`, `<|SAD|>`, `<|ANGRY|>`, `<|EMO_UNKNOWN|>`), audio-event tags
 (`<|Speech|>`, `<|BGM|>`, `<|Applause|>`, …), and an inverse-text-normalization
 flag (`<|withitn|>` / `<|woitn|>`). These are stripped from the transcript by
-default; pass `--raw-tokens` to keep them, and `--itn` to enable ITN.
+default; pass `--raw-tokens` to keep them.
+
+**ITN is on by default.** SenseVoice has no separate punctuation/capitalization
+control — the ITN flag is what produces casing, punctuation, and digits — so
+transcribe.cpp resolves the run-time default to on rather than following
+upstream's `itn=False`. Pass `--no-itn` (library: `itn = TRANSCRIBE_ITN_MODE_OFF`)
+for upstream's verbatim spoken form.
+
+ITN changes the CTC decode, not just the rendering, so it moves accuracy — and
+the direction depends on the language:
+
+- **English costs a little.** LibriSpeech test-clean (512 utts, F32/CPU):
+  **+0.110pp WER** (2.556% → 2.666%), diffuse single-word corruption
+  (`arcadian` → `arrcadian`). This is upstream behavior, not a port artifact —
+  the FunASR 1.3.1 reference shows a *larger* penalty on the same data
+  (+0.147pp) and mangles the same words byte-for-byte. The cost is inside the
+  table's bootstrap CI.
+- **Chinese gains a lot.** FLEURS-zh (945 utts, F32/CPU): **−2.030pp CER**
+  (10.100% → 8.070%). The FLEURS-zh reference is digit-normalized, so ITN-on's
+  `2011年8月` matches it where ITN-off's `二零一一年八月` does not. This is a
+  scoring-convention match, not measured evidence of better recognition.
+
+Readable output is judged the better default for interactive use; `--no-itn` is
+the right choice for a pipeline that scores or post-processes text. The WER
+numbers below are measured with ITN **off**, matching the reference runs; see
+[WER methodology](../tools/wer.md).
 
 See FunAudioLLM's [model card](https://huggingface.co/FunAudioLLM/SenseVoiceSmall)
 for training data, intended use, and upstream evaluation methodology.
@@ -76,14 +101,21 @@ build/bin/transcribe-cli \
 ```
 
 Pass `--language zh` / `yue` / `ja` / `ko` (or omit for auto-detection) for
-the other supported languages. Raw control tokens and ITN are opt-in:
+the other supported languages.
 
 ```bash
-# Keep <|en|><|HAPPY|><|Speech|><|woitn|>… in the output text:
+# Keep <|en|><|HAPPY|><|Speech|><|withitn|>… in the output text:
 build/bin/transcribe-cli --raw-tokens -m … samples/jfk.wav
 
-# Render numbers/punctuation in formal form:
-build/bin/transcribe-cli --itn -m … samples/jfk.wav
+# Default (ITN on):
+#   And so my fellow Americans ask not what your country can do for you, ask
+#   what you can do for your country.
+build/bin/transcribe-cli -m … samples/jfk.wav
+
+# Upstream spoken form — lowercase, unpunctuated, numbers as words:
+#   and so my fellow americans ask not what your country can do for you ask
+#   what you can do for your country
+build/bin/transcribe-cli --no-itn -m … samples/jfk.wav
 ```
 
 If your audio is not already 16 kHz mono WAV, convert it first:

docs/porting/families/sensevoice.md

@@ -215,6 +215,16 @@ labels.
 
 ## Notes
 
+- **Post-port change (ITN).** `TRANSCRIBE_ITN_MODE_DEFAULT` now resolves to ITN
+  **on** for this family, diverging from upstream's `itn=False`. SenseVoice has
+  no separate PNC toggle, so ITN off means users get lowercase, unpunctuated
+  text out of the box. Two knock-on corrections to the acceptance row above,
+  which is left as the record of what was verified at the time: the CLI pair is
+  now `--itn` / `--no-itn` (ITN-on is the unflagged path), and the family-param
+  struct it names (`transcribe_sensevoice_params{ .use_itn = true }`) no longer
+  exists — the control is the generic `transcribe_run_params::itn` enum. The
+  WER harness pins `--no-itn` (`scripts/wer/run.py --itn`), so the Stage 7
+  numbers still describe ITN-off text and the reference comparison is unchanged.
 - This is the first FunASR-native port in the repo. Existing ports
   (Whisper, Parakeet, Cohere, Qwen3-ASR) all live under HF Transformers
   or NeMo, both of which expose `config.json` / preprocessor / tokenizer

docs/tools/wer.md

@@ -39,12 +39,78 @@ LibriSpeech WER:
 | Fallback thresholds | compression `2.4`, logprob `-1.0`, no-speech `0.6` | Library defaults (`transcribe_whisper_run_ext_init`). |
 | Condition on prev | **off** | Library default; long-form conditioning is not part of short-form WER. |
 | Normalization | `EnglishTextNormalizer` (en) / `BasicTextNormalizer` (other) | Applied to both ref and hyp at score time (`score.py`). |
+| ITN | **off** (`--no-itn`) | Spoken form, matching what the reference runs produce (`run_reference_sensevoice.py` / `run_reference_funasr_nano.py` default `--use-itn` off). Only `sensevoice` and `funasr_nano` have a runtime ITN toggle; every other family ignores the flag. |
 | Dataset | full LibriSpeech `test-clean` (2620 utts) | — |
 
 The recipe is **stamped into the hyp JSONL `batch_header`** (`recipe` field)
 by `run.py`, so every artifact is self-describing and a methodology drift
 shows up in the file rather than silently shifting the number.
 
+> **ITN is pinned, not inherited.** The run-time ITN default is per-family and
+> is a product decision that can move: `sensevoice` resolves it to *on* (there
+> the ITN toggle is also the only source of casing and punctuation, so ITN-off
+> hands an unconfigured caller lowercase unpunctuated text), while
+> `funasr_nano` keeps upstream's `itn=False`. The benchmark follows neither:
+> `run.py` always passes `--no-itn`, because the reference runs it is gated
+> against produce spoken form. Explicit for every family, so a future default
+> flip cannot silently restate what a published number means.
+>
+> This is not just a formatting difference that normalization would absorb.
+> ITN changes the decode itself, so it moves WER on its own — **in the upstream
+> model, not only in this port.** Measured on LibriSpeech test-clean:
+>
+> | Arm | ITN off | ITN on | Δ | n |
+> | --- | ---: | ---: | ---: | ---: |
+> | SenseVoice — FunASR 1.3.1 reference, FP32/CPU | 2.538% | 2.685% | **+0.147pp** | 512 |
+> | SenseVoice — transcribe.cpp F32/CPU | 2.556% | 2.666% | **+0.110pp** | 512 |
+> | SenseVoice — transcribe.cpp Q8_0/Metal | 2.556% | 2.731% | +0.175pp | 512 |
+> | Fun-ASR-Nano-2512 BF16 | 1.754% | 1.840% | +0.086pp | 200 |
+> | Fun-ASR-MLT-Nano-2512 BF16 | 1.668% | 1.711% | +0.043pp | 200 |
+>
+> The ITN cost is a property of the model. At matched dtype the port's ITN
+> penalty (+0.110pp) is *smaller* than the reference's (+0.147pp), and the
+> degradation is the same degradation: of the 32 utterances where the port
+> regresses under ITN, 26 also regress in the reference, and the top cases are
+> byte-identical on both sides (`arcadian` → `arrcadian`, `sententiously` →
+> `sentiously`, `gilchrist` → `gilcht`, `pride` → `bride`). Comparing the Q8_0
+> arm against the FP32 reference overstates the gap: ITN-on is somewhat more
+> quant-sensitive, which is the +0.065pp between the F32 and Q8_0 rows.
+>
+> On English it is not number rendering — only 2.5% of ITN-on hypotheses
+> contain a digit, so `EnglishTextNormalizer` has almost nothing to absorb.
+>
+> **On Chinese the sign flips, and the reason is the reference's convention.**
+> `BasicTextNormalizer` (used for every non-English language) strips
+> punctuation but does no number mapping — and the FLEURS-zh reference is
+> itself written with digits (`桥下垂直净空 15 米 … 于 2011 年 8 月完工`). So
+> ITN-*off*, which emits spoken form (`二零一一年八月`), mismatches the
+> reference on every date and quantity, while ITN-on matches it:
+>
+> | Model | FLEURS-zh CER off | on | Δ | n |
+> | --- | ---: | ---: | ---: | ---: |
+> | SenseVoiceSmall F32 | 10.100% | 8.070% | **−2.030pp** | 945 |
+> | Fun-ASR-Nano-2512 BF16 | 7.920% | 6.640% | **−1.280pp** | 250 |
+> | Fun-ASR-MLT-Nano-2512 BF16 | 7.980% | 6.960% | **−1.020pp** | 250 |
+>
+> The SenseVoice row is the full 945-utterance split and its arms' 95% CIs
+> barely overlap ([9.19, 11.02] off vs [7.25, 8.96] on); the Fun-ASR rows are a
+> 250-utterance subset, so compare deltas within a row, not absolutes across
+> rows.
+>
+> Read that as "ITN-on matches the FLEURS-zh scoring convention," not "the
+> model recognizes Chinese better with ITN on" — the gain is digit rendering
+> lining up with the reference, not improved recognition.
+>
+> None of this argues for unpinning the harness. The published tables were
+> measured at ITN-off and `run_reference_*.py` defaults ITN off; the pin exists
+> to keep both sides on the same convention, whichever direction that
+> convention happens to favor.
+>
+> If the library default is ever revisited, this pin stays put unless the
+> reference side is re-run to match. `scripts/validate.py` pins `--no-itn` for
+> the same two families and the same reason — there the prefix embedding /
+> prompt change would break tensor comparison outright.
+
 **What does and doesn't move WER (measured on whisper-medium F16):**
 
 - **Timestamps move it ~0.2pp.** `segment` → 2.63%, `none` → 2.81%. This is

examples/cli/main.cpp

@@ -239,8 +239,10 @@ struct cli_args {
 
     // SenseVoice / FunASR-Nano family knobs. The `--itn` flag is shared:
     // it routes to whichever family the loaded model belongs to. Ignored
-    // by non-ITN-aware families.
-    bool use_itn           = false;  // --itn
+    // by non-ITN-aware families. Unset leaves the library default in place,
+    // which differs per family (sensevoice: on; funasr-nano: off), so the
+    // initializer here is only read once --itn / --no-itn has been seen.
+    bool use_itn           = false;  // --itn / --no-itn
     bool itn_set           = false;
     bool keep_special_tags = false;  // --raw-tokens
 
@@ -318,7 +320,10 @@ void print_usage(const char * argv0) {
                  "  --temperature F       (whisper) tier-0 sampling temperature (default 0 = greedy)\n"
                  "  --condition-on-prev-tokens (whisper) carry prev-chunk tokens across chunks\n"
                  "  --prompt-condition T  (whisper) prompt placement: first|all (default: first)\n"
-                 "  --itn                 (sensevoice/funasr-nano) enable inverse text normalization\n"
+                 "  --itn                 (sensevoice/funasr-nano) enable inverse text\n"
+                 "                        normalization (sensevoice: on unless --no-itn)\n"
+                 "  --no-itn              (sensevoice/funasr-nano) emit the upstream\n"
+                 "                        spoken-form text instead\n"
                  "  --pnc                 (canary) emit punctuation and capitalization (default)\n"
                  "  --no-pnc              (canary) emit lowercase de-punctuated text\n"
                  "  --diarize             (moss/granite-plus) speaker attribution: segments carry\n"
@@ -588,6 +593,9 @@ bool parse_args(int argc, char ** argv, cli_args & out) {
         } else if (a == "--itn") {
             out.use_itn = true;
             out.itn_set = true;
+        } else if (a == "--no-itn") {
+            out.use_itn = false;
+            out.itn_set = true;
         } else if (a == "--pnc") {
             out.canary_pnc     = true;
             out.canary_pnc_set = true;

include/transcribe.h

@@ -508,7 +508,16 @@ enum transcribe_pnc_mode {
  * transcribe_model_supports(model, TRANSCRIBE_FEATURE_ITN) returns false
  * emit a WARN and proceed with default behavior.
  *
- *   DEFAULT (0): family default. Zero-init gives this value.
+ *   DEFAULT (0): family default. Zero-init gives this value. Most families
+ *                follow their upstream default; `sensevoice` is the
+ *                exception and resolves DEFAULT to ON, because there the
+ *                ITN toggle is also the only source of casing and
+ *                punctuation, so ITN-off would hand an unconfigured caller
+ *                lowercase unpunctuated text. For sensevoice, DEFAULT is
+ *                therefore NOT the setting the published WER tables were
+ *                measured at — the harness pins ITN off (docs/tools/wer.md).
+ *                `funasr_nano` has the same shape of toggle but keeps the
+ *                upstream `itn=False` default.
  *   OFF:         explicit ITN off. Supporting families emit verbatim
  *                spoken-form text. Non-supporting families ignore (WARN).
  *   ON:          explicit ITN on. Supporting families apply ITN.

scripts/hf_cards/sensevoice-small.yaml

@@ -47,8 +47,10 @@ summary: |
   30 seconds per call, per upstream's direct-inference contract) and produces
   a transcript. Not a streaming model, no translation, no built-in long-form
   chunking. The same CTC head also emits language-ID, simple emotion labels,
-  audio-event tags, and an inverse-text-normalization flag — opt-in via
-  `--raw-tokens` and `--itn`.
+  audio-event tags, and inverse-text-normalization control tags. These tags are
+  hidden unless `--raw-tokens` is passed. ITN is on by default for readable
+  casing, punctuation, and digits; pass `--no-itn` for upstream's spoken-form
+  output.
 
 default_quant_index: 2  # Q8_0
 
@@ -79,7 +81,9 @@ wer:
     manifest: 3.13% (95% CI [2.93%, 3.34%]). transcribe.cpp's F32 port matches
     that baseline within +0.002 percentage-points. LibriSpeech is an English
     benchmark; SenseVoice's strongest case is Mandarin, and AISHELL-1 (CER)
-    is the recommended complementary check.
+    is the recommended complementary check. These table values were measured
+    with ITN off, matching the FunASR reference; `scripts/wer/run.py` pins
+    `--no-itn` so the benchmark does not inherit the runtime default.
 
 quants:
   - name: F32

scripts/validate.py

@@ -509,6 +509,20 @@ def cmd_cpp(args: argparse.Namespace) -> int:
             # strips these by default, so the validate dump must pass
             # --raw-tokens to keep them and match the reference exactly.
             cmd += ["--raw-tokens"]
+        if args.family in ("sensevoice", "funasr_nano"):
+            # Pin ITN off rather than inheriting the run-time default, which
+            # is per-family and can change (sensevoice already defaults to ITN
+            # *on* so an unconfigured caller gets readable text; funasr_nano
+            # follows upstream's `itn=False`). The reference dumpers always run
+            # `itn=False`, and ITN is not cosmetic on either side: sensevoice
+            # selects a different textnorm prefix *embedding* prepended to the
+            # encoder input, and funasr_nano changes the prompt token
+            # sequence. Inheriting the default would compare C++ ITN-on
+            # tensors against ITN-off reference tensors and fail the gate for
+            # a reason that has nothing to do with numerics. Explicit for both
+            # families so a future default flip cannot silently break the
+            # gate. Same pin, same reason, as scripts/wer/run.py.
+            cmd += ["--no-itn"]
         cmd.append(str(audio))
 
         print(f"\n{'=' * 60}", file=sys.stderr)