From a25cdb5233cb51fa01fa6c0bb213658431647715 Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Fri, 11 Sep 2026 02:24:05 -0700 Subject: [PATCH] hooks: one shared escape-hatch tag, and bare UNVERIFIED: excuses nothing Bare `UNVERIFIED:` silenced every check in the message it appeared in. It cost four characters and bought a whole turn, so it got used to stop checking rather than to report a check that could not run. Six hooks read that marker, each with a private regex, and they had drifted: hedge-runs-prove-it required a named blocker, diu-stop did not. engine/hooks/_markers now holds one definition of {{CAT-UNVERIFIED: -- cannot verify: }}, the names-a-blocker test, and the retired bare marker. install.sh links it beside the hooks. Every suppressor reads it: diu-stop, external-claim-gate (which borrows diu-stop's matcher at detect.py:13), hedge-runs-prove-it, history-claim-check, incidence-needs-repetition, named-verb-guard and prove-it-ship-gate. A well-formed tag excuses its paragraph. A tag that names no blocker excuses nothing. Bare UNVERIFIED: is prose, and the block text names the tag that replaced it. Same three messages through diu-stop, on 509b9cd and on this commit: legacy marker in front of a real claim before exit=2 contains an `UNVERIFIED:` claim... after exit=2 unverified-shaped claim ("because")... well-formed tag beside the claim before exit=2 contains an `UNVERIFIED:` claim... after exit=0 (silent) tag naming no blocker before exit=2 contains an `UNVERIFIED:` claim... after exit=2 unverified-shaped claim ("because")... Two bugs found by tests while building this, both now pinned: - A hook directory is itself a symlink, so joining ".." onto it resolves the link first and lands beside the checkout. Two textual dirnames reach the sibling module; test_installed_layout builds the symlinked layout. - Block texts are .format() templates, so a literal {{CAT-UNVERIFIED}} rendered as {CAT-UNVERIFIED}. Every existing test passed through it, because they asserted the substring CAT-UNVERIFIED. The tag is now a format argument, and _markers tests assert on the braces. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VKsvxJk65w6q7KnPSRYvNg --- engine/hooks/_markers/markers.py | 78 +++++++++++++ .../_markers/tests/test_installed_layout.py | 86 +++++++++++++++ engine/hooks/_markers/tests/test_markers.py | 103 ++++++++++++++++++ engine/hooks/diu-stop/COVERAGE.md | 2 +- engine/hooks/diu-stop/claude_stop_check.py | 67 ++++++++---- .../hooks/diu-stop/tests/test_fix_matrix.py | 27 +++-- engine/hooks/diu-stop/tests/test_hooks.py | 68 +++++++++--- engine/hooks/external-claim-gate/README.md | 4 +- engine/hooks/external-claim-gate/detect.py | 11 +- .../external-claim-gate/tests/test_hooks.py | 10 +- engine/hooks/hedge-runs-prove-it/README.md | 10 +- engine/hooks/hedge-runs-prove-it/detect.py | 27 +++-- .../tests/fixtures/diagnosis_silent.json | 4 +- .../tests/fixtures/hedges_silent.json | 4 +- engine/hooks/history-claim-check/README.md | 2 +- engine/hooks/history-claim-check/detect.py | 14 ++- .../history-claim-check/tests/test_hooks.py | 4 +- .../incidence-needs-repetition/README.md | 4 +- .../incidence-needs-repetition/detect.py | 18 ++- .../tests/fixtures/incidence_silent.json | 2 +- engine/hooks/named-verb-guard/README.md | 2 +- engine/hooks/named-verb-guard/detect.py | 15 ++- .../named-verb-guard/tests/test_hooks.py | 6 +- engine/hooks/prove-it-ship-gate/README.md | 2 +- engine/hooks/prove-it-ship-gate/detect.py | 15 ++- .../prove-it-ship-gate/tests/test_hooks.py | 7 +- install.sh | 1 + tests/scenarios/prove-it-claims.json | 30 ++++- 28 files changed, 513 insertions(+), 110 deletions(-) create mode 100644 engine/hooks/_markers/markers.py create mode 100644 engine/hooks/_markers/tests/test_installed_layout.py create mode 100644 engine/hooks/_markers/tests/test_markers.py diff --git a/engine/hooks/_markers/markers.py b/engine/hooks/_markers/markers.py new file mode 100644 index 00000000..422dff13 --- /dev/null +++ b/engine/hooks/_markers/markers.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""One definition of the escape-hatch marker, shared by every evidence hook. + +`{{CAT-UNVERIFIED: -- cannot verify: }}` is the only marker +that excuses a claim. It excuses the paragraph it sits in, and only when it +names a blocker after `cannot verify:`. A tag that names nothing does not +excuse anything -- that shape is the one an author reaches for to end a turn +without checking, which is what the marker exists to prevent. + +Bare `UNVERIFIED:` is ordinary prose. It used to be the escape hatch, so a +message still carrying it gets `legacy_marker` set and is told which tag to +use instead, rather than silently losing the suppression it expected. + +Why a module and not a regex copied into each hook: six hooks read this +marker. Two classifiers over the same input drift apart silently -- the +reason clause landed in hedge-runs-prove-it and never reached diu-stop. + +Hooks are installed as sibling symlinks under $HOME/.claude/hooks/, so a +hook reaches this module by its own parent directory: + + sys.path.insert(0, os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_markers")) + +Two dirnames, not a "..": the hook's own directory is a symlink, so the OS +resolves it before applying "..", landing beside the checkout instead of +beside the other hooks. Stripping two segments textually cannot do that, and +abspath (never realpath) is what keeps the installed path in place. +""" +from __future__ import annotations + +import re + +TAG_RE = re.compile(r"\{\{\s*CAT-UNVERIFIED\b(?P[^}]*)\}\}", re.IGNORECASE) +REASON_RE = re.compile(r"cannot\s+verify\s*:\s*(?P\S.*)", re.IGNORECASE | re.DOTALL) +LEGACY_RE = re.compile(r"(? -- cannot verify: }}" + +MALFORMED_TAG_MESSAGE = ( + "A `{{CAT-UNVERIFIED}}` tag here names no blocker. Per " + "skills/prove-it/SKILL.md the tag is for a check that cannot run, and it " + f"has to say why: `{TAG_TEMPLATE}`. If the check can run, run it and paste " + "the output instead." +) + +LEGACY_MARKER_MESSAGE = ( + "`UNVERIFIED:` is no longer an escape hatch -- it reads as ordinary prose " + "and the claim beside it is judged on its own. Per skills/prove-it/SKILL.md: " + "run the check and paste its output. Only if the check genuinely cannot run, " + f"write `{TAG_TEMPLATE}`." +) + + +def _names_a_blocker(body: str) -> bool: + match = REASON_RE.search(body) + if not match: + return False + return bool(match.group("reason").strip(" -_.:\t\r\n")) + + +def well_formed_tags(text: str) -> list[str]: + """Every `{{CAT-UNVERIFIED: ...}}` in `text` that names a blocker.""" + return [m.group(0) for m in TAG_RE.finditer(text) if _names_a_blocker(m.group("body"))] + + +def malformed_tags(text: str) -> list[str]: + """Every `{{CAT-UNVERIFIED: ...}}` in `text` that names no blocker.""" + return [m.group(0) for m in TAG_RE.finditer(text) if not _names_a_blocker(m.group("body"))] + + +def excuses_paragraph(paragraph: str) -> bool: + """True when this paragraph carries a tag that names a blocker.""" + return bool(well_formed_tags(paragraph)) + + +def has_legacy_marker(text: str) -> bool: + """True when `text` still uses the retired bare `UNVERIFIED:` marker.""" + return bool(LEGACY_RE.search(text)) diff --git a/engine/hooks/_markers/tests/test_installed_layout.py b/engine/hooks/_markers/tests/test_installed_layout.py new file mode 100644 index 00000000..dfc274cb --- /dev/null +++ b/engine/hooks/_markers/tests/test_installed_layout.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""The sibling-path import has to resolve where hooks actually run. + +install.sh links each hook directory separately into $HOME/.claude/hooks/, so +a hook reaches this module through its own parent directory. That only works +because the path is built with abspath, which leaves the symlink in place; +realpath would jump into the checkout and find nothing beside it in a partial +install. This test builds that layout for real rather than trusting the shape. + +Run: python3 -m unittest discover -s engine/hooks/_markers/tests -v +""" +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +MARKERS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HOOKS_DIR = os.path.dirname(MARKERS_DIR) + +IMPORT_LINE = ( + "import os, sys\n" + "sys.path.insert(0, os.path.join(" + "os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '_markers'))\n" + "import markers\n" + "print(len(markers.well_formed_tags(" + "'{{CAT-UNVERIFIED: x -- cannot verify: offline}}')))\n" +) + + +class InstalledLayout(unittest.TestCase): + def setUp(self): + self.home = tempfile.mkdtemp(prefix="markers-install-") + self.addCleanup(shutil.rmtree, self.home, ignore_errors=True) + self.installed_hooks = os.path.join(self.home, ".claude", "hooks") + os.makedirs(self.installed_hooks) + + def _link(self, name, src): + target = os.path.join(self.installed_hooks, name) + os.symlink(src, target) + return target + + def _run_consumer(self, consumer_dir): + script = os.path.join(consumer_dir, "detect.py") + with open(script, "w", encoding="utf-8") as handle: + handle.write(IMPORT_LINE) + return subprocess.run( + [sys.executable, os.path.join(self.installed_hooks, "consumer", "detect.py")], + capture_output=True, + text=True, + ) + + def test_consumer_imports_markers_through_symlinked_siblings(self): + self._link("_markers", MARKERS_DIR) + real_consumer = tempfile.mkdtemp(prefix="markers-consumer-") + self.addCleanup(shutil.rmtree, real_consumer, ignore_errors=True) + self._link("consumer", real_consumer) + + result = self._run_consumer(real_consumer) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "1") + + def test_missing_markers_link_fails_loudly_instead_of_passing_clean(self): + real_consumer = tempfile.mkdtemp(prefix="markers-consumer-") + self.addCleanup(shutil.rmtree, real_consumer, ignore_errors=True) + self._link("consumer", real_consumer) + + result = self._run_consumer(real_consumer) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("ModuleNotFoundError", result.stderr) + + +class InstallerWiring(unittest.TestCase): + def test_install_sh_links_the_markers_module(self): + install_sh = os.path.join(os.path.dirname(os.path.dirname(HOOKS_DIR)), "install.sh") + with open(install_sh, encoding="utf-8") as handle: + body = handle.read() + self.assertIn('link_item "_markers"', body) + self.assertIn('"$HOME/.claude/hooks/_markers"', body) + + +if __name__ == "__main__": + unittest.main() diff --git a/engine/hooks/_markers/tests/test_markers.py b/engine/hooks/_markers/tests/test_markers.py new file mode 100644 index 00000000..37d3e84a --- /dev/null +++ b/engine/hooks/_markers/tests/test_markers.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Unit tests for the shared escape-hatch marker. + +Run: python3 -m unittest discover -s engine/hooks/_markers/tests -v +""" +import os +import sys +import unittest + +MARKERS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, MARKERS_DIR) + +import markers # noqa: E402 + +WELL_FORMED = "{{CAT-UNVERIFIED: the glob covers it -- cannot verify: no CI access from here}}" +NO_BLOCKER = "{{CAT-UNVERIFIED: the glob covers it}}" +EMPTY_BLOCKER = "{{CAT-UNVERIFIED: the glob covers it -- cannot verify: }}" + + +class TagRecognition(unittest.TestCase): + def test_tag_naming_a_blocker_is_well_formed(self): + self.assertEqual(markers.well_formed_tags(WELL_FORMED), [WELL_FORMED]) + self.assertEqual(markers.malformed_tags(WELL_FORMED), []) + + def test_tag_naming_no_blocker_is_malformed(self): + self.assertEqual(markers.well_formed_tags(NO_BLOCKER), []) + self.assertEqual(markers.malformed_tags(NO_BLOCKER), [NO_BLOCKER]) + + def test_blocker_with_only_whitespace_does_not_count(self): + self.assertEqual(markers.well_formed_tags(EMPTY_BLOCKER), []) + self.assertEqual(markers.malformed_tags(EMPTY_BLOCKER), [EMPTY_BLOCKER]) + + def test_tag_is_case_insensitive_and_tolerates_inner_spaces(self): + text = "{{ cat-unverified: x -- Cannot Verify: the box is offline }}" + self.assertEqual(len(markers.well_formed_tags(text)), 1) + + def test_several_tags_are_each_classified(self): + text = f"{WELL_FORMED}\n\n{NO_BLOCKER}" + self.assertEqual(len(markers.well_formed_tags(text)), 1) + self.assertEqual(len(markers.malformed_tags(text)), 1) + + +class ParagraphExcuse(unittest.TestCase): + def test_well_formed_tag_excuses_its_paragraph(self): + self.assertTrue(markers.excuses_paragraph(f"The job passes. {WELL_FORMED}")) + + def test_malformed_tag_does_not_excuse_its_paragraph(self): + self.assertFalse(markers.excuses_paragraph(f"The job passes. {NO_BLOCKER}")) + + def test_clean_paragraph_is_not_excused(self): + self.assertFalse(markers.excuses_paragraph("The job passes.")) + + +class LegacyMarker(unittest.TestCase): + def test_bare_marker_is_reported_as_legacy(self): + self.assertTrue(markers.has_legacy_marker("UNVERIFIED: the glob covers it")) + + def test_bare_marker_never_excuses_a_paragraph(self): + self.assertFalse(markers.excuses_paragraph("UNVERIFIED: the glob covers it")) + + def test_new_tag_is_not_reported_as_legacy(self): + self.assertFalse(markers.has_legacy_marker(WELL_FORMED)) + + def test_clean_text_has_no_legacy_marker(self): + self.assertFalse(markers.has_legacy_marker("The job passes.")) + + +class Messages(unittest.TestCase): + def test_both_messages_show_the_literal_tag_template(self): + self.assertIn("{{CAT-UNVERIFIED: -- cannot verify: }}", markers.MALFORMED_TAG_MESSAGE) + self.assertIn("{{CAT-UNVERIFIED: -- cannot verify: }}", markers.LEGACY_MARKER_MESSAGE) + +class RenderedMessagesKeepTheirBraces(unittest.TestCase): + """Every hook that names the tag in a block message renders it with both + braces intact. + + These messages are `.format()` templates. A literal `{{CAT-UNVERIFIED}}` + written into one collapses to `{CAT-UNVERIFIED}` when it is rendered, and + the hook then tells the author to type a tag that no hook recognises. + That shipped once here and every existing test still passed, because they + all asserted on the substring `CAT-UNVERIFIED`, which survives the + collapse. These assert on the braces.""" + + def test_external_claim_gate_renders_both_braces(self): + sys.path.insert(0, os.path.join(os.path.dirname(MARKERS_DIR), "external-claim-gate")) + import detect as external_detect # noqa: PLC0415 + message = external_detect.block_message([ + external_detect.Finding(outcome="hit", destination="gh issue create", + claim="because", detail="x"), + ]) + self.assertIn(markers.TAG_TEMPLATE, message) + + def test_tag_template_itself_has_double_braces(self): + self.assertTrue(markers.TAG_TEMPLATE.startswith("{{")) + self.assertTrue(markers.TAG_TEMPLATE.endswith("}}")) + + def test_a_collapsed_tag_is_not_recognised(self): + collapsed = markers.TAG_TEMPLATE.replace("{{", "{").replace("}}", "}") + self.assertEqual(markers.well_formed_tags(collapsed), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/engine/hooks/diu-stop/COVERAGE.md b/engine/hooks/diu-stop/COVERAGE.md index 2d46018a..1d0770c1 100644 --- a/engine/hooks/diu-stop/COVERAGE.md +++ b/engine/hooks/diu-stop/COVERAGE.md @@ -51,7 +51,7 @@ What does not: - Claims the patterns do not match. The check is a text proxy. It looks for claim-shaped phrases (the banned phrases, a "confirmed" or "verified" opener, a causal closer, a hedged cause) and for evidence-shaped text in - the same paragraph (a fence, output-shaped inline code, `UNVERIFIED:`). It + the same paragraph (a fence, output-shaped inline code, a well-formed `{{CAT-UNVERIFIED}}`). It cannot tell whether the evidence is real. Of the 141 claims on this surface, the current patterns match 45 and miss 96. diff --git a/engine/hooks/diu-stop/claude_stop_check.py b/engine/hooks/diu-stop/claude_stop_check.py index eb51e692..09b3d506 100755 --- a/engine/hooks/diu-stop/claude_stop_check.py +++ b/engine/hooks/diu-stop/claude_stop_check.py @@ -19,18 +19,32 @@ log volume, not a crash loop), and "the fix... never pushed" (it had pushed; a downstream fetch just hadn't caught up yet). All three had the same shape: a bare declarative claim opening the message, no adjacent -evidence and no `UNVERIFIED:` prefix. This can't verify the evidence is +evidence and no escape-hatch marker. This can't verify the evidence is real -- only that *something evidence-shaped* (a fenced block, inline code -that looks like output, or `UNVERIFIED:` itself) sits near the claim. See +that looks like output, or the marker itself) sits near the claim. See skills/prove-it/SKILL.md in the Invoker repo for the full discipline this mechanically nudges toward. + +The marker is `{{CAT-UNVERIFIED: -- cannot verify: }}`, and +it excuses the paragraph it sits in, the way a fence does. Bare +`UNVERIFIED:` used to excuse the whole message and no longer excuses +anything: it was reachable by typing four characters, which made it the +cheapest way to end a turn, and it was used that way. A tag that names no +blocker is the same move wearing the new syntax, so it does not excuse +either. """ import json +import os import re import sys from diu_limit import WORD_LIMIT, counted_words +sys.path.insert(0, os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_markers")) + +import markers # noqa: E402 + # Phrases banned outright (from this user's global CLAUDE.md evidence # rules) -- rarely legitimate even mid-sentence, so no opener restriction. BANNED_PHRASES_UNCONDITIONAL = [ @@ -99,29 +113,39 @@ def _opening_word(message): stripped = message.lstrip() stripped = re.sub(r"^[*_\-#\s]+", "", stripped) + stripped = markers.LEGACY_RE.sub("", stripped, count=1).lstrip() match = re.match(r"[A-Za-z']+", stripped) return match.group(0).lower() if match else "" -def has_unresolved_unverified_marker(message): - return bool(UNVERIFIED_RE.search(message)) +def find_marker_problems(message): + """Return the marker complaints this message earns, in report order. + + A tag that names no blocker, and the retired bare `UNVERIFIED:`, each + draw their own message. Both can be present at once.""" + problems = [] + if markers.malformed_tags(message): + problems.append(markers.MALFORMED_TAG_MESSAGE) + if markers.has_legacy_marker(message): + problems.append(markers.LEGACY_MARKER_MESSAGE) + return problems def find_unverified_claim(message): """Return the offending phrase if a paragraph makes an unverified-shaped claim with no evidence marker in that same paragraph. - `UNVERIFIED:` anywhere still silences the whole message. A fence only - silences the paragraph it sits in -- not a later/earlier claim. Inline - code silences its paragraph only when it looks like command output or - the message carries a fenced block of output. This is a blunt proxy, - not a truth check.""" - if UNVERIFIED_RE.search(message): - return None + A well-formed `{{CAT-UNVERIFIED}}` tag silences the paragraph it sits + in, exactly like a fence -- not a later/earlier claim. Inline code + silences its paragraph only when it looks like command output or the + message carries a fenced block of output. This is a blunt proxy, not a + truth check.""" fenced_output = any(OUTPUT_SHAPE_RE.search(body) for body in FENCED_BODY_RE.findall(message)) for para in re.split(r"\n\s*\n", message): if FENCE_MARKER in para: continue + if markers.excuses_paragraph(para): + continue inline = INLINE_CODE_RE.findall(para) if inline and (fenced_output or any(OUTPUT_SHAPE_RE.search(code) for code in inline)): continue @@ -160,27 +184,22 @@ def main(): word_count = counted_words(message) over_limit = word_count > WORD_LIMIT claim = find_unverified_claim(message) - unverified_marker = has_unresolved_unverified_marker(message) + marker_problems = find_marker_problems(message) - if not over_limit and not claim and not unverified_marker: + if not over_limit and not claim and not marker_problems: return parts = [] if claim: parts.append( f"This message makes an unverified-shaped claim (\"{claim}\") with no " - "adjacent evidence (pasted command output, or an `UNVERIFIED:` " - "prefix). A backticked name or command alone is not output. Per " - "skills/prove-it/SKILL.md: either paste the output of what was " - "actually run/checked, or prefix the claim with `UNVERIFIED:`." - ) - if unverified_marker: - parts.append( - "This message contains an `UNVERIFIED:` claim. Per skills/prove-it/" - "SKILL.md: attempt to verify it now (run the actual check) before " - "finishing this turn, or tell the user explicitly what is blocking " - "verification and why it can't happen right now." + "adjacent evidence (pasted command output, or a " + f"`{markers.TAG_TEMPLATE}` tag). A backticked name or command alone " + "is not output. Per skills/prove-it/SKILL.md: either paste the " + "output of what was actually run/checked, or -- only if the check " + "cannot run -- tag the claim and say why." ) + parts.extend(marker_problems) if over_limit: parts.append( f"Apply diu: {word_count} words, over the {WORD_LIMIT}-word " diff --git a/engine/hooks/diu-stop/tests/test_fix_matrix.py b/engine/hooks/diu-stop/tests/test_fix_matrix.py index 53263043..0dd1761d 100644 --- a/engine/hooks/diu-stop/tests/test_fix_matrix.py +++ b/engine/hooks/diu-stop/tests/test_fix_matrix.py @@ -33,22 +33,33 @@ "fixed": "I think the deploy happened, confirmed via `deploy status` printing `SUCCESS`.", }, { - "name": "unverified_marker_resolved_by_verifying", + "name": "legacy_marker_resolved_by_verifying", "broken": "UNVERIFIED: the deploy caused the outage.", "fixed": "Confirmed via the deploy log line `PoolTimeoutError: connection pool exhausted` -- the deploy caused the outage.", }, + { + "name": "legacy_marker_resolved_by_naming_the_blocker", + "broken": "UNVERIFIED: the deploy caused the outage.", + "fixed": ( + "The deploy caused the outage. " + "{{CAT-UNVERIFIED: the deploy caused it -- cannot verify: the log host is offline}}" + ), + }, ] KNOWN_DOUBLE_BLOCKS = [ { - "name": "hedge_hedged_into_unverified_still_needs_a_second_look", + "name": "hedge_tagged_without_a_blocker_still_needs_a_second_look", "broken": "I think the deploy happened around 2am, so that's why the build is stale.", - "hedged": "UNVERIFIED: I think the deploy happened around 2am, so that's why the build is stale.", + "hedged": ( + "I think the deploy happened around 2am, so that's why the build is stale. " + "{{CAT-UNVERIFIED: the 2am deploy}}" + ), "reason": ( - "Prefixing UNVERIFIED: silences the hedge/causal/opener checks, " - "but the unverified-marker check still fires -- a claim that was " - "hedged into 'I don't know' should still prompt one more nudge " - "to go verify, not become a silent free pass." + "A tag that names no blocker does not excuse its paragraph, so the " + "hedge check still fires and the malformed-tag check fires beside " + "it. Two complaints about the same sentence is correct here: the " + "rewrite that fixes both is to name the blocker or go check." ), }, ] @@ -87,7 +98,7 @@ def test_hedged_message_still_blocks_once(self): with self.subTest(case=case["name"]): blocked, err = run_claude_check({"last_assistant_message": case["hedged"]}) self.assertTrue(blocked, f"{case['name']}: expected the documented double-block to fire") - self.assertIn("UNVERIFIED", err) + self.assertIn("CAT-UNVERIFIED", err) def test_hedged_message_passes_on_stop_hook_active_retry(self): for case in KNOWN_DOUBLE_BLOCKS: diff --git a/engine/hooks/diu-stop/tests/test_hooks.py b/engine/hooks/diu-stop/tests/test_hooks.py index bf208520..d18ae41d 100644 --- a/engine/hooks/diu-stop/tests/test_hooks.py +++ b/engine/hooks/diu-stop/tests/test_hooks.py @@ -223,25 +223,47 @@ def test_confirmed_opener_with_evidence_marker_is_allowed(self): self.assertFalse(blocked) self.assertEqual(err, "") - def test_unverified_prefix_suppresses_the_older_checks(self): - message = "UNVERIFIED: confirmed the crash loop, but I have not checked the actual log source yet." + def test_well_formed_tag_suppresses_the_older_checks(self): + message = ( + "Confirmed the crash loop. " + "{{CAT-UNVERIFIED: the loop is real -- cannot verify: no log access from here}}" + ) self.assertIsNone(claude_stop_check.find_unverified_claim(message)) - def test_unverified_marker_triggers_a_prove_it_block_once(self): + def test_well_formed_tag_alone_does_not_block(self): + message = ( + "Confirmed the crash loop. " + "{{CAT-UNVERIFIED: the loop is real -- cannot verify: no log access from here}}" + ) + blocked, err = run_claude_check({"last_assistant_message": message}) + self.assertFalse(blocked) + self.assertEqual(err, "") + + def test_tag_naming_no_blocker_is_blocked(self): + message = "Confirmed the crash loop. {{CAT-UNVERIFIED: the loop is real}}" + blocked, err = run_claude_check({"last_assistant_message": message}) + self.assertTrue(blocked) + self.assertIn("names no blocker", err) + self.assertIn("prove-it", err.lower()) + + def test_legacy_bare_marker_no_longer_suppresses_the_claim(self): + message = "UNVERIFIED: confirmed the crash loop, but I have not checked the actual log source yet." + self.assertIsNotNone(claude_stop_check.find_unverified_claim(message)) + + def test_legacy_bare_marker_is_blocked_and_names_the_new_tag(self): message = "UNVERIFIED: confirmed the crash loop, but I have not checked the actual log source yet." blocked, err = run_claude_check({"last_assistant_message": message}) self.assertTrue(blocked) - self.assertIn("UNVERIFIED", err) + self.assertIn("no longer an escape hatch", err) + self.assertIn("CAT-UNVERIFIED", err) self.assertIn("prove-it", err.lower()) - def test_unverified_marker_retry_is_allowed_via_stop_hook_active(self): - message = "UNVERIFIED: still can't verify this without the user's input." - blocked, err = run_claude_check({ - "last_assistant_message": message, - "stop_hook_active": True, - }) - self.assertFalse(blocked) - self.assertEqual(err, "") + def test_tag_excuses_only_its_own_paragraph(self): + message = ( + "{{CAT-UNVERIFIED: the runner is green -- cannot verify: CI is unreachable}}\n\n" + "The owner crashed because the lock never released." + ) + self.assertEqual(claude_stop_check.find_unverified_claim(message), "because") def test_ordinary_message_without_banned_language_passes(self): blocked, err = run_claude_check({"last_assistant_message": "I'll check the logs next and report back."}) @@ -252,10 +274,17 @@ def test_reproduces_the_incident_empty_because_send_never_executed(self): message = "The UI is empty because send never executed." self.assertIsNotNone(claude_stop_check.find_unverified_claim(message)) - def test_unverified_causal_closer_is_allowed(self): - message = "UNVERIFIED: The UI is empty because send never executed." + def test_tagged_causal_closer_is_allowed(self): + message = ( + "The UI is empty because send never executed. " + "{{CAT-UNVERIFIED: send never executed -- cannot verify: the app will not launch here}}" + ) self.assertIsNone(claude_stop_check.find_unverified_claim(message)) + def test_legacy_marker_no_longer_allows_a_causal_closer(self): + message = "UNVERIFIED: The UI is empty because send never executed." + self.assertIsNotNone(claude_stop_check.find_unverified_claim(message)) + def test_backticked_name_does_not_silence_causal_closer(self): message = "The UI is empty because `planning-chat-send` never executed." self.assertIsNotNone(claude_stop_check.find_unverified_claim(message)) @@ -294,10 +323,17 @@ def test_hedge_recommendation_is_not_a_claim_and_is_allowed(self): message = "I think we should refactor this module before adding more workers." self.assertIsNone(claude_stop_check.find_unverified_claim(message)) - def test_unverified_prefix_suppresses_hedge_claim_too(self): - message = "UNVERIFIED: I think this happened because of a stale process." + def test_tag_suppresses_hedge_claim_too(self): + message = ( + "I think this happened because of a stale process. " + "{{CAT-UNVERIFIED: a stale process -- cannot verify: the host is powered down}}" + ) self.assertIsNone(claude_stop_check.find_unverified_claim(message)) + def test_legacy_marker_no_longer_suppresses_hedge_claim(self): + message = "UNVERIFIED: I think this happened because of a stale process." + self.assertIsNotNone(claude_stop_check.find_unverified_claim(message)) + def test_both_violations_are_reported_when_both_present(self): # A real session (2026-08-31) let 3+ wrong root-cause claims reach # the user specifically because each one was ALSO over the word diff --git a/engine/hooks/external-claim-gate/README.md b/engine/hooks/external-claim-gate/README.md index 5ebd08ce..73ad5f4f 100644 --- a/engine/hooks/external-claim-gate/README.md +++ b/engine/hooks/external-claim-gate/README.md @@ -37,7 +37,7 @@ Any of these, anywhere in the same body: - a fenced block (three backticks or three tildes) - a `file:line` reference (`src/cache/store.py:88`, or `store.py#L88`) - pasted command output, shown by a prompt line (`$ cmd` or `>>> expr`) -- an explicit `UNVERIFIED:` marker +- an explicit `{{CAT-UNVERIFIED: -- cannot verify: }}` tag Inline backticks clear the paragraph they sit in, because the shared matcher works that way. @@ -48,7 +48,7 @@ A body with no claim passes: a feature request, a question, a status note. | Outcome | What happens | | --- | --- | -| hit | blocked (exit 2). The message quotes the claim, lists the evidence that is missing, and gives the two ways through: add the evidence to the body, or put `UNVERIFIED:` before the claim. | +| hit | blocked (exit 2). The message quotes the claim, lists the evidence that is missing, and gives the two ways through: add the evidence to the body, or tag the claim with `{{CAT-UNVERIFIED: ... -- cannot verify: }}`. | | clean | allowed | | UNCHECKED | blocked (exit 2). The body could not be read, so it has not been cleared. | diff --git a/engine/hooks/external-claim-gate/detect.py b/engine/hooks/external-claim-gate/detect.py index 2f5116b7..8673350a 100644 --- a/engine/hooks/external-claim-gate/detect.py +++ b/engine/hooks/external-claim-gate/detect.py @@ -7,6 +7,11 @@ import re import stat import sys + +sys.path.insert(0, os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_markers")) + +import markers # noqa: E402 from dataclasses import dataclass, field HERE = os.path.dirname(os.path.realpath(__file__)) @@ -494,7 +499,7 @@ def evaluate(command: str, cwd: str | None = None, depth: int = 0, "with no evidence in the same body.\n" " claim: \"{claim}\" in: \"{paragraph}\"\n" " missing: the body has no fenced block, no file:line reference, no pasted command " - "output (a `$ ` prompt line), and no `UNVERIFIED:` marker." + "output (a `$ ` prompt line), and no CAT-UNVERIFIED tag." ) UNCHECKED_MESSAGE = ( "external-claim-gate: UNCHECKED: the body of `{destination}` could not be read, so it " @@ -506,7 +511,7 @@ def evaluate(command: str, cwd: str | None = None, depth: int = 0, "Two ways through:\n" " 1. Add the evidence to the body: the command you ran with its real output in a " "fenced block, or the file:line you read.\n" - " 2. Prefix the claim with `UNVERIFIED:`." + " 2. If the check cannot run, tag the claim and say why: `{tag}`." ) @@ -518,5 +523,5 @@ def block_message(findings: list[Finding]) -> str: destination=finding.destination, claim=finding.claim, paragraph=finding.detail)) else: parts.append(UNCHECKED_MESSAGE.format(destination=finding.destination, detail=finding.detail)) - parts.append(EXITS) + parts.append(EXITS.format(tag=markers.TAG_TEMPLATE)) return "\n\n".join(parts) diff --git a/engine/hooks/external-claim-gate/tests/test_hooks.py b/engine/hooks/external-claim-gate/tests/test_hooks.py index 45ad3004..312edf17 100644 --- a/engine/hooks/external-claim-gate/tests/test_hooks.py +++ b/engine/hooks/external-claim-gate/tests/test_hooks.py @@ -12,6 +12,8 @@ HOOK_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, HOOK_DIR) +TAGGED = "{{CAT-UNVERIFIED: the cause -- cannot verify: the log host is offline}}" + import detect # noqa: E402 from detect import block_message, evaluate # noqa: E402 @@ -78,7 +80,7 @@ def test_block_message_names_claim_missing_evidence_and_both_exits(self): self.assertIn('claim: "because"', message) self.assertIn("no fenced block, no file:line reference, no pasted command output", message) self.assertIn("Add the evidence to the body", message) - self.assertIn("Prefix the claim with `UNVERIFIED:`", message) + self.assertIn("CAT-UNVERIFIED", message) class TestSilentWhenEvidenceOrNoClaim(unittest.TestCase): @@ -97,7 +99,7 @@ def test_silent_with_pasted_command_output(self): self.assertEqual(outcomes("gh pr comment 7 --body-file b.md", d), []) def test_silent_with_unverified_marker(self): - self.assertEqual(outcomes(f"gh release create v2 --notes 'UNVERIFIED: {CAUSE}'"), []) + self.assertEqual(outcomes(f"gh release create v2 --notes '{CAUSE} {TAGGED}'"), []) def test_silent_feature_request(self): body = "Feature request: add a --json flag to the status command." @@ -178,7 +180,7 @@ def test_unchecked_message_says_unchecked_and_how_to_make_it_readable(self): message = block_message(evaluate('gh pr comment 9 --body "$BODY"')) self.assertIn("UNCHECKED", message) self.assertIn("could not be read", message) - self.assertIn("Prefix the claim with `UNVERIFIED:`", message) + self.assertIn("CAT-UNVERIFIED", message) class TestReusesTheDiuStopMatcher(unittest.TestCase): @@ -208,7 +210,7 @@ def test_hook_blocks_unchecked_body_with_exit_2(self): def test_hook_allows_evidenced_body_with_exit_0(self): with tempfile.TemporaryDirectory() as d: - result = run_hook(bash_payload(f"gh issue create --title X --body 'UNVERIFIED: {CAUSE}'", d)) + result = run_hook(bash_payload(f"gh issue create --title X --body '{CAUSE} {TAGGED}'", d)) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stderr, "") diff --git a/engine/hooks/hedge-runs-prove-it/README.md b/engine/hooks/hedge-runs-prove-it/README.md index bf36d657..266b85f3 100644 --- a/engine/hooks/hedge-runs-prove-it/README.md +++ b/engine/hooks/hedge-runs-prove-it/README.md @@ -2,14 +2,14 @@ Stop hook: a hedge about code or repo state is a check the agent has not run. When the outgoing reply says "I think", "I believe", "probably", -"should work", "presumably", or carries an `UNVERIFIED:` prefix within 200 +"should work", "presumably", or carries a retired bare `UNVERIFIED:` within 200 characters of a code noun (a path, a backticked name, test, CI, build, bug, fix, script, hook, PR, merge, branch, commit, tree) and the turn ran no verification tool (Bash, Read, Grep, Glob), the turn is blocked (exit 2) with "run prove-it now: verify in this turn or state why it cannot be verified". -Passes when the turn ran a verification tool, when the `UNVERIFIED:` +Passes when the turn ran a verification tool, when a well-formed `{{CAT-UNVERIFIED}}` sentence carries a cannot-verify reason ("cannot verify: no network", "would need the live token"), when the hedge is quoted, or when the hedge is about something that is not code or state (a company's motive). @@ -42,7 +42,7 @@ instrument-level proof in the same message. Running a tool in the turn does not clear this one; a projection that omits a field is not proof the state is absent. Only same-message proof clears it: a fenced block of real output, a `file:line`, a pid, an exit code, a -`/proc/` path, or an explicit `UNVERIFIED:` prefix. +`/proc/` path, or a well-formed `{{CAT-UNVERIFIED}}` tag. Stays silent on a diagnosis inside a fence, a double-quoted span, a backtick span, a markdown blockquote, a hypothetical ("if it's a zombie, ..."), and @@ -50,7 +50,7 @@ past-tense narration of an old incident ("the task was a zombie"). Mechanical half of `corpus/skills/cat-mode/SKILL.md`'s Verify rule: "Unhedged root-cause or fix claims about live system behavior need -instrument-level proof in the same message, or `UNVERIFIED:`." Four +instrument-level proof in the same message, or `{{CAT-UNVERIFIED}}`." Four independent conditions must hold before it blocks, because a Stop hook's effective false-positive rate is what decides whether anyone keeps it on (Sadowski et al., "Lessons from Building Static Analysis Tools at Google," @@ -67,7 +67,7 @@ hook only blocks the offending reply and tells that same agent to go get evidence -- as correct inside a subagent as outside it. Mechanical half of the evidence rules in `engine/CLAUDE.core.md` ("never -claim ... without evidence in the SAME message"; `UNVERIFIED:` is the +claim ... without evidence in the SAME message"; a well-formed `{{CAT-UNVERIFIED}}` is the escape hatch, not a free pass). Fail-open on parse or read errors; `stop_hook_active` skips. diff --git a/engine/hooks/hedge-runs-prove-it/detect.py b/engine/hooks/hedge-runs-prove-it/detect.py index 69f35f70..d4ff2e66 100644 --- a/engine/hooks/hedge-runs-prove-it/detect.py +++ b/engine/hooks/hedge-runs-prove-it/detect.py @@ -3,10 +3,10 @@ Two shapes, two bars. A hedge -- "I think", "I believe", "probably", "should work", "presumably", -or an `UNVERIFIED:` prefix next to a code noun (a path, a backticked name, +or a `{{CAT-UNVERIFIED}}` tag next to a code noun (a path, a backticked name, test, CI, build, bug, fix, script, hook, PR, merge, branch, commit) -- means the agent has a check it has not run. The reply passes only when the -turn ran a verification tool (Bash, Read, Grep, Glob) or the `UNVERIFIED:` +turn ran a verification tool (Bash, Read, Grep, Glob) or the `{{CAT-UNVERIFIED}}` clause says why it cannot be verified ("cannot verify: no network"). An unhedged diagnosis -- "it's a zombie", "that's the bug", "the root cause @@ -16,7 +16,7 @@ that a check is outstanding. Running a tool in the turn does not clear it, because a projection that omits a field is not proof the state is absent. Only instrument-level proof in the same message clears it: pasted output, a -`file:line`, a pid, an exit code, or an explicit `UNVERIFIED:` prefix. +`file:line`, a pid, an exit code, or an explicit `{{CAT-UNVERIFIED}}` tag. Hedges about things that are not code or state (a company's motive, a filing date) are out of scope, and so is either shape quoted rather than @@ -28,7 +28,14 @@ from __future__ import annotations import json +import os import re +import sys + +sys.path.insert(0, os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_markers")) + +import markers # noqa: E402 HEDGE_RE = re.compile( r"\bI think\b|\bI believe\b|\bprobably\b|\bshould work\b|\bpresumably\b|" @@ -76,7 +83,7 @@ re.IGNORECASE, ) INSTRUMENT_EVIDENCE_RE = re.compile( - r"```|\bUNVERIFIED:|/proc/\d+|" + r"```|\{\{CAT-UNVERIFIED\b|/proc/\d+|" r"\b[\w./-]+\.[A-Za-z]{1,6}:\d+\b|" r"\bpids?\b\s*[:=#]?\s*\d+|\bMainPID\b|" r"\bexit\s+(?:code|status)\b|\bexit[_-]?code\b", @@ -98,7 +105,7 @@ MESSAGE = ( "hedge-runs-prove-it: this reply hedges about code or repo state ({hedge}) and " "the turn ran no verification (no Bash / Read / Grep). Run prove-it now: verify in " - "this turn, or write `UNVERIFIED: -- cannot verify: `." + "this turn, or write `{tag}`." ) DIAGNOSIS_MESSAGE = ( @@ -108,7 +115,7 @@ "signals an outstanding check -- and a projection that omits a field is not " "proof the state is absent. Attach the instrument output here: pasted ps / " "strace / /proc output, a live query's real result, a pid, an exit code, or a " - "file:line. Otherwise prefix the claim with `UNVERIFIED:`." + "file:line. Otherwise tag the claim: `{tag}`." ) @@ -157,7 +164,9 @@ def code_hedges(text: str) -> list[str]: window = text[max(0, match.start() - PROXIMITY): match.end() + PROXIMITY] if not CODE_NOUN_RE.search(window): continue - if match.group(0).upper().startswith("UNVERIFIED") and REASON_RE.search(_sentence_after(text, match.end())): + if match.group(0).upper().startswith("UNVERIFIED") and markers.well_formed_tags( + text[max(0, match.start() - len("{{CAT-")): match.end() + PROXIMITY] + ): continue hits.append(match.group(0)) return hits @@ -240,7 +249,7 @@ def _diagnosis_feedback(message: str) -> str | None: claims = diagnosis_claims(message) if not claims: return None - return DIAGNOSIS_MESSAGE.format(claim=", ".join(f'"{c}"' for c in claims[:3])) + return DIAGNOSIS_MESSAGE.format(tag=markers.TAG_TEMPLATE, claim=", ".join(f'"{c}"' for c in claims[:3])) def decide_from_lines(message: str, lines: list[dict]) -> str | None: @@ -252,7 +261,7 @@ def decide_from_lines(message: str, lines: list[dict]) -> str | None: return None if verified_this_turn(lines): return None - return MESSAGE.format(hedge=", ".join(f'"{h}"' for h in hedges[:3])) + return MESSAGE.format(tag=markers.TAG_TEMPLATE, hedge=", ".join(f'"{h}"' for h in hedges[:3])) def decide(payload: dict) -> str | None: diff --git a/engine/hooks/hedge-runs-prove-it/tests/fixtures/diagnosis_silent.json b/engine/hooks/hedge-runs-prove-it/tests/fixtures/diagnosis_silent.json index 57965200..3e2823f1 100644 --- a/engine/hooks/hedge-runs-prove-it/tests/fixtures/diagnosis_silent.json +++ b/engine/hooks/hedge-runs-prove-it/tests/fixtures/diagnosis_silent.json @@ -20,8 +20,8 @@ "verified": false }, { - "label": "diagnosis explicitly prefixed UNVERIFIED", - "reply": "UNVERIFIED: it's a zombie, not slow -- the pool shows zero slots but I have not looked at the process table.", + "label": "diagnosis carrying a well-formed CAT-UNVERIFIED tag", + "reply": "it's a zombie, not slow -- the pool shows zero slots. {{CAT-UNVERIFIED: it is a zombie -- cannot verify: no access to the process table here}}", "verified": true }, { diff --git a/engine/hooks/hedge-runs-prove-it/tests/fixtures/hedges_silent.json b/engine/hooks/hedge-runs-prove-it/tests/fixtures/hedges_silent.json index 30705d8e..ef3dae5b 100644 --- a/engine/hooks/hedge-runs-prove-it/tests/fixtures/hedges_silent.json +++ b/engine/hooks/hedge-runs-prove-it/tests/fixtures/hedges_silent.json @@ -1,7 +1,7 @@ [ { - "label": "UNVERIFIED with a cannot-verify reason clause", - "reply": "UNVERIFIED: the Linear write path -- cannot verify: no network in this sandbox, so the live ticket call was not run.", + "label": "CAT-UNVERIFIED tag with a cannot-verify reason clause", + "reply": "The Linear write path is unproven. {{CAT-UNVERIFIED: the Linear write path -- cannot verify: no network in this sandbox, so the live ticket call was not run}}", "verified": false }, { diff --git a/engine/hooks/history-claim-check/README.md b/engine/hooks/history-claim-check/README.md index 35277841..9e30d3c8 100644 --- a/engine/hooks/history-claim-check/README.md +++ b/engine/hooks/history-claim-check/README.md @@ -24,7 +24,7 @@ was 24 days, one pass found it, and a human wrote the file. ## What clears it Evidence within six lines of the claim — a fenced block, a named `git log` / -`git blame` / `git show`, a commit sha, or an explicit `UNVERIFIED:`. The point +`git blame` / `git show`, a commit sha, or a well-formed `{{CAT-UNVERIFIED}}` tag. The point is not to forbid the claim but to make the query cheaper than the guess. ## Scope diff --git a/engine/hooks/history-claim-check/detect.py b/engine/hooks/history-claim-check/detect.py index dc3a5f20..0a15e117 100644 --- a/engine/hooks/history-claim-check/detect.py +++ b/engine/hooks/history-claim-check/detect.py @@ -15,6 +15,13 @@ import re import shlex from pathlib import Path +import sys + +sys.path.insert(0, os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_markers")) + +import markers # noqa: E402 + PUBLISH_RE = re.compile( r"\bgh\s+pr\s+(?:create|edit)\b|" @@ -62,14 +69,15 @@ ] EVIDENCE = re.compile( - r"```|\bgit (?:log|blame|show|rev-list)\b|\b[0-9a-f]{7,40}\b|\bUNVERIFIED\b", re.I + r"```|\bgit (?:log|blame|show|rev-list)\b|\b[0-9a-f]{7,40}\b|" + r"\{\{CAT-UNVERIFIED\b[^}]*cannot\s+verify\s*:\s*\S", re.I ) WINDOW = 6 MESSAGE = ( "history-claim-check: this PR body states {n} claim(s) about repo history with no " "adjacent evidence. Each is one git command:\n{detail}\n" - "Run the command, paste its output beside the claim, or write UNVERIFIED: before it. " + "Run the command, paste its output beside the claim, or tag it `{tag}`. " "A wrong duration or count in a PR body is read as measured and outlives the session." ) @@ -368,4 +376,4 @@ def decide(command: str, cwd: str | None = None) -> str | None: problems = unsourced_claims(body_text(command, cwd)) if not problems: return None - return MESSAGE.format(n=len(problems), detail="\n".join(problems)) + return MESSAGE.format(tag=markers.TAG_TEMPLATE, n=len(problems), detail="\n".join(problems)) diff --git a/engine/hooks/history-claim-check/tests/test_hooks.py b/engine/hooks/history-claim-check/tests/test_hooks.py index f3c820a3..23f5aa19 100644 --- a/engine/hooks/history-claim-check/tests/test_hooks.py +++ b/engine/hooks/history-claim-check/tests/test_hooks.py @@ -71,8 +71,8 @@ def test_non_publication_command_is_untouched(self): self.assertFalse(is_publication("git commit -m 'stale for five months'")) self.assertIsNone(decide("git commit -m 'written by an agent'")) - def test_unverified_prefix_is_accepted(self): - self.assertEqual(unsourced_claims("UNVERIFIED: stale for five months"), []) + def test_well_formed_tag_is_accepted(self): + self.assertEqual(unsourced_claims("Stale for five months. {{CAT-UNVERIFIED: the five months -- cannot verify: the reflog is truncated}}"), []) def test_empty_and_malformed_do_not_raise(self): self.assertIsNone(decide("")) diff --git a/engine/hooks/incidence-needs-repetition/README.md b/engine/hooks/incidence-needs-repetition/README.md index f05017e6..5fa12a1f 100644 --- a/engine/hooks/incidence-needs-repetition/README.md +++ b/engine/hooks/incidence-needs-repetition/README.md @@ -4,7 +4,7 @@ Blocks a reply that claims behaviour **across runs** while showing evidence from ## Why this exists, and why the sibling hook could not do it -`hedge-runs-prove-it` fires on the absence of confidence — "probably", "should work", `UNVERIFIED:`. All six of its positive fixtures are hedges. +`hedge-runs-prove-it` fires on the absence of confidence — "probably", "should work", a retired bare `UNVERIFIED:`. All six of its positive fixtures are hedges. This failure mode is the opposite. On 2026-09-09 a session fixed a flaky test, ran it once green, and wrote: @@ -22,7 +22,7 @@ The user's verdict that day was `our /prove-it is not enough`, after asking the - a declared sample size of two or more: `12 iterations`, `8/12 runs`, `spread=…` - the same Bash command actually invoked twice or more in the turn -- an `UNVERIFIED:` prefix, which stops the claim being asserted +- a well-formed `{{CAT-UNVERIFIED}}` tag, which stops the claim being asserted ## What does not clear it diff --git a/engine/hooks/incidence-needs-repetition/detect.py b/engine/hooks/incidence-needs-repetition/detect.py index 398868bb..28721206 100644 --- a/engine/hooks/incidence-needs-repetition/detect.py +++ b/engine/hooks/incidence-needs-repetition/detect.py @@ -1,7 +1,7 @@ """incidence-needs-repetition: a claim about behaviour ACROSS runs needs more than one run. The sibling guard `hedge-runs-prove-it` catches the absence of confidence -- -"probably", "should work", `UNVERIFIED:`. This one catches the opposite and +"probably", "should work", the `{{CAT-UNVERIFIED}}` tag. This one catches the opposite and more dangerous shape: a confident claim whose subject is incidence. "Deterministic", "flaky", "every run", "consistently" are not claims about @@ -13,7 +13,7 @@ The bar here is a declared sample size of two or more: a pasted "12 iterations", an "8/12 runs" ratio, or the same command actually invoked -twice in the turn. An `UNVERIFIED:` prefix also clears it, because it stops +twice in the turn. A well-formed `{{CAT-UNVERIFIED}}` tag also clears it, because it stops the claim being asserted at all. Incidence words quoted rather than claimed are out of scope, as is any run @@ -24,6 +24,14 @@ import json import re +import os +import sys + +sys.path.insert(0, os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_markers")) + +import markers # noqa: E402 + INCIDENCE_RE = re.compile( r"\bnon-?deterministic\b|\bdeterministic(?:ally)?\b|\bflak(?:y|e|es|iness)\b|" @@ -38,7 +46,7 @@ r"\b\d+\s*/\s*(?:[2-9]|\d{2,})\s*(?:runs?|iterations?|samples?|trials?)\b|" r"\b\d+\s+of\s+(?:[2-9]|\d{2,})\s+(?:runs?|iterations?|samples?|trials?)\b|" r"\bruns_under_[\w,]+=\d+/\d+|\bspread\s*=\s*[\d,]+|" - r"\bUNVERIFIED:", + r"\{\{CAT-UNVERIFIED\b[^}]*cannot\s+verify\s*:\s*\S", re.IGNORECASE, ) @@ -53,7 +61,7 @@ "but shows evidence from a single run. A green run, and a file:line, both prove " "what happened once -- neither is a distribution. Re-run the measurement at least " "twice and paste the spread (e.g. \"12 iterations ... spread=...\"), or prefix the " - "claim with `UNVERIFIED:`." + "claim with `{tag}`." ) @@ -143,7 +151,7 @@ def decide_from_lines(message: str, lines: list[dict]) -> str | None: return None if repeated_command_this_turn(lines): return None - return MESSAGE.format(term=", ".join(f'"{c}"' for c in claims[:3])) + return MESSAGE.format(tag=markers.TAG_TEMPLATE, term=", ".join(f'"{c}"' for c in claims[:3])) def decide(payload: dict) -> str | None: diff --git a/engine/hooks/incidence-needs-repetition/tests/fixtures/incidence_silent.json b/engine/hooks/incidence-needs-repetition/tests/fixtures/incidence_silent.json index f5d6d944..c7fff1b0 100644 --- a/engine/hooks/incidence-needs-repetition/tests/fixtures/incidence_silent.json +++ b/engine/hooks/incidence-needs-repetition/tests/fixtures/incidence_silent.json @@ -6,7 +6,7 @@ }, { "label": "incidence word hedged rather than asserted", - "reply": "UNVERIFIED: the new instrument looks deterministic; I have only run it once.", + "reply": "The new instrument looks deterministic; I have only run it once. {{CAT-UNVERIFIED: it is deterministic -- cannot verify: one run is not a distribution and the box is busy}}", "verified": true }, { diff --git a/engine/hooks/named-verb-guard/README.md b/engine/hooks/named-verb-guard/README.md index d5142477..1b80280c 100644 --- a/engine/hooks/named-verb-guard/README.md +++ b/engine/hooks/named-verb-guard/README.md @@ -14,7 +14,7 @@ is blocked (exit 2) with guidance. | stop (message of 8 words or fewer) | no Bash / Edit / Write tool calls after the message | | a repeated proof demand | a fenced block, `path:line`, or URL | -`UNVERIFIED:` anywhere in the reply, or a reply that ends in a question, +a well-formed `{{CAT-UNVERIFIED}}` tag anywhere in the reply, or a reply that ends in a question, always passes: the guard wants proof or an honest "not proven", never a prettier assurance. diff --git a/engine/hooks/named-verb-guard/detect.py b/engine/hooks/named-verb-guard/detect.py index 709b8fb3..bfeed8b2 100755 --- a/engine/hooks/named-verb-guard/detect.py +++ b/engine/hooks/named-verb-guard/detect.py @@ -8,10 +8,10 @@ turn's tool calls) must carry. 2. Proof polling. The second or later "prove it" / "show me" / "are you sure" / "did you actually run it" in one session. The reply must then - carry a command-and-output block, a file:line, a URL, or `UNVERIFIED:`. + carry a command-and-output block, a file:line, a URL, or a CAT-UNVERIFIED tag. Evidence is shape only: a closed fenced block, a `path:line` reference, a -URL, a markdown table row, or the literal `UNVERIFIED:` prefix. The hook +URL, a markdown table row, or a well-formed CAT-UNVERIFIED tag. The hook cannot judge whether the evidence is real; it only refuses a bare assurance where the user asked for proof. @@ -23,6 +23,13 @@ import json import os import re +import sys + +sys.path.insert(0, os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_markers")) + +import markers # noqa: E402 + OUTPUT_VERBS = ("repro", "reproduce", "test", "rerun", "re-run", "run", "regenerate", "prove") LINK_OK_VERBS = ("run", "regenerate", "show") @@ -156,12 +163,12 @@ def bash_commands(tool_uses: list[dict]) -> list[str]: def missing_evidence(verbs: list[str], polled: bool, message: str, tool_uses: list[dict]) -> str | None: """Return what is missing, or None when the reply carries the evidence.""" - if UNVERIFIED_RE.search(message) or message.rstrip().endswith("?"): + if markers.well_formed_tags(message) or message.rstrip().endswith("?"): return None output_ok = has_output_evidence(message) link_ok = output_ok or has_link_evidence(message) if polled and not link_ok: - return "the user has asked for proof more than once this session; paste the command and its real output (fenced), a file:line, or a URL, or prefix the claim with `UNVERIFIED:`" + return "the user has asked for proof more than once this session; paste the command and its real output (fenced), a file:line, or a URL, or tag the claim with a CAT-UNVERIFIED that names the blocker" for verb in verbs: if verb == "stop": mutating = [b.get("name") for b in tool_uses if b.get("name") in MUTATING_TOOLS] diff --git a/engine/hooks/named-verb-guard/tests/test_hooks.py b/engine/hooks/named-verb-guard/tests/test_hooks.py index 462902d9..a0f5ba7f 100644 --- a/engine/hooks/named-verb-guard/tests/test_hooks.py +++ b/engine/hooks/named-verb-guard/tests/test_hooks.py @@ -23,6 +23,8 @@ sys.path.insert(0, HOOKS_DIR) import claude_stop_check # noqa: E402 +TAGGED = "{{CAT-UNVERIFIED: the suite result -- cannot verify: it did not finish inside the sandbox timeout}}" + import detect # noqa: E402 @@ -198,10 +200,10 @@ def test_hook_feedback_line_is_not_the_users_message_silent(self): finally: os.unlink(tmp.name) - def test_unverified_prefix_allows(self): + def test_well_formed_tag_allows(self): path = transcript_with(["run the regression suite"]) try: - blocked, _ = run_hook(path, "UNVERIFIED: the suite did not finish inside the sandbox timeout; rerun with a longer timeout.") + blocked, _ = run_hook(path, "The suite did not finish. " + TAGGED) self.assertFalse(blocked) finally: os.unlink(path) diff --git a/engine/hooks/prove-it-ship-gate/README.md b/engine/hooks/prove-it-ship-gate/README.md index 008e5d2b..f0067907 100644 --- a/engine/hooks/prove-it-ship-gate/README.md +++ b/engine/hooks/prove-it-ship-gate/README.md @@ -6,7 +6,7 @@ Slack, external API), the same message must carry evidence a reviewer can chase (URL, sha, ticket or PR id, fenced output, exit code, PID, timestamp), or a live command must have run this turn (`ssh`, `curl`, `gh api`, `gh pr view`, `systemctl`, ...), or the claim must carry the literal prefix -`UNVERIFIED: live path`. Otherwise the turn is blocked (exit 2). +`{{CAT-UNVERIFIED: -- cannot verify: }}`. Otherwise the turn is blocked (exit 2). Fixture tests and UI registration do not count. That is the whole point: Invoker PRs #10553-#10558 shipped cross-repo-research after unit + fixture + diff --git a/engine/hooks/prove-it-ship-gate/detect.py b/engine/hooks/prove-it-ship-gate/detect.py index 8e7fc314..42a674d3 100644 --- a/engine/hooks/prove-it-ship-gate/detect.py +++ b/engine/hooks/prove-it-ship-gate/detect.py @@ -4,7 +4,7 @@ outgoing message says done / shipped / deployed / live about work with live side effects (Linear, deploy, production host, webhook, Slack, external API), the same message must carry live evidence or the literal prefix -`UNVERIFIED: live path`. Unit tests, fixtures, and UI registration do not +a well-formed CAT-UNVERIFIED tag. Unit tests, fixtures, and UI registration do not count. This lived only in prose; this file is the mechanical half. Judgment (is this work really live-side-effect work) stays with the model: the hook only matches shapes and fails open on parse errors. @@ -16,6 +16,14 @@ import json import re +import os +import sys + +sys.path.insert(0, os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_markers")) + +import markers # noqa: E402 + # A claim is a status assertion about the work, not any mention of the word. # Sentence-initial status words ("Deployed. DO1 is now...") and copula forms @@ -162,7 +170,7 @@ def decide(payload: dict) -> str | None: message = payload.get("last_assistant_message") or "" if not claims_live_ship(message): return None - if UNVERIFIED_RE.search(message) or has_evidence(message): + if markers.well_formed_tags(message) or has_evidence(message): return None transcript_path = ( payload.get("agent_transcript_path") @@ -184,5 +192,6 @@ def decide(payload: dict) -> str | None: "registration, a dry run, and the PR number of this change do not prove the " "live path ran; only an id the pipeline itself emitted does (a workflow id, " "an Actions run URL, a release tag, a live-owner dispatch). Paste that " - "evidence in this message, or prefix the claim with `UNVERIFIED: live path`." + "evidence in this message, or tag the claim: " + f"`{markers.TAG_TEMPLATE}`." ) diff --git a/engine/hooks/prove-it-ship-gate/tests/test_hooks.py b/engine/hooks/prove-it-ship-gate/tests/test_hooks.py index 0d290233..0666bea2 100644 --- a/engine/hooks/prove-it-ship-gate/tests/test_hooks.py +++ b/engine/hooks/prove-it-ship-gate/tests/test_hooks.py @@ -89,8 +89,9 @@ def transcript_with_commands(commands): "the fully-fixed code (`8262dc40a6`).\n3. Linear ticket **INV-284** filed for the " "deeper architecture cleanup.", # explicit escape hatch - "UNVERIFIED: live path — the Linear-sync worker is shipped; unit tests pass " - "and it registers in the settings panel, but I have not seen a real ticket write.", + "The Linear-sync worker is shipped; unit tests pass " + "and it registers in the settings panel, but I have not seen a real ticket write. " + "{{CAT-UNVERIFIED: the live ticket write -- cannot verify: no Linear credentials in this sandbox}}", # backtest false positives (2026-09-01): progress talk and adjectives "Yes — DO1 (the production server) is actively working on it right now, checked " "a couple minutes ago. Most PRs are stuck — blocked mainly by a failing check.", @@ -116,7 +117,7 @@ def test_fires_on_each_real_unproven_ship_claim(self): def test_hook_blocks_with_exit_2_and_guidance(self): code, err = run_claude({"last_assistant_message": REAL_FIRE[0]}) self.assertEqual(code, 2) - self.assertIn("UNVERIFIED: live path", err) + self.assertIn("{{CAT-UNVERIFIED", err) def test_fires_when_only_fixture_tests_ran_this_turn(self): path = transcript_with_commands(["python3 -m pytest packages/app -q", "npm test"]) diff --git a/install.sh b/install.sh index 12c9ff17..c5a61523 100755 --- a/install.sh +++ b/install.sh @@ -216,6 +216,7 @@ fi # bakes in a machine-specific absolute path or username. echo "--- claude hooks (\$HOME/.claude/hooks) ---" mkdir -p "$HOME/.claude/hooks" +link_item "_markers" "$REPO_DIR/engine/hooks/_markers" "$HOME/.claude/hooks/_markers" link_item "diu-stop" "$REPO_DIR/engine/hooks/diu-stop" "$HOME/.claude/hooks/diu-stop" link_item "bug-complaint-leak" "$REPO_DIR/engine/hooks/bug-complaint-leak" "$HOME/.claude/hooks/bug-complaint-leak" link_item "demo-freeze" "$REPO_DIR/engine/hooks/demo-freeze" "$HOME/.claude/hooks/demo-freeze" diff --git a/tests/scenarios/prove-it-claims.json b/tests/scenarios/prove-it-claims.json index 98bfaea5..0157f987 100644 --- a/tests/scenarios/prove-it-claims.json +++ b/tests/scenarios/prove-it-claims.json @@ -3,16 +3,16 @@ "name": "ship-claim-no-live-evidence", "situation": "Agent says a worker is deployed and running, nothing run this turn.", "user": "ship the notification worker", - "reply": "Done \u2014 the notification worker is deployed and running in production. It files the Linear ticket on every failure now.", + "reply": "Done — the notification worker is deployed and running in production. It files the Linear ticket on every failure now.", "expect_fire": [ "prove-it-ship-gate" ] }, { - "name": "ship-claim-with-unverified-prefix", - "situation": "Same claim, but honestly labelled as unproven.", + "name": "ship-claim-tagged-with-a-named-blocker", + "situation": "Same claim, labelled unproven with a tag that names the blocker.", "user": "ship the notification worker", - "reply": "UNVERIFIED: live path \u2014 the notification worker is deployed and should file the Linear ticket on failure. I have not exercised the live path.", + "reply": "The notification worker is deployed and should file the Linear ticket on failure. I have not exercised the live path. {{CAT-UNVERIFIED: the live path -- cannot verify: no Linear credentials in this sandbox}}", "expect_silent": [ "prove-it-ship-gate" ] @@ -21,7 +21,7 @@ "name": "hedge-about-code-nothing-run", "situation": "Agent hedges about repo state without running anything.", "user": "does the retry limit still apply after the refactor?", - "reply": "I think the retry limit is probably still honoured after the refactor \u2014 `queue/worker.py` should work the same way and the tests cover it.", + "reply": "I think the retry limit is probably still honoured after the refactor — `queue/worker.py` should work the same way and the tests cover it.", "expect_fire": [ "hedge-runs-prove-it" ] @@ -42,9 +42,27 @@ "name": "hedge-with-no-code-noun-is-a-known-gap", "situation": "KNOWN LIMITATION, pinned deliberately: hedge-runs-prove-it needs a hedge within 200 chars of a code noun (detect.py CODE_NOUN_RE). A hedge about 'the retry limit' / 'the helper' / 'the refactor' names no listed noun, so it is missed. Widening that list risks false positives on non-code hedges, so this records the boundary rather than asserting behavior the hook does not have.", "user": "does the retry limit still apply after the refactor?", - "reply": "I think the retry limit is probably still honoured after the refactor \u2014 the helper should work the same way.", + "reply": "I think the retry limit is probably still honoured after the refactor — the helper should work the same way.", "expect_silent": [ "hedge-runs-prove-it" ] + }, + { + "name": "ship-claim-tagged-without-a-blocker", + "situation": "Same claim, tagged but naming no blocker -- the old free pass in new syntax.", + "user": "ship the notification worker", + "reply": "Done — the notification worker is deployed and running in production. {{CAT-UNVERIFIED: the live path}}", + "expect_fire": [ + "prove-it-ship-gate" + ] + }, + { + "name": "ship-claim-with-retired-bare-marker", + "situation": "The retired bare marker no longer excuses a ship claim.", + "user": "ship the notification worker", + "reply": "UNVERIFIED: live path — the notification worker is deployed and running in production.", + "expect_fire": [ + "prove-it-ship-gate" + ] } ]