From 24387099733f00a9e4a040440acf3edde28bb5c1 Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Thu, 10 Sep 2026 11:50:12 -0700 Subject: [PATCH] hook(diu-stop): scope the unverified marker to its own paragraph One `UNVERIFIED:` anywhere in a reply silenced every other check in that reply, so a hedge in the opening line let an unhedged causal claim three paragraphs later through untouched. The marker now excuses the paragraph it sits in, exactly like a backtick or a fence. The prove-it nudge also stops firing on a marker that names its blocker ("cannot verify: no network"), which is the documented use, and keeps firing on one that names nothing. The block text now says the prefix is for a check that cannot run and has to name why. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KU2pPKob4MJ1NqjsfTNyYJ Change-Id: I11fd3df007c88b3751292e581fd337b98b9ebaf1 --- engine/hooks/diu-stop/claude_stop_check.py | 50 ++++++++++++++++------ engine/hooks/diu-stop/tests/test_hooks.py | 46 ++++++++++++++++++++ 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/engine/hooks/diu-stop/claude_stop_check.py b/engine/hooks/diu-stop/claude_stop_check.py index 4c3b0148..f252668e 100755 --- a/engine/hooks/diu-stop/claude_stop_check.py +++ b/engine/hooks/diu-stop/claude_stop_check.py @@ -47,8 +47,6 @@ BANNED_OPENERS = ["confirmed", "verified"] # Evidence-shaped content next to the claim: a fenced/inline code block. -# UNVERIFIED: is a whole-message escape hatch (checked separately). Presence -# doesn't prove the evidence is real -- only that something was shown. EVIDENCE_MARKER_RE = re.compile(r"```|`[^`]+`|\bUNVERIFIED:", re.IGNORECASE) FENCE_MARKER = "```" FENCED_BODY_RE = re.compile(r"```[^\n]*\n(.*?)```", re.DOTALL) @@ -117,24 +115,48 @@ def _opening_word(message): return match.group(0).lower() if match else "" +REASON_RE = re.compile( + r"\b(?:cannot|can'?t|could not|couldn'?t|unable to|no way to|not possible to|" + r"would need|requires|needs|no access|no network|offline|not reachable|sandbox|" + r"blocked|denied|locked|down|unavailable)\b", + re.IGNORECASE, +) + + +def _sentence_at(message, start): + end = len(message) + for stop in (". ", ".\n", "\n\n"): + idx = message.find(stop, start) + if idx != -1: + end = min(end, idx) + return message[start:end] + + def has_unresolved_unverified_marker(message): - return bool(UNVERIFIED_RE.search(message)) + """True when a marker carries no reason the check could not run. + + A marker that names its blocker ("cannot verify: no network") is the + documented use. One that names nothing is a claim waiting for a check + the turn could have run, so it still gets the prove-it nudge.""" + for match in UNVERIFIED_RE.finditer(message or ""): + if not REASON_RE.search(_sentence_at(message, match.end())): + return True + return False def find_unverified_claim(message): """Return the offending phrase if a paragraph makes an unverified-shaped claim with no evidence marker in that same paragraph. - `UNVERIFIED:` anywhere still silences the whole message. A fence only - silences the paragraph it sits in -- not a later/earlier claim. Inline - code silences its paragraph only when it looks like command output or - the message carries a fenced block of output. This is a blunt proxy, - not a truth check.""" - if UNVERIFIED_RE.search(message): - return None + `UNVERIFIED:` excuses the paragraph it sits in, exactly like a fence -- + not a later or earlier claim. A marker in the opening line used to + silence every other paragraph in the message, which turned one hedge + into a mute for the whole reply. Inline code silences its paragraph only + when it looks like command output or the message carries a fenced block + of output. This is a blunt proxy, not a truth check.""" fenced_output = any(OUTPUT_SHAPE_RE.search(body) for body in FENCED_BODY_RE.findall(message)) for para in re.split(r"\n\s*\n", message): - if FENCE_MARKER in para: + if FENCE_MARKER in para or UNVERIFIED_RE.search(para): continue inline = INLINE_CODE_RE.findall(para) if inline and (fenced_output or any(OUTPUT_SHAPE_RE.search(code) for code in inline)): @@ -185,8 +207,10 @@ def main(): f"This message makes an unverified-shaped claim (\"{claim}\") with no " "adjacent evidence (pasted command output, or an `UNVERIFIED:` " "prefix). A backticked name or command alone is not output. Per " - "skills/prove-it/SKILL.md: either paste the output of what was " - "actually run/checked, or prefix the claim with `UNVERIFIED:`." + "skills/prove-it/SKILL.md: paste the output of what was actually " + "run or checked. The prefix is for a check that cannot run, and it " + "has to name the blocker: `UNVERIFIED: -- cannot verify: " + "`." ) if unverified_marker: parts.append( diff --git a/engine/hooks/diu-stop/tests/test_hooks.py b/engine/hooks/diu-stop/tests/test_hooks.py index bf208520..d76b83f8 100644 --- a/engine/hooks/diu-stop/tests/test_hooks.py +++ b/engine/hooks/diu-stop/tests/test_hooks.py @@ -177,6 +177,52 @@ def test_malformed_stdin_json_does_not_crash(self): self.assertEqual(buf.getvalue(), "") +class TestMarkerScopeAndBlocker(unittest.TestCase): + """The marker excuses its own paragraph, not the whole reply. + + A reply once opened with a hedge carrying the marker and then made an + unhedged causal claim three paragraphs later. The whole-message mute + let that claim through untouched, which is how one hedge silenced an + entire message. + """ + + MARKED_THEN_UNMARKED = ( + "UNVERIFIED: the planner keeps its own session memory -- cannot verify: " + "the log for that process is gone.\n\n" + "Here is what the deploy did instead.\n\n" + "The queue drained because the worker never claimed the lease." + ) + + def test_hit_later_paragraph_is_still_checked(self): + self.assertIsNotNone( + claude_stop_check.find_unverified_claim(self.MARKED_THEN_UNMARKED) + ) + + def test_no_hit_inside_the_marked_paragraph_itself(self): + marked_only = ( + "UNVERIFIED: the queue drained because the worker never claimed the " + "lease -- cannot verify: no access to that host." + ) + self.assertIsNone(claude_stop_check.find_unverified_claim(marked_only)) + + def test_no_hit_prove_it_nag_when_the_blocker_is_named(self): + self.assertFalse(claude_stop_check.has_unresolved_unverified_marker( + "UNVERIFIED: the live path -- cannot verify: no network in this sandbox." + )) + + def test_hit_prove_it_nag_when_no_blocker_is_named(self): + self.assertTrue(claude_stop_check.has_unresolved_unverified_marker( + "UNVERIFIED: that comes from the planner's own session memory." + )) + + def test_hit_blocks_the_real_reply_that_prompted_this(self): + blocked, err = run_claude_check({ + "last_assistant_message": self.MARKED_THEN_UNMARKED, + }) + self.assertTrue(blocked) + self.assertIn("cannot verify", err) + + class TestUnverifiedClaimCheck(unittest.TestCase): """Regression tests for the three real unverified claims a session let through before self-correcting or being corrected by the user (see