Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions engine/hooks/_markers/markers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""One definition of the escape-hatch marker, shared by every evidence hook.

`{{CAT-UNVERIFIED: <claim> -- cannot verify: <reason>}}` is the only marker
that excuses a claim. It excuses the paragraph it sits in, and only when it
names a blocker after `cannot verify:`. A tag that names nothing does not
excuse anything -- that shape is the one an author reaches for to end a turn
without checking, which is what the marker exists to prevent.

Bare `UNVERIFIED:` is ordinary prose. It used to be the escape hatch, so a
message still carrying it gets `legacy_marker` set and is told which tag to
use instead, rather than silently losing the suppression it expected.

Why a module and not a regex copied into each hook: six hooks read this
marker. Two classifiers over the same input drift apart silently -- the
reason clause landed in hedge-runs-prove-it and never reached diu-stop.

Hooks are installed as sibling symlinks under $HOME/.claude/hooks/, so a
hook reaches this module by its own parent directory:

sys.path.insert(0, os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_markers"))

Two dirnames, not a "..": the hook's own directory is a symlink, so the OS
resolves it before applying "..", landing beside the checkout instead of
beside the other hooks. Stripping two segments textually cannot do that, and
abspath (never realpath) is what keeps the installed path in place.
"""
from __future__ import annotations

import re

TAG_RE = re.compile(r"\{\{\s*CAT-UNVERIFIED\b(?P<body>[^}]*)\}\}", re.IGNORECASE)
REASON_RE = re.compile(r"cannot\s+verify\s*:\s*(?P<reason>\S.*)", re.IGNORECASE | re.DOTALL)
LEGACY_RE = re.compile(r"(?<!CAT-)\bUNVERIFIED:", re.IGNORECASE)

TAG_TEMPLATE = "{{CAT-UNVERIFIED: <claim> -- cannot verify: <reason>}}"

MALFORMED_TAG_MESSAGE = (
"A `{{CAT-UNVERIFIED}}` tag here names no blocker. Per "
"skills/prove-it/SKILL.md the tag is for a check that cannot run, and it "
f"has to say why: `{TAG_TEMPLATE}`. If the check can run, run it and paste "
"the output instead."
)

LEGACY_MARKER_MESSAGE = (
"`UNVERIFIED:` is no longer an escape hatch -- it reads as ordinary prose "
"and the claim beside it is judged on its own. Per skills/prove-it/SKILL.md: "
"run the check and paste its output. Only if the check genuinely cannot run, "
f"write `{TAG_TEMPLATE}`."
)


def _names_a_blocker(body: str) -> bool:
match = REASON_RE.search(body)
if not match:
return False
return bool(match.group("reason").strip(" -_.:\t\r\n"))


def well_formed_tags(text: str) -> list[str]:
"""Every `{{CAT-UNVERIFIED: ...}}` in `text` that names a blocker."""
return [m.group(0) for m in TAG_RE.finditer(text) if _names_a_blocker(m.group("body"))]


def malformed_tags(text: str) -> list[str]:
"""Every `{{CAT-UNVERIFIED: ...}}` in `text` that names no blocker."""
return [m.group(0) for m in TAG_RE.finditer(text) if not _names_a_blocker(m.group("body"))]


def excuses_paragraph(paragraph: str) -> bool:
"""True when this paragraph carries a tag that names a blocker."""
return bool(well_formed_tags(paragraph))


def has_legacy_marker(text: str) -> bool:
"""True when `text` still uses the retired bare `UNVERIFIED:` marker."""
return bool(LEGACY_RE.search(text))
86 changes: 86 additions & 0 deletions engine/hooks/_markers/tests/test_installed_layout.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
#!/usr/bin/env python3
"""The sibling-path import has to resolve where hooks actually run.

install.sh links each hook directory separately into $HOME/.claude/hooks/, so
a hook reaches this module through its own parent directory. That only works
because the path is built with abspath, which leaves the symlink in place;
realpath would jump into the checkout and find nothing beside it in a partial
install. This test builds that layout for real rather than trusting the shape.

Run: python3 -m unittest discover -s engine/hooks/_markers/tests -v
"""
import os
import shutil
import subprocess
import sys
import tempfile
import unittest

MARKERS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
HOOKS_DIR = os.path.dirname(MARKERS_DIR)

IMPORT_LINE = (
"import os, sys\n"
"sys.path.insert(0, os.path.join("
"os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '_markers'))\n"
"import markers\n"
"print(len(markers.well_formed_tags("
"'{{CAT-UNVERIFIED: x -- cannot verify: offline}}')))\n"
)


class InstalledLayout(unittest.TestCase):
def setUp(self):
self.home = tempfile.mkdtemp(prefix="markers-install-")
self.addCleanup(shutil.rmtree, self.home, ignore_errors=True)
self.installed_hooks = os.path.join(self.home, ".claude", "hooks")
os.makedirs(self.installed_hooks)

def _link(self, name, src):
target = os.path.join(self.installed_hooks, name)
os.symlink(src, target)
return target

def _run_consumer(self, consumer_dir):
script = os.path.join(consumer_dir, "detect.py")
with open(script, "w", encoding="utf-8") as handle:
handle.write(IMPORT_LINE)
return subprocess.run(
[sys.executable, os.path.join(self.installed_hooks, "consumer", "detect.py")],
capture_output=True,
text=True,
)

def test_consumer_imports_markers_through_symlinked_siblings(self):
self._link("_markers", MARKERS_DIR)
real_consumer = tempfile.mkdtemp(prefix="markers-consumer-")
self.addCleanup(shutil.rmtree, real_consumer, ignore_errors=True)
self._link("consumer", real_consumer)

result = self._run_consumer(real_consumer)

self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stdout.strip(), "1")

def test_missing_markers_link_fails_loudly_instead_of_passing_clean(self):
real_consumer = tempfile.mkdtemp(prefix="markers-consumer-")
self.addCleanup(shutil.rmtree, real_consumer, ignore_errors=True)
self._link("consumer", real_consumer)

result = self._run_consumer(real_consumer)

self.assertNotEqual(result.returncode, 0)
self.assertIn("ModuleNotFoundError", result.stderr)


class InstallerWiring(unittest.TestCase):
def test_install_sh_links_the_markers_module(self):
install_sh = os.path.join(os.path.dirname(os.path.dirname(HOOKS_DIR)), "install.sh")
with open(install_sh, encoding="utf-8") as handle:
body = handle.read()
self.assertIn('link_item "_markers"', body)
self.assertIn('"$HOME/.claude/hooks/_markers"', body)


if __name__ == "__main__":
unittest.main()
103 changes: 103 additions & 0 deletions engine/hooks/_markers/tests/test_markers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""Unit tests for the shared escape-hatch marker.

Run: python3 -m unittest discover -s engine/hooks/_markers/tests -v
"""
import os
import sys
import unittest

MARKERS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, MARKERS_DIR)

import markers # noqa: E402

WELL_FORMED = "{{CAT-UNVERIFIED: the glob covers it -- cannot verify: no CI access from here}}"
NO_BLOCKER = "{{CAT-UNVERIFIED: the glob covers it}}"
EMPTY_BLOCKER = "{{CAT-UNVERIFIED: the glob covers it -- cannot verify: }}"


class TagRecognition(unittest.TestCase):
def test_tag_naming_a_blocker_is_well_formed(self):
self.assertEqual(markers.well_formed_tags(WELL_FORMED), [WELL_FORMED])
self.assertEqual(markers.malformed_tags(WELL_FORMED), [])

def test_tag_naming_no_blocker_is_malformed(self):
self.assertEqual(markers.well_formed_tags(NO_BLOCKER), [])
self.assertEqual(markers.malformed_tags(NO_BLOCKER), [NO_BLOCKER])

def test_blocker_with_only_whitespace_does_not_count(self):
self.assertEqual(markers.well_formed_tags(EMPTY_BLOCKER), [])
self.assertEqual(markers.malformed_tags(EMPTY_BLOCKER), [EMPTY_BLOCKER])

def test_tag_is_case_insensitive_and_tolerates_inner_spaces(self):
text = "{{ cat-unverified: x -- Cannot Verify: the box is offline }}"
self.assertEqual(len(markers.well_formed_tags(text)), 1)

def test_several_tags_are_each_classified(self):
text = f"{WELL_FORMED}\n\n{NO_BLOCKER}"
self.assertEqual(len(markers.well_formed_tags(text)), 1)
self.assertEqual(len(markers.malformed_tags(text)), 1)


class ParagraphExcuse(unittest.TestCase):
def test_well_formed_tag_excuses_its_paragraph(self):
self.assertTrue(markers.excuses_paragraph(f"The job passes. {WELL_FORMED}"))

def test_malformed_tag_does_not_excuse_its_paragraph(self):
self.assertFalse(markers.excuses_paragraph(f"The job passes. {NO_BLOCKER}"))

def test_clean_paragraph_is_not_excused(self):
self.assertFalse(markers.excuses_paragraph("The job passes."))


class LegacyMarker(unittest.TestCase):
def test_bare_marker_is_reported_as_legacy(self):
self.assertTrue(markers.has_legacy_marker("UNVERIFIED: the glob covers it"))

def test_bare_marker_never_excuses_a_paragraph(self):
self.assertFalse(markers.excuses_paragraph("UNVERIFIED: the glob covers it"))

def test_new_tag_is_not_reported_as_legacy(self):
self.assertFalse(markers.has_legacy_marker(WELL_FORMED))

def test_clean_text_has_no_legacy_marker(self):
self.assertFalse(markers.has_legacy_marker("The job passes."))


class Messages(unittest.TestCase):
def test_both_messages_show_the_literal_tag_template(self):
self.assertIn("{{CAT-UNVERIFIED: <claim> -- cannot verify: <reason>}}", markers.MALFORMED_TAG_MESSAGE)
self.assertIn("{{CAT-UNVERIFIED: <claim> -- cannot verify: <reason>}}", markers.LEGACY_MARKER_MESSAGE)

class RenderedMessagesKeepTheirBraces(unittest.TestCase):
"""Every hook that names the tag in a block message renders it with both
braces intact.

These messages are `.format()` templates. A literal `{{CAT-UNVERIFIED}}`
written into one collapses to `{CAT-UNVERIFIED}` when it is rendered, and
the hook then tells the author to type a tag that no hook recognises.
That shipped once here and every existing test still passed, because they
all asserted on the substring `CAT-UNVERIFIED`, which survives the
collapse. These assert on the braces."""

def test_external_claim_gate_renders_both_braces(self):
sys.path.insert(0, os.path.join(os.path.dirname(MARKERS_DIR), "external-claim-gate"))
import detect as external_detect # noqa: PLC0415
message = external_detect.block_message([
external_detect.Finding(outcome="hit", destination="gh issue create",
claim="because", detail="x"),
])
self.assertIn(markers.TAG_TEMPLATE, message)

def test_tag_template_itself_has_double_braces(self):
self.assertTrue(markers.TAG_TEMPLATE.startswith("{{"))
self.assertTrue(markers.TAG_TEMPLATE.endswith("}}"))

def test_a_collapsed_tag_is_not_recognised(self):
collapsed = markers.TAG_TEMPLATE.replace("{{", "{").replace("}}", "}")
self.assertEqual(markers.well_formed_tags(collapsed), [])


if __name__ == "__main__":
unittest.main()
2 changes: 1 addition & 1 deletion engine/hooks/diu-stop/COVERAGE.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ What does not:
- Claims the patterns do not match. The check is a text proxy. It looks for
claim-shaped phrases (the banned phrases, a "confirmed" or "verified"
opener, a causal closer, a hedged cause) and for evidence-shaped text in
the same paragraph (a fence, output-shaped inline code, `UNVERIFIED:`). It
the same paragraph (a fence, output-shaped inline code, a well-formed `{{CAT-UNVERIFIED}}`). It
cannot tell whether the evidence is real. Of the 141 claims on this
surface, the current patterns match 45 and miss 96.

Expand Down
67 changes: 43 additions & 24 deletions engine/hooks/diu-stop/claude_stop_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,32 @@
log volume, not a crash loop), and "the fix... never pushed" (it had
pushed; a downstream fetch just hadn't caught up yet). All three had the
same shape: a bare declarative claim opening the message, no adjacent
evidence and no `UNVERIFIED:` prefix. This can't verify the evidence is
evidence and no escape-hatch marker. This can't verify the evidence is
real -- only that *something evidence-shaped* (a fenced block, inline code
that looks like output, or `UNVERIFIED:` itself) sits near the claim. See
that looks like output, or the marker itself) sits near the claim. See
skills/prove-it/SKILL.md in the Invoker repo for the full discipline this
mechanically nudges toward.

The marker is `{{CAT-UNVERIFIED: <claim> -- cannot verify: <reason>}}`, and
it excuses the paragraph it sits in, the way a fence does. Bare
`UNVERIFIED:` used to excuse the whole message and no longer excuses
anything: it was reachable by typing four characters, which made it the
cheapest way to end a turn, and it was used that way. A tag that names no
blocker is the same move wearing the new syntax, so it does not excuse
either.
"""
import json
import os
import re
import sys

from diu_limit import WORD_LIMIT, counted_words

sys.path.insert(0, os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_markers"))

import markers # noqa: E402

# Phrases banned outright (from this user's global CLAUDE.md evidence
# rules) -- rarely legitimate even mid-sentence, so no opener restriction.
BANNED_PHRASES_UNCONDITIONAL = [
Expand Down Expand Up @@ -99,29 +113,39 @@
def _opening_word(message):
stripped = message.lstrip()
stripped = re.sub(r"^[*_\-#\s]+", "", stripped)
stripped = markers.LEGACY_RE.sub("", stripped, count=1).lstrip()
match = re.match(r"[A-Za-z']+", stripped)
return match.group(0).lower() if match else ""


def has_unresolved_unverified_marker(message):
return bool(UNVERIFIED_RE.search(message))
def find_marker_problems(message):
"""Return the marker complaints this message earns, in report order.

A tag that names no blocker, and the retired bare `UNVERIFIED:`, each
draw their own message. Both can be present at once."""
problems = []
if markers.malformed_tags(message):
problems.append(markers.MALFORMED_TAG_MESSAGE)
if markers.has_legacy_marker(message):
problems.append(markers.LEGACY_MARKER_MESSAGE)
return problems


def find_unverified_claim(message):
"""Return the offending phrase if a paragraph makes an unverified-shaped
claim with no evidence marker in that same paragraph.

`UNVERIFIED:` anywhere still silences the whole message. A fence only
silences the paragraph it sits in -- not a later/earlier claim. Inline
code silences its paragraph only when it looks like command output or
the message carries a fenced block of output. This is a blunt proxy,
not a truth check."""
if UNVERIFIED_RE.search(message):
return None
A well-formed `{{CAT-UNVERIFIED}}` tag silences the paragraph it sits
in, exactly like a fence -- not a later/earlier claim. Inline code
silences its paragraph only when it looks like command output or the
message carries a fenced block of output. This is a blunt proxy, not a
truth check."""
fenced_output = any(OUTPUT_SHAPE_RE.search(body) for body in FENCED_BODY_RE.findall(message))
for para in re.split(r"\n\s*\n", message):
if FENCE_MARKER in para:
continue
if markers.excuses_paragraph(para):
continue
inline = INLINE_CODE_RE.findall(para)
if inline and (fenced_output or any(OUTPUT_SHAPE_RE.search(code) for code in inline)):
continue
Expand Down Expand Up @@ -160,27 +184,22 @@ def main():
word_count = counted_words(message)
over_limit = word_count > WORD_LIMIT
claim = find_unverified_claim(message)
unverified_marker = has_unresolved_unverified_marker(message)
marker_problems = find_marker_problems(message)

if not over_limit and not claim and not unverified_marker:
if not over_limit and not claim and not marker_problems:
return

parts = []
if claim:
parts.append(
f"This message makes an unverified-shaped claim (\"{claim}\") with no "
"adjacent evidence (pasted command output, or an `UNVERIFIED:` "
"prefix). A backticked name or command alone is not output. Per "
"skills/prove-it/SKILL.md: either paste the output of what was "
"actually run/checked, or prefix the claim with `UNVERIFIED:`."
)
if unverified_marker:
parts.append(
"This message contains an `UNVERIFIED:` claim. Per skills/prove-it/"
"SKILL.md: attempt to verify it now (run the actual check) before "
"finishing this turn, or tell the user explicitly what is blocking "
"verification and why it can't happen right now."
"adjacent evidence (pasted command output, or a "
f"`{markers.TAG_TEMPLATE}` tag). A backticked name or command alone "
"is not output. Per skills/prove-it/SKILL.md: either paste the "
"output of what was actually run/checked, or -- only if the check "
"cannot run -- tag the claim and say why."
)
parts.extend(marker_problems)
if over_limit:
parts.append(
f"Apply diu: {word_count} words, over the {WORD_LIMIT}-word "
Expand Down
Loading
Loading