From 6a541b6718d319f06bc7aa417f6ea9d0184352a2 Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Fri, 11 Sep 2026 02:24:05 -0700 Subject: [PATCH] docs: the engine rules tell authors to write the tag The hooks clear on the tag and nothing else, but engine/CLAUDE.core.md still said to write bare `UNVERIFIED:`. An agent following them was blocked for doing what they said. The core rule changes meaning, not only spelling. "If you have no evidence, write UNVERIFIED:" made the marker the default answer to missing evidence. It now says: run the check; if it genuinely cannot run, tag it and name the blocker. Also updated: the make-pr fixture-vs-live gate and the wrong-check-reflect Cursor rule. tests/escape_hatch_vocab.py finds any line that still instructs the retired marker. On 509b9cd it flags 30 lines across the repo; with the whole stack applied it flags 0. make-pr's new test_skill_text.py uses it. Left alone on purpose, since the word means something else there: "UNVERIFIED schema" (Codex hooks.json) and "UNVERIFIED end-to-end" (a status cell), plus recorded transcripts and fixtures, which are history. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VKsvxJk65w6q7KnPSRYvNg --- engine/CLAUDE.core.md | 4 +- .../wrong-check-reflect/evidence-check.mdc | 2 +- engine/skills/make-pr/SKILL.md | 2 +- .../skills/make-pr/tests/test_skill_text.py | 30 ++++++++++++ tests/escape_hatch_vocab.py | 46 +++++++++++++++++++ 5 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 engine/skills/make-pr/tests/test_skill_text.py create mode 100644 tests/escape_hatch_vocab.py diff --git a/engine/CLAUDE.core.md b/engine/CLAUDE.core.md index dcb73620..2e4c4253 100644 --- a/engine/CLAUDE.core.md +++ b/engine/CLAUDE.core.md @@ -19,7 +19,7 @@ These override brevity. If proof makes a message longer, the message gets longer 1. A command I actually ran, shown with its real pasted output (not summarized, not paraphrased). 2. A `file:line` reference to code I read this session. 3. A test name plus its real pass/fail line from the runner. -- If I have none of those, I must write `UNVERIFIED:` immediately before the claim. No exceptions, no softer wording. +- If I have none of those, I must run the check. If the check genuinely cannot run, I tag the claim `{{CAT-UNVERIFIED: -- cannot verify: }}` and name the blocker. A tag that names no blocker is not an exception, it is the same miss in new syntax. Bare `UNVERIFIED:` is retired and excuses nothing. - Banned phrases about code I have not executed: "this should work", "this fixes it", "that's the bug", "now it works", "verified", "confirmed". - A repro script is proof **only** if I show it FAILING before the change and PASSING after, with both outputs pasted. A script that only passes proves nothing. - Absence of output is not proof of success. A command that printed nothing needs its exit code shown. @@ -28,7 +28,7 @@ These override brevity. If proof makes a message longer, the message gets longer - When I catch myself about to assert something I did not observe, stop and run the check instead of writing the sentence. - **A check that could not run is not a pass.** When a guard, gate, scan, or query meets input it cannot read — a file past a size cap, an unresolved path, a field a projection omits, a probe that errored — it says so or refuses. It never returns clean. Give such a check three outcomes (hit, clean, unchecked), not two, and pin the third with a test; whether it then fails open or closed is a per-check decision that gets written down. Saltzer and Schroeder put the burden the same way in "Basic Principles of Information Protection" (1975): base access decisions on permission rather than exclusion, so the default is lack of access and the scheme names the conditions under which access is permitted (https://web.mit.edu/Saltzer/www/publications/protection/Basic.html). - The same rule applies to claims about the conversation itself, not just about code: "I ignored/missed/forgot X" is a claim that needs evidence too. Grep the actual transcript for the instruction before saying that. If nothing turns up, say "I don't have a record of that instruction in this session" — not self-blaming language for something that was never said. -- A Grep or name hit is not a check. Do not cite a file, line, or "the bug is X" until this turn's Read or command output is in the same message. If two files could match, Read both. Prefix `UNVERIFIED:` until then. Saying "my earlier check was wrong" means the claim went out before the check — that is a process failure, not a polite recovery. +- A Grep or name hit is not a check. Do not cite a file, line, or "the bug is X" until this turn's Read or command output is in the same message. If two files could match, Read both. Tag it `{{CAT-UNVERIFIED: -- cannot verify: }}` until then. Saying "my earlier check was wrong" means the claim went out before the check — that is a process failure, not a polite recovery. # Session hygiene (apply everywhere, every project) diff --git a/engine/hooks/wrong-check-reflect/evidence-check.mdc b/engine/hooks/wrong-check-reflect/evidence-check.mdc index e3f3230f..9d839def 100644 --- a/engine/hooks/wrong-check-reflect/evidence-check.mdc +++ b/engine/hooks/wrong-check-reflect/evidence-check.mdc @@ -12,7 +12,7 @@ A Grep or name hit is **not** a check. - Do not cite a file, line, or "the bug is X" until this turn's Read or command output is in the same message. - If two files could match, Read both before picking one. -- Prefix `UNVERIFIED:` until you have that evidence. +- Run the check. Only when it cannot run, tag the claim `{{CAT-UNVERIFIED: -- cannot verify: }}`; bare `UNVERIFIED:` is retired and excuses nothing. - "My earlier check was wrong" means the claim went out before the check — that is a process failure, not a polite recovery. The `wrong-check-reflect` hook will force `/reflect` when it sees that diff --git a/engine/skills/make-pr/SKILL.md b/engine/skills/make-pr/SKILL.md index 42246f68..1b4ae0b8 100644 --- a/engine/skills/make-pr/SKILL.md +++ b/engine/skills/make-pr/SKILL.md @@ -114,7 +114,7 @@ side effects (Linear, deploy, live mine, external APIs): - Require an explicit **fixture vs live** split in the Test Plan and Summary. - Either include live evidence from the same turn, or prefix unsettled live - claims with `UNVERIFIED: live path`. + claims with `{{CAT-UNVERIFIED: -- cannot verify: }}`. - Visual Proof that only shows UI registration must not be framed as product e2e of the live side effect. diff --git a/engine/skills/make-pr/tests/test_skill_text.py b/engine/skills/make-pr/tests/test_skill_text.py new file mode 100644 index 00000000..b829a3b9 --- /dev/null +++ b/engine/skills/make-pr/tests/test_skill_text.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""The fixture-vs-live gate names the tag every hook accepts. + +Run: python3 -m unittest discover -s engine/skills/make-pr/tests -v +""" +import os +import sys +import unittest + +SKILL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(SKILL_DIR))) +sys.path.insert(0, os.path.join(REPO_ROOT, "tests")) + +import escape_hatch_vocab as vocab # noqa: E402 + + +class FixtureVsLiveGate(unittest.TestCase): + def setUp(self): + with open(os.path.join(SKILL_DIR, "SKILL.md"), encoding="utf-8") as handle: + self.text = handle.read() + + def test_unsettled_live_claims_use_the_tag(self): + self.assertIn("{{CAT-UNVERIFIED: -- cannot verify: }}", self.text) + + def test_skill_never_tells_the_author_to_write_the_retired_marker(self): + self.assertEqual(vocab.instructs_retired_marker(self.text), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/escape_hatch_vocab.py b/tests/escape_hatch_vocab.py new file mode 100644 index 00000000..4cc7d430 --- /dev/null +++ b/tests/escape_hatch_vocab.py @@ -0,0 +1,46 @@ +"""Find prose that still tells an author to write the retired escape hatch. + +Every evidence hook clears on `{{CAT-UNVERIFIED: -- cannot verify: +}}` and nothing else. A rule or skill that still says "write +`UNVERIFIED:`" sends an agent straight into a block, so the instruction and +the hooks have to agree. + +A line counts when it carries a bare `UNVERIFIED:` (not the `CAT-` tag) and +does not describe it as retired. Other uses of the word without a colon -- +"UNVERIFIED schema", "UNVERIFIED end-to-end" -- are not the marker. +""" +from __future__ import annotations + +import os +import re + +BARE_MARKER_RE = re.compile(r"(? list[str]: + return [ + line.strip() + for line in text.splitlines() + if BARE_MARKER_RE.search(line) and not RETIRED_RE.search(line) + ] + + +def instructional_files() -> list[str]: + fixed = [ + "engine/CLAUDE.core.md", + "corpus/CLAUDE.learned.md", + "README.md", + "docs/hooks-gap-analysis.md", + ] + found = [os.path.join(REPO_ROOT, rel) for rel in fixed] + for top in ("always-on", "engine", "corpus", "product"): + for dirpath, dirnames, filenames in os.walk(os.path.join(REPO_ROOT, top)): + dirnames[:] = [d for d in dirnames if d not in ("tests", "__pycache__", "node_modules")] + for name in filenames: + if name == "SKILL.md" or name == "README.md" or name.endswith(".mdc") \ + or (name.endswith(".md") and os.path.basename(dirpath) in ("references", "always-on")): + found.append(os.path.join(dirpath, name)) + return sorted(set(p for p in found if os.path.isfile(p)))