pr-schema-gate: an exit-0 validator that printed UNCHECKED is not a clean run

engine/CLAUDE.core.md

@@ -23,6 +23,7 @@ These override brevity. If proof makes a message longer, the message gets longer
 - Banned phrases about code I have not executed: "this should work", "this fixes it", "that's the bug", "now it works", "verified", "confirmed".
 - A repro script is proof **only** if I show it FAILING before the change and PASSING after, with both outputs pasted. A script that only passes proves nothing.
 - Absence of output is not proof of success. A command that printed nothing needs its exit code shown.
+- A command can succeed and still answer from the wrong source. When a tool can degrade to a cache, a replica, a read-only copy, or an offline mode, exit 0 and printed data are not evidence the data is live — the degradation notice goes to stderr, not into the payload. Before reporting live state from such a tool, show the exit code and stderr of the same invocation, or name the source the answer came from. A reading also expires: after a restart, kill, or redeploy of the thing being read, every value taken before that event is stale and must be re-read before it is quoted.
 - If a test was skipped, timed out, or I ran a subset, say exactly which and why — never let a partial run stand in for a full one.
 - If the user asks "did you verify X?", answer yes or no first, then show the evidence or admit there is none. Do not re-argue the original claim.
 - When I catch myself about to assert something I did not observe, stop and run the check instead of writing the sentence.

engine/hooks/pr-schema-gate/detect.py

@@ -38,6 +38,7 @@
 import hashlib
 import json
 import os
+import re
 import subprocess
 import sys
 import tempfile
@@ -49,6 +50,8 @@
 VALIDATOR_RELATIVE_PATH = os.path.join("scripts", "validate-pr-body.mjs")
 VALIDATOR_TIMEOUT_SECONDS = 3.0
 VALIDATOR_OUTPUT_MAX_LINES = 20
+VACUOUS_PASS_RE = re.compile(
+    r"\bUNCHECKED\b|\bSKIPPED?\b|\bnot installed\b|\bno rules loaded\b", re.IGNORECASE)
 
 PENDING_TTL_SECONDS = 2 * 60 * 60
 STATE_DIR_ENV = "PR_SCHEMA_GATE_STATE_DIR"
@@ -300,6 +303,9 @@ def check_body_file(repo_root: str, body_path: str | None, start_dir: str) -> tu
     lines = [line for line in (proc.stdout + "\n" + proc.stderr).splitlines() if line.strip()]
     lines = lines[:VALIDATOR_OUTPUT_MAX_LINES]
     if proc.returncode == 0:
+        for line in lines:
+            if VACUOUS_PASS_RE.search(line):
+                return "unchecked", f"the validator exited 0 without checking: {line.strip()}"
         return "clean", ""
     if proc.returncode == 1:
         return "failed", "\n".join(lines)

engine/hooks/pr-schema-gate/tests/test_advisory.py

@@ -31,6 +31,9 @@
     "process.exit(1);\n"
 )
 VALIDATOR_CRASHES = 'console.error("Error: Cannot find module typescript");\nprocess.exit(3);\n'
+VALIDATOR_EXITS_ZERO_UNCHECKED = (
+    'console.log("UNCHECKED: PR body rules not checked (drafter-core not installed)");\n'
+)
 VALIDATOR_HANGS = "setTimeout(() => {}, 60000);\n"
 
 
@@ -165,6 +168,21 @@ def test_validator_crash_is_reported_as_unchecked_not_clean(self):
             self.assertIn("could not check", context)
             self.assertIn("exit 3", context)
 
+    def test_validator_exit_zero_with_an_unchecked_line_is_not_clean(self):
+        with _repo(VALIDATOR_EXITS_ZERO_UNCHECKED) as repo:
+            body = _body_file(repo)
+            code, _, context = _run(GH_PR + "edit 7 --body-file " + body, repo)
+            self.assertEqual(code, 0)
+            self.assertIn("could not check", context)
+            self.assertIn("exited 0 without checking", context)
+
+    def test_validator_real_pass_stays_clean(self):
+        with _repo(VALIDATOR_PASSES) as repo:
+            body = _body_file(repo)
+            code, _, context = _run(GH_PR + "edit 7 --body-file " + body, repo)
+            self.assertEqual(code, 0)
+            self.assertNotIn("could not check", context)
+
     def test_validator_timeout_is_reported_as_unchecked(self):
         original = detect.VALIDATOR_TIMEOUT_SECONDS
         detect.VALIDATOR_TIMEOUT_SECONDS = 0.5