diff --git a/engine/hooks/hedge-runs-prove-it/README.md b/engine/hooks/hedge-runs-prove-it/README.md index 266b85f..fd9c243 100644 --- a/engine/hooks/hedge-runs-prove-it/README.md +++ b/engine/hooks/hedge-runs-prove-it/README.md @@ -48,6 +48,20 @@ Stays silent on a diagnosis inside a fence, a double-quoted span, a backtick span, a markdown blockquote, a hypothetical ("if it's a zombie, ..."), and past-tense narration of an old incident ("the task was a zombie"). +## Capability lists copied from errors + +An error message can expose a local fallback or hardcoded enumeration without +proving the capabilities of the system named in that error. The hook records +values in bracketed or comma-joined lists from error-shaped tool results. It +blocks an outgoing reply that repeats at least two of those values within 200 +characters of `accepts`, `supports`, `known models`, `valid`, or `only` when +those values appeared nowhere in a non-error tool result. + +One repeated value stays silent to avoid collisions on common tokens. The rule +also stays silent when a non-error tool result supplies the repeated values, or +when the reply attributes or retracts the enumeration with wording such as +`fallback`, `hardcoded`, `built-in`, `in the error`, or `retract`. + Mechanical half of `corpus/skills/cat-mode/SKILL.md`'s Verify rule: "Unhedged root-cause or fix claims about live system behavior need instrument-level proof in the same message, or `{{CAT-UNVERIFIED}}`." Four @@ -73,7 +87,8 @@ escape hatch, not a free pass). Fail-open on parse or read errors; ## Files -- `detect.py` -- hedge, code-noun, and reason patterns; turn scan; `decide()`. +- `detect.py` -- hedge, diagnosis, and error-only capability-list patterns; + turn scan; `decide()`. - `claude_stop_check.py` -- Claude Stop entrypoint. - `claude.hook.json` / `install_claude_hook.py` -- settings.json merge (idempotent). - `tests/fixtures/hedges_{fires,silent}.json` -- sanitized real replies and diff --git a/engine/hooks/hedge-runs-prove-it/detect.py b/engine/hooks/hedge-runs-prove-it/detect.py index d4ff2e6..99094d8 100644 --- a/engine/hooks/hedge-runs-prove-it/detect.py +++ b/engine/hooks/hedge-runs-prove-it/detect.py @@ -1,6 +1,6 @@ -"""hedge-runs-prove-it: an unrun check about code or repo state is a prompt to verify. +"""hedge-runs-prove-it: unproven code, state, and capability claims must be verified. -Two shapes, two bars. +Three shapes, three bars. A hedge -- "I think", "I believe", "probably", "should work", "presumably", or a `{{CAT-UNVERIFIED}}` tag next to a code noun (a path, a backticked name, @@ -18,6 +18,11 @@ Only instrument-level proof in the same message clears it: pasted output, a `file:line`, a pid, an exit code, or an explicit `{{CAT-UNVERIFIED}}` tag. +A capability enumeration copied from error-shaped tool output is not proof of +what another system accepts or supports. A reply that repeats two or more of +those values beside a capability verb is blocked unless a non-error tool result +also supplied them, or the reply attributes the list as fallback/error data. + Hedges about things that are not code or state (a company's motive, a filing date) are out of scope, and so is either shape quoted rather than claimed -- anywhere inside a double-quoted, backticked or single-quoted run, @@ -118,6 +123,36 @@ "file:line. Otherwise tag the claim: `{tag}`." ) +ERROR_OUTPUT_RE = re.compile( + r"\b(?:error|exception|fatal|failure|failed|invalid|unsupported|traceback)\b|" + r"\bnot\s+supported\b", + re.IGNORECASE, +) +BRACKETED_ENUM_RE = re.compile(r"\[([^\[\]\n]+)\]") +COMMA_ENUM_RE = re.compile( + r"(? str: end = len(text) @@ -215,6 +250,80 @@ def _is_human_user_line(data: dict) -> bool: return bool(text.strip()) and not text.lstrip().startswith("<") +def _tool_result_text(block: dict) -> str: + content = block.get("content") + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + chunks: list[str] = [] + for item in content: + if isinstance(item, str): + chunks.append(item) + elif isinstance(item, dict) and isinstance(item.get("text"), str): + chunks.append(item["text"]) + return "\n".join(chunks) + + +def _tool_results(lines: list[dict]): + for data in lines: + message = data.get("message") + content = message.get("content") if isinstance(message, dict) else data.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_result": + yield block, _tool_result_text(block) + + +def _enumerated_values(text: str) -> set[str]: + runs = [match.group(1) for match in BRACKETED_ENUM_RE.finditer(text or "")] + runs.extend(match.group(1) for match in COMMA_ENUM_RE.finditer(text or "")) + values: set[str] = set() + for run in runs: + for raw_value in run.split(","): + value = raw_value.strip().strip("'\"`") + if len(value) >= 2 and ENUM_VALUE_RE.fullmatch(value): + values.add(value.lower()) + return values + + +def _value_occurs(text: str, value: str) -> bool: + return bool(re.search( + rf"(? set[str]: + """Enumerated values whose tool-result sources are all error-shaped.""" + error_values: set[str] = set() + non_error_results: list[str] = [] + for block, text in _tool_results(lines): + error_shaped = bool(block.get("is_error") or ERROR_OUTPUT_RE.search(text)) + if error_shaped: + error_values.update(_enumerated_values(text)) + else: + non_error_results.append(text) + return { + value for value in error_values + if not any(_value_occurs(text, value) for text in non_error_results) + } + + +def _capability_feedback(message: str, lines: list[dict]) -> str | None: + if not CAPABILITY_RE.search(message or "") or CAPABILITY_ATTRIBUTION_RE.search(message or ""): + return None + values = error_only_capability_values(lines) + for verb in CAPABILITY_RE.finditer(message): + window = message[max(0, verb.start() - PROXIMITY): verb.end() + PROXIMITY] + repeated = sorted(value for value in values if _value_occurs(window, value)) + if len(repeated) >= 2: + return CAPABILITY_MESSAGE.format(values=", ".join(repeated[:4])) + return None + + def parse_lines(raw_lines) -> list[dict]: parsed: list[dict] = [] for raw in raw_lines: @@ -257,11 +366,12 @@ def decide_from_lines(message: str, lines: list[dict]) -> str | None: if diagnosis: return diagnosis hedges = code_hedges(message) - if not hedges: - return None - if verified_this_turn(lines): - return None - return MESSAGE.format(tag=markers.TAG_TEMPLATE, hedge=", ".join(f'"{h}"' for h in hedges[:3])) + if hedges and not verified_this_turn(lines): + return MESSAGE.format( + tag=markers.TAG_TEMPLATE, + hedge=", ".join(f'"{h}"' for h in hedges[:3]), + ) + return _capability_feedback(message, lines) def decide(payload: dict) -> str | None: @@ -272,7 +382,7 @@ def decide(payload: dict) -> str | None: diagnosis = _diagnosis_feedback(message) if diagnosis: return diagnosis - if not code_hedges(message): + if not code_hedges(message) and not CAPABILITY_RE.search(message): return None transcript_path = payload.get("transcript_path") or payload.get("transcriptPath") or "" lines: list[dict] = [] diff --git a/engine/hooks/hedge-runs-prove-it/tests/test_hooks.py b/engine/hooks/hedge-runs-prove-it/tests/test_hooks.py index 8f8c6ad..314a2a8 100644 --- a/engine/hooks/hedge-runs-prove-it/tests/test_hooks.py +++ b/engine/hooks/hedge-runs-prove-it/tests/test_hooks.py @@ -43,6 +43,35 @@ def turn_lines(verified): return lines +ERROR_CAPABILITY_LIST = ( + 'Error: Execution model "gpt-5.6-luna" is not supported for execution agent ' + '"codex". Known models: [gpt-5.5, gpt-5.5-pro, gpt-5.4, gpt-5.4-pro].' +) + + +def capability_lines(with_non_error_source=False): + lines = turn_lines(False) + [ + {"type": "assistant", "message": {"role": "assistant", "content": [ + {"type": "tool_use", "id": "models-error", "name": "Task", "input": {}} + ]}}, + {"type": "user", "message": {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "models-error", "content": ERROR_CAPABILITY_LIST} + ]}}, + ] + if with_non_error_source: + lines.extend([ + {"type": "assistant", "message": {"role": "assistant", "content": [ + {"type": "tool_use", "id": "models-answer", "name": "Task", "input": {}} + ]}}, + {"type": "user", "message": {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "models-answer", "content": ( + "Supported execution models: gpt-5.5, gpt-5.5-pro, gpt-5.4, gpt-5.4-pro." + )} + ]}}, + ]) + return lines + + def transcript_file(lines): tmp = tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False, encoding="utf-8") tmp.write("\n".join(json.dumps(line) for line in lines) + "\n") @@ -218,5 +247,37 @@ def test_diagnosis_gate_ignores_an_unreadable_transcript(self): })) +class TestBlocksCapabilitiesCopiedOnlyFromErrors(unittest.TestCase): + def test_hook_blocks_two_values_repeated_beside_only_accepts(self): + path = transcript_file(capability_lines()) + try: + code, err = run_hook({ + "last_assistant_message": "codex only accepts [gpt-5.5, gpt-5.5-pro].", + "transcript_path": path, + }) + finally: + os.unlink(path) + self.assertEqual(code, 2) + self.assertIn("error-shaped tool output", err) + + def test_allows_values_also_listed_by_non_error_tool_result(self): + self.assertIsNone(detect.decide_from_lines( + "codex only accepts [gpt-5.5, gpt-5.5-pro].", + capability_lines(with_non_error_source=True), + )) + + def test_allows_attributed_fallback_list(self): + self.assertIsNone(detect.decide_from_lines( + "The built-in fallback list says codex only accepts [gpt-5.5, gpt-5.5-pro].", + capability_lines(), + )) + + def test_allows_a_single_repeated_value(self): + self.assertIsNone(detect.decide_from_lines( + "codex only accepts gpt-5.5.", + capability_lines(), + )) + + if __name__ == "__main__": unittest.main()