diff --git a/docs/ecosystem.md b/docs/ecosystem.md index 91347b6..6964c4a 100644 --- a/docs/ecosystem.md +++ b/docs/ecosystem.md @@ -73,7 +73,7 @@ again. | `explicit-failures` | hook (advisory; always on) | | `external-claim-gate` | hook (PreToolUse on `Bash`; blocks a gh issue/comment/release/api write whose body claims a cause or fix with no evidence; blocks as UNCHECKED when the body cannot be read) | | `playbook-router` | hook (UserPromptSubmit; injects the steps of the one playbook a prompt names) | -| `diu-stop` | hook | +| `diu-stop` | hook (Stop; blocks a reply over the word limit or with an unproven claim, and asks the background judge whether the reply used wording from its `phrases/` lists, waiting for that answer so a hit blocks the same turn) | | `frustration-watchdog` | hook | | `named-verb-guard` | hook | | `plan-discipline` | hook (not always installed) | diff --git a/engine/hooks/diu-stop/README.md b/engine/hooks/diu-stop/README.md index 1eb0e92..d9fef8a 100644 --- a/engine/hooks/diu-stop/README.md +++ b/engine/hooks/diu-stop/README.md @@ -26,10 +26,12 @@ power at that point: ## Files - `claude.hook.json` -- the `Stop` hook `"hooks"` object to merge into `~/.claude/settings.json`. -- `claude_stop_check.py` -- the script that hook runs. No LLM, no machine-specific paths. +- `claude_stop_check.py` -- the script that hook runs. Its word count and claim checks use no model; it also asks the background judge about the `phrases/` word lists and waits for that answer, so a hit blocks the same turn. No machine-specific paths. - `claude.prompt.hook.json` -- the `UserPromptSubmit` hook `"hooks"` object, merged the same way. - `claude_prompt_reminder.py` -- the script that hook runs. No LLM, no per-turn conditional logic -- always emits the same short reminder. - `diu_limit.py` -- the word limit and what it does not count. The reminder's wording and the Stop hook's check both read it, so they cannot disagree; `tests/test_limit_agreement.py` pins that. +- `plain_words.py` -- turns every `phrases/` word list into one question about the user's last message and the finished reply, hands it to the background judge, and waits for the answer so a hit blocks the same turn. A verdict delivered on the next prompt is one the user may never see. No answer in time means the turn ends unblocked. +- `phrases/` -- the word lists themselves, one file per kind of wording to avoid, in the format `engine/hooks/llm-judge/phrases.py` loads. - `install_claude_hook.py` -- merges both of the above into `~/.claude/settings.json`, idempotently, without touching anything else there. - `cursor.hooks.json` -- the whole file to install as `~/.cursor/hooks.json`. - `codex_notify.py` -- the script to point Codex's `notify` at. No machine-specific paths. diff --git a/engine/hooks/diu-stop/claude.hook.json b/engine/hooks/diu-stop/claude.hook.json index 483b97e..b230175 100644 --- a/engine/hooks/diu-stop/claude.hook.json +++ b/engine/hooks/diu-stop/claude.hook.json @@ -7,7 +7,7 @@ { "type": "command", "command": "python3 $HOME/.claude/hooks/diu-stop/claude_stop_check.py", - "timeout": 10 + "timeout": 60 } ] } diff --git a/engine/hooks/diu-stop/claude_stop_check.py b/engine/hooks/diu-stop/claude_stop_check.py index 09b3d50..079f8f7 100755 --- a/engine/hooks/diu-stop/claude_stop_check.py +++ b/engine/hooks/diu-stop/claude_stop_check.py @@ -39,6 +39,7 @@ import sys from diu_limit import WORD_LIMIT, counted_words +from plain_words import try_check_reply sys.path.insert(0, os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_markers")) @@ -181,15 +182,19 @@ def main(): message = data.get("last_assistant_message") or "" + plain_words_note = try_check_reply(data) + word_count = counted_words(message) over_limit = word_count > WORD_LIMIT claim = find_unverified_claim(message) marker_problems = find_marker_problems(message) - if not over_limit and not claim and not marker_problems: + if not over_limit and not claim and not marker_problems and not plain_words_note: return parts = [] + if plain_words_note: + parts.append(plain_words_note) if claim: parts.append( f"This message makes an unverified-shaped claim (\"{claim}\") with no " diff --git a/engine/hooks/diu-stop/phrases/plain-words-code-names.json b/engine/hooks/diu-stop/phrases/plain-words-code-names.json new file mode 100644 index 0000000..85b9f69 --- /dev/null +++ b/engine/hooks/diu-stop/phrases/plain-words-code-names.json @@ -0,0 +1,17 @@ +{ + "checker": "plain-words-code-names", + "meaning": "The reply puts a raw code, config, or function name in front of the user as if it were a word, without saying in everyday words what it does.", + "reads": "exchange", + "match": [ + "Set disable-model-invocation: true on the skill.", + "It flips desiredEnabled: true.", + "materialize_revision now calls realpath.", + "The check reads EXIT_CODES and PULL_REF_RE." + ], + "not_match": [ + "The skill is hidden from the automatic skill list, so only a typed command can start it.", + "The worker is switched on.", + "The replay tool now follows folder shortcuts before comparing paths (backtest_detector.py:102)." + ], + "on_hit": "plain-words: the last reply named code or settings without saying what they do in everyday words." +} diff --git a/engine/hooks/diu-stop/phrases/plain-words-internal-names.json b/engine/hooks/diu-stop/phrases/plain-words-internal-names.json new file mode 100644 index 0000000..2f0201c --- /dev/null +++ b/engine/hooks/diu-stop/phrases/plain-words-internal-names.json @@ -0,0 +1,18 @@ +{ + "checker": "plain-words-internal-names", + "meaning": "The reply names an internal tool, worker, queue, or process step as if the user already knows what it is, without saying what it does.", + "reads": "exchange", + "match": [ + "It points at a real design gap in e2e-autofix.", + "Run the invoker-watcher first.", + "The backlog grew after the redeploy before it started shrinking.", + "Preflight passes and the review unit is engine-runtime.", + "The merge gate is review_ready." + ], + "not_match": [ + "The worker that retries failed browser tests is making many copies of one job.", + "The repo's pre-publish check passes.", + "You asked about the merge queue: it is the line of PRs waiting for checks before they merge." + ], + "on_hit": "plain-words: the last reply named an internal tool or step without saying what it does." +} diff --git a/engine/hooks/diu-stop/phrases/plain-words-made-up-labels.json b/engine/hooks/diu-stop/phrases/plain-words-made-up-labels.json new file mode 100644 index 0000000..cc21570 --- /dev/null +++ b/engine/hooks/diu-stop/phrases/plain-words-made-up-labels.json @@ -0,0 +1,18 @@ +{ + "checker": "plain-words-made-up-labels", + "meaning": "The reply uses a label the assistant made up while working, which the user has not used, and does not say in everyday words what it means.", + "reads": "exchange", + "match": [ + "No hook decides differently.", + "The hooks only gain replay functions.", + "Is this the right safety line?", + "That was the fake problem.", + "Each slice passes on its own." + ], + "not_match": [ + "The hook's logic does not change; this is a simple refactor.", + "Each hook gets a small extra function that replays old chats; the live hook never calls it.", + "You asked about the safety line: it is one sentence saying why the change cannot break anything." + ], + "on_hit": "plain-words: the last reply used a made-up label; say what it means in everyday words or drop it." +} diff --git a/engine/hooks/diu-stop/phrases/plain-words-status-words.json b/engine/hooks/diu-stop/phrases/plain-words-status-words.json new file mode 100644 index 0000000..c018486 --- /dev/null +++ b/engine/hooks/diu-stop/phrases/plain-words-status-words.json @@ -0,0 +1,18 @@ +{ + "checker": "plain-words-status-words", + "meaning": "The reply uses a status word or state label without saying what actually happened or why.", + "reads": "exchange", + "match": [ + "Three tests were capped.", + "The reset is blocked.", + "That would double count.", + "Only a restart picks up newer code.", + "The branch is stale." + ], + "not_match": [ + "Three tests hit the retry limit, so they stopped and now need a person.", + "The restart could not run because a check refused it.", + "The running program still uses the version it loaded earlier; a restart loads the current version." + ], + "on_hit": "plain-words: the last reply used a status word without saying what happened." +} diff --git a/engine/hooks/diu-stop/phrases/plain-words-tech-jargon.json b/engine/hooks/diu-stop/phrases/plain-words-tech-jargon.json new file mode 100644 index 0000000..7e0e6ab --- /dev/null +++ b/engine/hooks/diu-stop/phrases/plain-words-tech-jargon.json @@ -0,0 +1,18 @@ +{ + "checker": "plain-words-tech-jargon", + "meaning": "The reply uses technical jargon or an abbreviation the user has not used, without a plain explanation next to it.", + "reads": "exchange", + "match": [ + "It died on a 401 unauthorized.", + "That is a remote infra gap.", + "The script ships as an SEA.", + "Mostly regex false positives.", + "Force-push with lease, then a three-way apply." + ], + "not_match": [ + "The login had expired, so the server refused the request.", + "The problem is on the other machines, not this laptop.", + "Most of the alarms were wrong: the word matched but the meaning did not." + ], + "on_hit": "plain-words: the last reply used jargon or an abbreviation without a plain explanation." +} diff --git a/engine/hooks/diu-stop/plain_words.py b/engine/hooks/diu-stop/plain_words.py new file mode 100644 index 0000000..061822a --- /dev/null +++ b/engine/hooks/diu-stop/plain_words.py @@ -0,0 +1,184 @@ +"""Ask the background judge whether the last reply used wording the user has +to ask about, and wait for the answer. + +The word lists live in phrases/, one file per kind of wording. They become one +question, so one model call covers every kind and names the one it found. The +Stop hook waits for that answer and shows it in the same turn, because a +verdict that waits for the user's next message is a verdict the user may never +see. When no answer arrives in time, the turn ends unblocked. +""" +from __future__ import annotations + +import json +import os +import sys +import time +import uuid + +HOOK_DIR = os.path.dirname(os.path.abspath(__file__)) +LLM_JUDGE_DIR = os.path.join(os.path.dirname(HOOK_DIR), "llm-judge") +PHRASES_DIR = os.path.join(HOOK_DIR, "phrases") +PREFIX = "plain-words-" +WAIT_ENV = "DIU_PLAIN_WORDS_WAIT_SECONDS" +DEFAULT_WAIT_SECONDS = 40.0 +POLL_SECONDS = 0.5 +HOOK_NAME = "diu-plain-words" +TEXT_LIMIT = 4000 +META_USER_PREFIXES = (" list[str]: + try: + names = os.listdir(directory) + except OSError as exc: + raise ValueError(f"{directory}: word lists could not be listed: {exc}") from exc + return sorted(name[: -len(".json")] for name in names if name.startswith(PREFIX) and name.endswith(".json")) + + +def _is_user_line(data: dict) -> bool: + if data.get("type") == "user": + return True + message = data.get("message") + return isinstance(message, dict) and message.get("role") == "user" + + +def _message_text(data: dict) -> str: + message = data.get("message") + content = message.get("content") if isinstance(message, dict) else data.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + parts.append(block.get("text") or "") + elif isinstance(block, str): + parts.append(block) + return "\n".join(parts) + return "" + + +def last_user_message(path: str) -> str: + if not path or not os.path.isfile(path): + return "" + found = "" + with open(path, encoding="utf-8", errors="replace") as handle: + for line in handle: + try: + data = json.loads(line) + except ValueError: + continue + if not isinstance(data, dict) or data.get("isSidechain") or data.get("isMeta"): + continue + if not _is_user_line(data): + continue + text = _message_text(data).strip() + if text and not text.startswith(META_USER_PREFIXES): + found = text + return found + + +def prompt(dictionaries: list[dict], asked: str, reply: str) -> str: + lines = [ + f"Return exactly one line of JSON: {ANSWER_SHAPE}", + "Each list below names a kind of wording to avoid when writing to this user.", + ] + for dictionary in dictionaries: + lines.append("") + lines.append(f"List {dictionary['checker']}: {dictionary['meaning']}") + lines.append(f"Examples that match: {json.dumps(dictionary['match'], ensure_ascii=False)}") + lines.append(f"Examples that do not match: {json.dumps(dictionary['not_match'], ensure_ascii=False)}") + lines.extend( + [ + "", + "Set match to true only when the ASSISTANT text below uses such wording.", + "closest must be copied word for word from the ASSISTANT text, never from the lists.", + "A word the USER used first does not count. A word that is only quoted, negated, or described does not count.", + "", + f"USER:\n{asked[-TEXT_LIMIT:]}", + "", + f"ASSISTANT:\n{reply[-TEXT_LIMIT:]}", + ] + ) + return "\n".join(lines) + + +def job(payload: dict) -> dict | None: + reply = payload.get("last_assistant_message") or "" + transcript = payload.get("transcript_path") or "" + if not reply.strip() or not transcript or not os.path.isfile(transcript): + return None + _, phrases = _llm_judge() + dictionaries = [phrases.load(name, directory=PHRASES_DIR) for name in list_names()] + if not dictionaries: + return None + return { + "id": uuid.uuid4().hex, + "hook": HOOK_NAME, + "transcript": transcript, + "prompt": prompt(dictionaries, last_user_message(transcript), reply), + "hit_if_all_true": ["match"], + "on_hit": "diu: the last reply used wording the user has had to ask about; say it in everyday words.", + } + + +def message_for(verdict: dict, reply: str) -> str: + if verdict.get("outcome") != "hit": + return "" + answer = verdict.get("answer") or {} + closest = str(answer.get("closest") or "").strip() + if not closest or closest not in reply: + return "" + return MESSAGE.format(category=str(answer.get("category") or "unnamed list"), closest=closest) + + +def wait_seconds() -> float: + raw = os.environ.get(WAIT_ENV) + if raw is None: + return DEFAULT_WAIT_SECONDS + try: + return max(0.0, float(raw)) + except ValueError: + return DEFAULT_WAIT_SECONDS + + +def check_reply(payload: dict) -> str: + if not isinstance(payload, dict) or payload.get("agent_id") or payload.get("stop_hook_active"): + return "" + built = job(payload) + if built is None: + return "" + judge, _ = _llm_judge() + if judge.enqueue(built) is None: + return "" + deadline = time.monotonic() + wait_seconds() + transcript = built["transcript"] + while time.monotonic() < deadline: + for verdict in judge.drain(transcript): + if verdict.get("id") == built["id"]: + return message_for(verdict, payload.get("last_assistant_message") or "") + time.sleep(POLL_SECONDS) + return "" + + +def try_check_reply(payload: dict) -> str: + """check_reply for the hook scripts: an error is logged, never raised.""" + try: + return check_reply(payload) + except Exception as exc: + sys.stderr.write(f"diu-stop: plain-words check failed: {type(exc).__name__}: {exc}\n") + return "" diff --git a/engine/hooks/diu-stop/tests/test_plain_words.py b/engine/hooks/diu-stop/tests/test_plain_words.py new file mode 100644 index 0000000..9b7a2d3 --- /dev/null +++ b/engine/hooks/diu-stop/tests/test_plain_words.py @@ -0,0 +1,75 @@ +import os +import sys +import unittest + +HOOK_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LLM_JUDGE_DIR = os.path.join(os.path.dirname(HOOK_DIR), "llm-judge") +sys.path.insert(0, LLM_JUDGE_DIR) + +import phrases + +PHRASES_DIR = os.path.join(HOOK_DIR, "phrases") +PREFIX = "plain-words-" +CATEGORIES = ( + "plain-words-made-up-labels", + "plain-words-code-names", + "plain-words-internal-names", + "plain-words-tech-jargon", + "plain-words-status-words", +) +EDGE = ".,;:()!?\"'" + + +def _is_pr_number(token): + return token.startswith("#") and token.strip(EDGE).lstrip("#").isdigit() + + +def _is_date(token): + parts = token.strip(EDGE).split("-") + return len(parts) == 3 and len(parts[0]) == 4 and all(part.isdigit() for part in parts) + + +def _load(checker): + return phrases.load(checker, directory=PHRASES_DIR) + + +class TestPlainWords(unittest.TestCase): + def test_every_category_file_is_listed(self): + found = sorted(name[:-5] for name in os.listdir(PHRASES_DIR) if name.startswith(PREFIX) and name.endswith(".json")) + self.assertEqual(found, sorted(CATEGORIES)) + + def test_each_category_loads_and_reads_the_exchange(self): + for checker in CATEGORIES: + with self.subTest(checker=checker): + dictionary = _load(checker) + self.assertEqual(dictionary["reads"], "exchange") + self.assertGreaterEqual(len(dictionary["match"]), 3) + self.assertGreaterEqual(len(dictionary["not_match"]), 2) + self.assertTrue(dictionary["on_hit"].startswith("plain-words:")) + + def test_no_phrase_names_a_pr_number_or_a_date(self): + for checker in CATEGORIES: + dictionary = _load(checker) + for phrase in [dictionary["meaning"], *dictionary["match"], *dictionary["not_match"]]: + for token in phrase.split(): + with self.subTest(checker=checker, token=token): + self.assertFalse(_is_pr_number(token)) + self.assertFalse(_is_date(token)) + + def test_no_phrase_sits_in_two_categories(self): + seen = {} + for checker in CATEGORIES: + dictionary = _load(checker) + for phrase in dictionary["match"] + dictionary["not_match"]: + self.assertNotIn(phrase, seen, f"{phrase!r} is in {seen.get(phrase)} and {checker}") + seen[phrase] = checker + + def test_a_pr_number_and_a_date_are_recognised(self): + self.assertTrue(_is_pr_number("#412.")) + self.assertTrue(_is_date("2026-09-11")) + self.assertFalse(_is_pr_number("#tag")) + self.assertFalse(_is_date("engine-runtime")) + + +if __name__ == "__main__": + unittest.main() diff --git a/engine/hooks/diu-stop/tests/test_plain_words_check.py b/engine/hooks/diu-stop/tests/test_plain_words_check.py new file mode 100644 index 0000000..67e61d1 --- /dev/null +++ b/engine/hooks/diu-stop/tests/test_plain_words_check.py @@ -0,0 +1,136 @@ +import io +import json +import os +import sys +import tempfile +import unittest +from contextlib import redirect_stderr +from unittest.mock import patch + +HOOK_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, HOOK_DIR) + +import plain_words + +REPLY = "No hook decides differently. Preflight passes and the review unit is engine-runtime." +ASKED = "Is it safe?" + + +def _transcript(folder, rows): + path = os.path.join(folder, "chat.jsonl") + with open(path, "w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row) + "\n") + return path + + +def _user(text): + return {"type": "user", "message": {"role": "user", "content": text}} + + +def _assistant(text): + return {"type": "assistant", "message": {"role": "assistant", "content": [{"type": "text", "text": text}]}} + + +def _hit(job_id, closest, category="plain-words-made-up-labels"): + return {"id": job_id, "hook": plain_words.HOOK_NAME, "outcome": "hit", "answer": {"match": True, "category": category, "closest": closest}} + + +class PlainWordsCase(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.tmp = self._tmp.name + self.judge, _ = plain_words._llm_judge() + + def tearDown(self): + self._tmp.cleanup() + + def payload(self, rows=(_user(ASKED), _assistant(REPLY)), **extra): + payload = {"last_assistant_message": REPLY} + if rows is not None: + payload["transcript_path"] = _transcript(self.tmp, rows) + payload.update(extra) + return payload + + +class TestJob(PlainWordsCase): + def test_one_job_names_every_word_list(self): + built = plain_words.job(self.payload()) + self.assertEqual(built["hook"], plain_words.HOOK_NAME) + self.assertEqual(built["hit_if_all_true"], ["match"]) + for name in plain_words.list_names(): + self.assertIn(name, built["prompt"]) + + def test_the_job_carries_the_user_message_and_the_reply(self): + built = plain_words.job(self.payload()) + self.assertIn(ASKED, built["prompt"]) + self.assertIn(REPLY, built["prompt"]) + self.assertIn("copied word for word from the ASSISTANT text", built["prompt"]) + + def test_no_job_without_a_transcript_file(self): + self.assertIsNone(plain_words.job({"last_assistant_message": REPLY})) + self.assertIsNone(plain_words.job({"last_assistant_message": REPLY, "transcript_path": "/no/such/file.jsonl"})) + + def test_no_job_when_the_reply_is_empty(self): + self.assertIsNone(plain_words.job(self.payload(last_assistant_message=" "))) + + +class TestMessage(PlainWordsCase): + def test_a_hit_quoting_the_reply_becomes_a_message(self): + text = plain_words.message_for(_hit("j1", "No hook decides differently."), REPLY) + self.assertIn("No hook decides differently.", text) + self.assertIn("plain-words-made-up-labels", text) + + def test_a_hit_quoting_the_word_list_is_dropped(self): + self.assertEqual(plain_words.message_for(_hit("j1", "Force-push with lease, then a three-way apply."), REPLY), "") + + def test_a_clean_or_unchecked_verdict_says_nothing(self): + for outcome in ("clean", "unchecked"): + with self.subTest(outcome=outcome): + self.assertEqual(plain_words.message_for({"id": "j1", "outcome": outcome}, REPLY), "") + + +class TestCheckReply(PlainWordsCase): + def test_the_hit_comes_back_in_the_same_turn(self): + payload = self.payload() + built = { + "id": "fixed", + "hook": plain_words.HOOK_NAME, + "transcript": payload["transcript_path"], + "prompt": "p", + "hit_if_all_true": ["match"], + "on_hit": "o", + } + with patch.object(plain_words, "job", return_value=built): + with patch.object(self.judge, "enqueue", return_value="fixed"): + with patch.object(self.judge, "drain", return_value=[_hit("fixed", "No hook decides differently.")]): + text = plain_words.check_reply(payload) + self.assertIn("No hook decides differently.", text) + self.assertIn("plain-words-made-up-labels", text) + + def test_nothing_is_asked_for_a_rewrite_or_a_subagent(self): + for extra in ({"stop_hook_active": True}, {"agent_id": "sub"}): + with self.subTest(extra=extra): + with patch.object(self.judge, "enqueue", side_effect=AssertionError("must not be called")): + self.assertEqual(plain_words.check_reply(self.payload(**extra)), "") + + def test_no_answer_in_time_ends_the_turn_unblocked(self): + os.environ[plain_words.WAIT_ENV] = "0" + try: + with patch.object(self.judge, "enqueue", side_effect=lambda job: job["id"]): + with patch.object(self.judge, "drain", return_value=[]): + self.assertEqual(plain_words.check_reply(self.payload()), "") + finally: + del os.environ[plain_words.WAIT_ENV] + + def test_a_failure_is_reported_and_not_raised(self): + err = io.StringIO() + with patch.object(self.judge, "enqueue", side_effect=OSError("disk full")): + with redirect_stderr(err): + self.assertEqual(plain_words.try_check_reply(self.payload()), "") + self.assertIn("plain-words check failed", err.getvalue()) + self.assertIn("disk full", err.getvalue()) + + +if __name__ == "__main__": + unittest.main()