From 4ffa3f1e8a0cfa2cca71d55c4b529fce43fc53ab Mon Sep 17 00:00:00 2001 From: John Menke Date: Fri, 11 Sep 2026 16:06:45 -0400 Subject: [PATCH] Put a harder-case job on the MCP apply path so unproven local cannot apply. Co-authored-by: Cursor --- docs/evaluation-protocol.md | 3 +++ src/local_coding_slm/eval/jobs.py | 12 ++++++++++++ tests/test_eval_mcp_orchestrate.py | 25 +++++++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/docs/evaluation-protocol.md b/docs/evaluation-protocol.md index b9531cb..6a20b04 100644 --- a/docs/evaluation-protocol.md +++ b/docs/evaluation-protocol.md @@ -169,6 +169,9 @@ The same gate runs after **real stdio MCP** calls when you pass never call `local_*`. Accept / rewrite / reject then run on the scored candidate. Security-sensitive delegated jobs first call `local_review` and attach those notes to the premium packet; that still cannot approve. +`MCP_JOBS` includes `rename_exception_across_files` so the apply gate +sees a harder multi-file case; premium accept of an unproven local +result is blocked (`accept_unproven_local`). The MCP server also refuses secret filenames, private-key / token blobs, oversized file sets, and `max_tokens` above 4096 **before** diff --git a/src/local_coding_slm/eval/jobs.py b/src/local_coding_slm/eval/jobs.py index 5a29c77..679eae4 100644 --- a/src/local_coding_slm/eval/jobs.py +++ b/src/local_coding_slm/eval/jobs.py @@ -3,6 +3,7 @@ from __future__ import annotations from local_coding_slm.eval.cases import CASES_BY_ID +from local_coding_slm.eval.cases_harder import RENAME_EXCEPTION_PARTIAL from local_coding_slm.eval.orchestrate import OrchestrationJob from local_coding_slm.eval.review import accept, reject, rewrite from local_coding_slm.eval.routing import RouteSignals, mechanical_signals @@ -96,6 +97,17 @@ def clamp(value: int, lo: int, hi: int) -> int: eval_case=CASES_BY_ID["implement_clamp"], review=reject(notes="do not apply until bounds are reviewed"), ), + OrchestrationJob( + id="mcp_rename_exception_unproven", + signals=mechanical_signals(), + eval_case=CASES_BY_ID["rename_exception_across_files"], + local_replies=( + RENAME_EXCEPTION_PARTIAL, + RENAME_EXCEPTION_PARTIAL, + RENAME_EXCEPTION_PARTIAL, + ), + review=accept(notes="do not apply unproven harder local"), + ), ) diff --git a/tests/test_eval_mcp_orchestrate.py b/tests/test_eval_mcp_orchestrate.py index c95c1a1..0fafefa 100644 --- a/tests/test_eval_mcp_orchestrate.py +++ b/tests/test_eval_mcp_orchestrate.py @@ -4,8 +4,10 @@ import unittest +from local_coding_slm.eval.cases_harder import HARDER_CASE_IDS from local_coding_slm.eval.harness import run_campaign, run_orchestrated_campaign from local_coding_slm.eval.jobs import MCP_JOBS +from local_coding_slm.eval.orchestrate import run_job from local_coding_slm.eval.record import summarize @@ -108,6 +110,29 @@ def test_job_table_covers_keep_and_delegate(self) -> None: self.assertTrue(any(job.eval_case is None for job in MCP_JOBS)) self.assertTrue(any(job.eval_case is not None for job in MCP_JOBS)) + def test_mcp_jobs_intersect_harder_cases(self) -> None: + case_ids = { + job.eval_case.id for job in MCP_JOBS if job.eval_case is not None + } + overlap = case_ids & set(HARDER_CASE_IDS) + self.assertIn("rename_exception_across_files", overlap) + self.assertTrue(overlap) + + def test_harder_rename_exception_job_blocks_accept_unproven_local(self) -> None: + job = next( + item + for item in MCP_JOBS + if item.eval_case is not None + and item.eval_case.id == "rename_exception_across_files" + ) + result = run_job(job) + self.assertEqual(result.case_id, "rename_exception_across_files") + self.assertTrue(result.delegated) + self.assertFalse(result.local_passed) + self.assertFalse(result.applied) + self.assertEqual(result.outcome, "blocked") + self.assertEqual(result.blocked, "accept_unproven_local") + if __name__ == "__main__": unittest.main()