test: prevent false-positive Java test runner results

test-fixtures/maven-junit/src/test/java/com/example/CalculatorTest.java

@@ -1,13 +1,17 @@
 package com.example;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class CalculatorTest {
 
     @Test
-    public void testAdd() {
+    public void testAdd() throws IOException {
         Calculator c = new Calculator();
         assertEquals(5, c.add(2, 3));
+        Files.writeString(Path.of("target", "autotest-executed.txt"), "testAdd passed");
     }
 }

test-plans/java-debugger.yaml

@@ -56,9 +56,15 @@ steps:
 
   # ── Set breakpoint ───────────────────────────────────────
   # App.java line 5: System.out.println("Hello Java");
+  # Do not use LLM screenshot verification for the gutter dot: breakpoint
+  # decorations can render too late on macOS. The later debugStepOver action
+  # is authoritative — without a working breakpoint this one-line program
+  # exits during the settle wait and debugStepOver fails with no active
+  # debug session.
   - id: "set-breakpoint"
     action: "setBreakpoint 5"
-    verify: "Red breakpoint dot is shown in the gutter of App.java line 5"
+    verify: "Debug: Toggle Breakpoint was dispatched for App.java line 5"
+    skipLlmVerify: true
 
   # ── Start debug session ─────────────────────────────────
   - id: "start-debug"
@@ -67,17 +73,12 @@ steps:
     timeout: 30
 
   # ── Verify breakpoint hit ───────────────────────────────
-  # wiki: "verify if the breakpoint is hit". The deterministic ground
-  # truth is the next step `debugStepOver` — it can only succeed if the
-  # debugger is paused. The verify text is intentionally lenient: the
-  # yellow execution-line marker can be off-viewport when the debug
-  # toolbar pushes the editor down, so we accept either the marker or
-  # the debug toolbar in paused state as evidence.
-  - id: "verify-breakpoint"
+  # Allow the program to reach the breakpoint. The next debugStepOver action
+  # deterministically fails if the debug session already exited.
+  - id: "wait-for-breakpoint"
     action: "wait 10 seconds"
-    verify: "Program is paused at the breakpoint — debug toolbar visible in paused state or the yellow execution-line marker appears on/near App.java line 5"
 
-  # ── Continue execution ──────────────────────────────────
+  # ── Verify breakpoint and continue execution ────────────
   - id: "continue-debug"
     action: "debugStepOver"
     verify: "Program has stepped one statement and remains paused (debug toolbar still in paused state)"

test-plans/java-test-navigation.yaml

@@ -0,0 +1,86 @@
+# Test Plan: Java Test Navigation
+#
+# Regression coverage for Java Test Runner navigation between a production
+# class and its corresponding test class.
+#
+# Verifies: LS ready -> Go to Test opens CalculatorTest.java ->
+# Go to Test Subject returns to Calculator.java.
+#
+# Prerequisites:
+#   - JDK 11+ installed and available
+#
+# Usage: autotest run test-plans/java-test-navigation.yaml
+
+name: "Java Test Runner - Navigate Between Test and Subject"
+description: |
+  Verifies that Java Test Runner can discover the corresponding test and
+  production classes and navigate between them using command palette actions.
+
+setup:
+  extension: "redhat.java"
+  extensions:
+    - "vscjava.vscode-java-pack"
+  preRelease: true
+  vscodeVersion: "stable"
+  workspace: "../test-fixtures/maven-junit"
+  timeout: 360
+  workspaceSettings:
+    java.configuration.updateBuildConfiguration: "automatic"
+    java.import.maven.enabled: true
+
+steps:
+  - id: "ls-ready"
+    action: "waitForLanguageServer"
+    verify: "The maven-junit workspace has loaded and the Java language server is ready"
+    timeout: 300
+    skipLlmVerify: true
+
+  - id: "open-subject"
+    action: "open file Calculator.java"
+    verify: "Calculator.java is open in the editor"
+    verifyEditor:
+      fileName: "Calculator.java"
+      contains: "public class Calculator"
+    timeout: 15
+    waitBefore: 5
+    skipLlmVerify: true
+
+  # Test discovery and Java search indexing continue asynchronously after the
+  # language server reports ready, especially on cold CI runners.
+  - id: "wait-for-indexing"
+    action: "wait 90 seconds"
+
+  - id: "go-to-test"
+    action: "run command Java: Go to Test"
+    verify: "The Java: Go to Test command was selected and dispatched from the command palette"
+    timeout: 30
+    waitBefore: 3
+    skipLlmVerify: true
+
+  # Command Palette dismissal only proves that VS Code dispatched the command.
+  # Navigation completes asynchronously on macOS, so verify the resulting
+  # editor state in a separate settle step.
+  - id: "verify-test-open"
+    action: "wait 15 seconds"
+    verify: "CalculatorTest.java is the active editor after Java: Go to Test"
+    verifyEditor:
+      fileName: "CalculatorTest.java"
+      contains: "public class CalculatorTest"
+    timeout: 30
+    skipLlmVerify: true
+
+  - id: "go-to-test-subject"
+    action: "run command Java: Go to Test Subject"
+    verify: "The Java: Go to Test Subject command was selected and dispatched from the command palette"
+    timeout: 30
+    waitBefore: 3
+    skipLlmVerify: true
+
+  - id: "verify-subject-open"
+    action: "wait 15 seconds"
+    verify: "Calculator.java is the active editor after Java: Go to Test Subject"
+    verifyEditor:
+      fileName: "Calculator.java"
+      contains: "public class Calculator"
+    timeout: 30
+    skipLlmVerify: true

test-plans/java-test-runner.yaml

@@ -1,7 +1,7 @@
 # Test Plan: Java Test Runner (from vscode-java-pack.wiki)
 #
 # Source: wiki Test-Plan.md "Java Test Runner" scenario
-# Verify: Test panel display → run all tests → CodeLens visible
+# Verify: test discovery → run tests → execution marker
 #
 # Uses test-fixtures/maven-junit — a self-contained Maven + JUnit 5 fixture
 # owned by this repo. Upstream `vscode-java/maven/salut` has no @Test files,
@@ -13,10 +13,10 @@
 #
 # Usage: autotest run test-plans/java-test-runner.yaml
 
-name: "Java Test Runner — Test Panel and CodeLens"
+name: "Java Test Runner — Discovery and Execution"
 description: |
   Corresponds to the Java Test Runner scenario in the wiki Test Plan:
-  Verify that the test panel displays test cases and tests can be run via the panel or CodeLens.
+  Verify that Java tests are discovered and actually executed.
 
 setup:
   extension: "redhat.java"
@@ -25,12 +25,6 @@ setup:
   vscodeVersion: "stable"
   workspace: "../test-fixtures/maven-junit"
   timeout: 360  # First import needs to download JUnit jars on cold caches
-  # Force java.test.editor.enableCodelens=true so the Run|Debug CodeLens
-  # gutter links render reliably in the reopen-test-file verify screenshot.
-  # The vscode-java-test extension defaults to true but a stale user/Machine
-  # settings.json on the runner can override it.
-  workspaceSettings:
-    java.test.editor.enableCodelens: true
 
 steps:
   # ── Wait for LS ready ─────────────────────────────────────────
@@ -45,10 +39,19 @@ steps:
     action: "open file CalculatorTest.java"
     verify: "CalculatorTest.java is open in the editor and shows a JUnit @Test method"
     verifyEditor:
+      fileName: "CalculatorTest.java"
       contains: "@Test"
     timeout: 15
     waitBefore: 5
 
+  - id: "clear-execution-marker"
+    action: "deleteFile target/autotest-executed.txt"
+    verify: "Any execution marker from an earlier test run has been removed"
+    verifyFile:
+      path: "~/target/autotest-executed.txt"
+      exists: false
+    skipLlmVerify: true
+
   # Give the Java Test Runner extension time to scan the project after LS
   # ready — discovery is asynchronous and Test Explorer is initially empty.
   # On cold-cache CI runners 20s is sometimes too short; bump to 45s.
@@ -63,39 +66,19 @@ steps:
   # and runs every test in the project from any context (matches the wiki
   # scenario "Run all tests").
   #
-  # The verify text is intentionally agnostic to whether the run produced a
-  # results panel, an inline "No tests found" hint, or simply dismissed the
-  # palette — on a cold-cache runner the Java Test Runner's discovery may
-  # still be in flight when the screenshot is captured. The wait-test-complete
-  # step below provides the deterministic settle window before any further
-  # assertion is made.
   - id: "run-all-tests"
     action: "run command Java: Run Tests"
-    verify: "Java: Run Tests command has been invoked from the palette; the Java Test Runner extension has responded (this may show as a Testing view becoming active, a run indicator in the status bar, or an informational notification such as 'No tests found in this file' if discovery is still in progress — all of these indicate the command executed successfully)"
+    verify: "Java: Run Tests was selected and dispatched from the command palette"
     waitBefore: 3
-    # Test discovery is asynchronous in vscode-java-test; on cold-cache
-    # CI runners the first Run Tests invocation can land before discovery
-    # completes ("No tests have been found"). Allow one retry — by the
-    # second attempt the discovery cache is usually warm.
-    retries: 1
+    skipLlmVerify: true
 
-  - id: "wait-test-complete"
+  # CalculatorTest writes this marker only after its assertion passes. Missing
+  # discovery, "No tests found", launch failures, and failed tests therefore
+  # cannot pass this deterministic verification.
+  - id: "verify-run-all-executed"
     action: "wait 45 seconds"
-    verify: "Test discovery / execution has settled after the wait; the editor still shows CalculatorTest.java with the @Test method"
-
-  # ── Step 3: Re-open test file ──────────
-  # The @Test annotation in the editor's text content is the deterministic
-  # ground truth that the test file is loaded and visible. CodeLens
-  # (Run|Debug) gutter links require the vscode-java-test extension to
-  # have completed its discovery scan; on a cold runner with newly-
-  # downloaded JUnit jars they may not be rendered at screenshot time.
-  # Keep verify text strictly about visible editor content so the LLM
-  # doesn't downgrade on the CodeLens absence.
-  - id: "reopen-test-file"
-    action: "open file CalculatorTest.java"
-    verify: "CalculatorTest.java is re-opened in the editor; the file's content is shown including the @Test-annotated method"
-    verifyEditor:
-      contains: "@Test"
-    timeout: 10
-    waitBefore: 5
-
+    verify: "CalculatorTest.testAdd completed successfully and wrote its execution marker"
+    verifyFile:
+      path: "~/target/autotest-executed.txt"
+      contains: "testAdd passed"
+    skipLlmVerify: true