From 502ee3ddcb9d52a0953de66f91470c5851877963 Mon Sep 17 00:00:00 2001 From: wenytang-ms Date: Wed, 9 Sep 2026 11:30:51 +0800 Subject: [PATCH 1/5] test: add Java test navigation autotest plan --- test-plans/java-test-navigation.yaml | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 test-plans/java-test-navigation.yaml diff --git a/test-plans/java-test-navigation.yaml b/test-plans/java-test-navigation.yaml new file mode 100644 index 00000000..0bea5d0e --- /dev/null +++ b/test-plans/java-test-navigation.yaml @@ -0,0 +1,71 @@ +# Test Plan: Java Test Navigation +# +# Regression coverage for Java Test Runner navigation between a production +# class and its corresponding test class. +# +# Verifies: LS ready -> Go to Test opens CalculatorTest.java -> +# Go to Test Subject returns to Calculator.java. +# +# Prerequisites: +# - JDK 11+ installed and available +# +# Usage: autotest run test-plans/java-test-navigation.yaml + +name: "Java Test Runner - Navigate Between Test and Subject" +description: | + Verifies that Java Test Runner can discover the corresponding test and + production classes and navigate between them using command palette actions. + +setup: + extension: "redhat.java" + extensions: + - "vscjava.vscode-java-pack" + preRelease: true + vscodeVersion: "stable" + workspace: "../test-fixtures/maven-junit" + timeout: 360 + workspaceSettings: + java.configuration.updateBuildConfiguration: "automatic" + java.import.maven.enabled: true + +steps: + - id: "ls-ready" + action: "waitForLanguageServer" + verify: "The maven-junit workspace has loaded and the Java language server is ready" + timeout: 300 + skipLlmVerify: true + + - id: "open-subject" + action: "open file Calculator.java" + verify: "Calculator.java is open in the editor" + verifyEditor: + fileName: "Calculator.java" + contains: "public class Calculator" + timeout: 15 + waitBefore: 5 + skipLlmVerify: true + + # Test discovery and Java search indexing continue asynchronously after the + # language server reports ready, especially on cold CI runners. + - id: "wait-for-indexing" + action: "wait 90 seconds" + + - id: "go-to-test" + action: "run command Java: Go to Test" + verify: "The editor navigated from Calculator.java to CalculatorTest.java" + verifyEditor: + fileName: "CalculatorTest.java" + contains: "public class CalculatorTest" + timeout: 30 + waitBefore: 3 + skipLlmVerify: true + + - id: "go-to-test-subject" + action: "run command Java: Go to Test Subject" + verify: "The editor navigated from CalculatorTest.java back to Calculator.java" + verifyEditor: + fileName: "Calculator.java" + contains: "public class Calculator" + timeout: 30 + waitBefore: 3 + skipLlmVerify: true From c0df2dd170502c82c3fdb8b71aae2e890d0006e1 Mon Sep 17 00:00:00 2001 From: wenytang-ms Date: Wed, 9 Sep 2026 13:14:45 +0800 Subject: [PATCH 2/5] test: wait for Java navigation completion --- test-plans/java-test-navigation.yaml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/test-plans/java-test-navigation.yaml b/test-plans/java-test-navigation.yaml index 0bea5d0e..751e9204 100644 --- a/test-plans/java-test-navigation.yaml +++ b/test-plans/java-test-navigation.yaml @@ -52,20 +52,35 @@ steps: - id: "go-to-test" action: "run command Java: Go to Test" - verify: "The editor navigated from Calculator.java to CalculatorTest.java" + verify: "The Java: Go to Test command was selected and dispatched from the command palette" + timeout: 30 + waitBefore: 3 + skipLlmVerify: true + + # Command Palette dismissal only proves that VS Code dispatched the command. + # Navigation completes asynchronously on macOS, so verify the resulting + # editor state in a separate settle step. + - id: "verify-test-open" + action: "wait 15 seconds" + verify: "CalculatorTest.java is the active editor after Java: Go to Test" verifyEditor: fileName: "CalculatorTest.java" contains: "public class CalculatorTest" timeout: 30 - waitBefore: 3 skipLlmVerify: true - id: "go-to-test-subject" action: "run command Java: Go to Test Subject" - verify: "The editor navigated from CalculatorTest.java back to Calculator.java" + verify: "The Java: Go to Test Subject command was selected and dispatched from the command palette" + timeout: 30 + waitBefore: 3 + skipLlmVerify: true + + - id: "verify-subject-open" + action: "wait 15 seconds" + verify: "Calculator.java is the active editor after Java: Go to Test Subject" verifyEditor: fileName: "Calculator.java" contains: "public class Calculator" timeout: 30 - waitBefore: 3 skipLlmVerify: true From 07d3a09d9285ad77b63738f5e8fe7dab2e2a29fc Mon Sep 17 00:00:00 2001 From: wenytang-ms Date: Wed, 9 Sep 2026 13:27:06 +0800 Subject: [PATCH 3/5] test: require proof of Java test execution --- .../test/java/com/example/CalculatorTest.java | 6 +- test-plans/java-test-runner.yaml | 70 ++++++++++++------- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/test-fixtures/maven-junit/src/test/java/com/example/CalculatorTest.java b/test-fixtures/maven-junit/src/test/java/com/example/CalculatorTest.java index 2157d887..136d916a 100644 --- a/test-fixtures/maven-junit/src/test/java/com/example/CalculatorTest.java +++ b/test-fixtures/maven-junit/src/test/java/com/example/CalculatorTest.java @@ -1,13 +1,17 @@ package com.example; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; public class CalculatorTest { @Test - public void testAdd() { + public void testAdd() throws IOException { Calculator c = new Calculator(); assertEquals(5, c.add(2, 3)); + Files.writeString(Path.of("target", "autotest-executed.txt"), "testAdd passed"); } } diff --git a/test-plans/java-test-runner.yaml b/test-plans/java-test-runner.yaml index ae12f83f..a8bb64c7 100644 --- a/test-plans/java-test-runner.yaml +++ b/test-plans/java-test-runner.yaml @@ -1,7 +1,7 @@ # Test Plan: Java Test Runner (from vscode-java-pack.wiki) # # Source: wiki Test-Plan.md "Java Test Runner" scenario -# Verify: Test panel display → run all tests → CodeLens visible +# Verify: test discovery → run all tests → execution marker → CodeLens run # # Uses test-fixtures/maven-junit — a self-contained Maven + JUnit 5 fixture # owned by this repo. Upstream `vscode-java/maven/salut` has no @Test files, @@ -45,10 +45,19 @@ steps: action: "open file CalculatorTest.java" verify: "CalculatorTest.java is open in the editor and shows a JUnit @Test method" verifyEditor: + fileName: "CalculatorTest.java" contains: "@Test" timeout: 15 waitBefore: 5 + - id: "clear-execution-marker" + action: "deleteFile target/autotest-executed.txt" + verify: "Any execution marker from an earlier test run has been removed" + verifyFile: + path: "~/target/autotest-executed.txt" + exists: false + skipLlmVerify: true + # Give the Java Test Runner extension time to scan the project after LS # ready — discovery is asynchronous and Test Explorer is initially empty. # On cold-cache CI runners 20s is sometimes too short; bump to 45s. @@ -63,39 +72,52 @@ steps: # and runs every test in the project from any context (matches the wiki # scenario "Run all tests"). # - # The verify text is intentionally agnostic to whether the run produced a - # results panel, an inline "No tests found" hint, or simply dismissed the - # palette — on a cold-cache runner the Java Test Runner's discovery may - # still be in flight when the screenshot is captured. The wait-test-complete - # step below provides the deterministic settle window before any further - # assertion is made. - id: "run-all-tests" action: "run command Java: Run Tests" - verify: "Java: Run Tests command has been invoked from the palette; the Java Test Runner extension has responded (this may show as a Testing view becoming active, a run indicator in the status bar, or an informational notification such as 'No tests found in this file' if discovery is still in progress — all of these indicate the command executed successfully)" + verify: "Java: Run Tests was selected and dispatched from the command palette" waitBefore: 3 - # Test discovery is asynchronous in vscode-java-test; on cold-cache - # CI runners the first Run Tests invocation can land before discovery - # completes ("No tests have been found"). Allow one retry — by the - # second attempt the discovery cache is usually warm. - retries: 1 + skipLlmVerify: true - - id: "wait-test-complete" + # CalculatorTest writes this marker only after its assertion passes. Missing + # discovery, "No tests found", launch failures, and failed tests therefore + # cannot pass this deterministic verification. + - id: "verify-run-all-executed" action: "wait 45 seconds" - verify: "Test discovery / execution has settled after the wait; the editor still shows CalculatorTest.java with the @Test method" + verify: "CalculatorTest.testAdd completed successfully and wrote its execution marker" + verifyFile: + path: "~/target/autotest-executed.txt" + contains: "testAdd passed" + skipLlmVerify: true + + # ── Step 3: Verify CodeLens discovery and execution ────────── + - id: "clear-codelens-marker" + action: "deleteFile target/autotest-executed.txt" + verify: "The execution marker has been removed before the CodeLens run" + verifyFile: + path: "~/target/autotest-executed.txt" + exists: false + skipLlmVerify: true - # ── Step 3: Re-open test file ────────── - # The @Test annotation in the editor's text content is the deterministic - # ground truth that the test file is loaded and visible. CodeLens - # (Run|Debug) gutter links require the vscode-java-test extension to - # have completed its discovery scan; on a cold runner with newly- - # downloaded JUnit jars they may not be rendered at screenshot time. - # Keep verify text strictly about visible editor content so the LLM - # doesn't downgrade on the CodeLens absence. - id: "reopen-test-file" action: "open file CalculatorTest.java" - verify: "CalculatorTest.java is re-opened in the editor; the file's content is shown including the @Test-annotated method" + verify: "CalculatorTest.java is the active editor before invoking its Run CodeLens" verifyEditor: + fileName: "CalculatorTest.java" contains: "@Test" timeout: 10 waitBefore: 5 + skipLlmVerify: true + - id: "run-via-codelens" + action: "clickCodeLens Run" + verify: "The visible Run CodeLens was clicked" + timeout: 30 + skipLlmVerify: true + + - id: "verify-codelens-executed" + action: "wait 45 seconds" + verify: "The CodeLens test run completed successfully and recreated the execution marker" + verifyFile: + path: "~/target/autotest-executed.txt" + contains: "testAdd passed" + skipLlmVerify: true From 44d3d3967a3af8658bc0b9fa921ae904e3b09e56 Mon Sep 17 00:00:00 2001 From: wenytang-ms Date: Wed, 9 Sep 2026 13:36:28 +0800 Subject: [PATCH 4/5] test: focus runner plan on executed tests --- test-plans/java-test-runner.yaml | 45 +++----------------------------- 1 file changed, 3 insertions(+), 42 deletions(-) diff --git a/test-plans/java-test-runner.yaml b/test-plans/java-test-runner.yaml index a8bb64c7..86e21a9f 100644 --- a/test-plans/java-test-runner.yaml +++ b/test-plans/java-test-runner.yaml @@ -1,7 +1,7 @@ # Test Plan: Java Test Runner (from vscode-java-pack.wiki) # # Source: wiki Test-Plan.md "Java Test Runner" scenario -# Verify: test discovery → run all tests → execution marker → CodeLens run +# Verify: test discovery → run tests → execution marker # # Uses test-fixtures/maven-junit — a self-contained Maven + JUnit 5 fixture # owned by this repo. Upstream `vscode-java/maven/salut` has no @Test files, @@ -13,10 +13,10 @@ # # Usage: autotest run test-plans/java-test-runner.yaml -name: "Java Test Runner — Test Panel and CodeLens" +name: "Java Test Runner — Discovery and Execution" description: | Corresponds to the Java Test Runner scenario in the wiki Test Plan: - Verify that the test panel displays test cases and tests can be run via the panel or CodeLens. + Verify that Java tests are discovered and actually executed. setup: extension: "redhat.java" @@ -25,12 +25,6 @@ setup: vscodeVersion: "stable" workspace: "../test-fixtures/maven-junit" timeout: 360 # First import needs to download JUnit jars on cold caches - # Force java.test.editor.enableCodelens=true so the Run|Debug CodeLens - # gutter links render reliably in the reopen-test-file verify screenshot. - # The vscode-java-test extension defaults to true but a stale user/Machine - # settings.json on the runner can override it. - workspaceSettings: - java.test.editor.enableCodelens: true steps: # ── Wait for LS ready ───────────────────────────────────────── @@ -88,36 +82,3 @@ steps: path: "~/target/autotest-executed.txt" contains: "testAdd passed" skipLlmVerify: true - - # ── Step 3: Verify CodeLens discovery and execution ────────── - - id: "clear-codelens-marker" - action: "deleteFile target/autotest-executed.txt" - verify: "The execution marker has been removed before the CodeLens run" - verifyFile: - path: "~/target/autotest-executed.txt" - exists: false - skipLlmVerify: true - - - id: "reopen-test-file" - action: "open file CalculatorTest.java" - verify: "CalculatorTest.java is the active editor before invoking its Run CodeLens" - verifyEditor: - fileName: "CalculatorTest.java" - contains: "@Test" - timeout: 10 - waitBefore: 5 - skipLlmVerify: true - - - id: "run-via-codelens" - action: "clickCodeLens Run" - verify: "The visible Run CodeLens was clicked" - timeout: 30 - skipLlmVerify: true - - - id: "verify-codelens-executed" - action: "wait 45 seconds" - verify: "The CodeLens test run completed successfully and recreated the execution marker" - verifyFile: - path: "~/target/autotest-executed.txt" - contains: "testAdd passed" - skipLlmVerify: true From 6e98b3d6120a98b92d9efc662a2b214d64178548 Mon Sep 17 00:00:00 2001 From: wenytang-ms Date: Wed, 9 Sep 2026 14:38:56 +0800 Subject: [PATCH 5/5] test: avoid flaky breakpoint screenshot check --- test-plans/java-debugger.yaml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/test-plans/java-debugger.yaml b/test-plans/java-debugger.yaml index 92e241b2..3ef70aff 100644 --- a/test-plans/java-debugger.yaml +++ b/test-plans/java-debugger.yaml @@ -56,9 +56,15 @@ steps: # ── Set breakpoint ─────────────────────────────────────── # App.java line 5: System.out.println("Hello Java"); + # Do not use LLM screenshot verification for the gutter dot: breakpoint + # decorations can render too late on macOS. The later debugStepOver action + # is authoritative — without a working breakpoint this one-line program + # exits during the settle wait and debugStepOver fails with no active + # debug session. - id: "set-breakpoint" action: "setBreakpoint 5" - verify: "Red breakpoint dot is shown in the gutter of App.java line 5" + verify: "Debug: Toggle Breakpoint was dispatched for App.java line 5" + skipLlmVerify: true # ── Start debug session ───────────────────────────────── - id: "start-debug" @@ -67,17 +73,12 @@ steps: timeout: 30 # ── Verify breakpoint hit ─────────────────────────────── - # wiki: "verify if the breakpoint is hit". The deterministic ground - # truth is the next step `debugStepOver` — it can only succeed if the - # debugger is paused. The verify text is intentionally lenient: the - # yellow execution-line marker can be off-viewport when the debug - # toolbar pushes the editor down, so we accept either the marker or - # the debug toolbar in paused state as evidence. - - id: "verify-breakpoint" + # Allow the program to reach the breakpoint. The next debugStepOver action + # deterministically fails if the debug session already exited. + - id: "wait-for-breakpoint" action: "wait 10 seconds" - verify: "Program is paused at the breakpoint — debug toolbar visible in paused state or the yellow execution-line marker appears on/near App.java line 5" - # ── Continue execution ────────────────────────────────── + # ── Verify breakpoint and continue execution ──────────── - id: "continue-debug" action: "debugStepOver" verify: "Program has stepped one statement and remains paused (debug toolbar still in paused state)"