From 28e96c4374279790046ae68d207ae76f3fd3ff99 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 3 Sep 2026 16:34:52 -0700 Subject: [PATCH 1/6] Reduce pull request benchmark matrix Signed-off-by: James Sturtevant --- .github/workflows/dep_rust.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 2f196845..d9c7b5b9 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -62,6 +62,7 @@ jobs: # On Linux, always pass the hypervisor feature (kvm/mshv3). # On Linux latest builds, also add wasmtime_latest. FEATURES: ${{ matrix.hypervisor != 'hyperv' && format('{0}{1}', matrix.hypervisor, matrix.wasmtime == 'latest' && ',wasmtime_latest' || '') || (matrix.wasmtime == 'latest' && 'wasmtime_latest' || '') }} + RUN_BENCHMARKS: ${{ matrix.config == 'release' && (matrix.wasmtime == 'lts' || (matrix.hypervisor == 'kvm' && matrix.cpu == 'amd')) }} steps: - uses: actions/checkout@v5 @@ -145,10 +146,10 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} continue-on-error: true working-directory: ./src/hyperlight_wasm - if: ${{ matrix.config == 'release' }} + if: ${{ env.RUN_BENCHMARKS == 'true' }} - name: Run benchmarks run: | just bench-ci dev ${{ matrix.config }} ${{ env.FEATURES }} working-directory: ./src/hyperlight_wasm - if: ${{ matrix.config == 'release' }} + if: ${{ env.RUN_BENCHMARKS == 'true' }} From 35f2bc25a95a4eb0d7d5dd8c06cd6a891dd2dc89 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 4 Sep 2026 09:56:03 -0700 Subject: [PATCH 2/6] Harden CI against runner disconnects Signed-off-by: James Sturtevant --- .github/workflows/Benchmarks.yml | 7 +++++-- .github/workflows/CreateRelease.yml | 2 +- .github/workflows/dep_rust.yml | 4 +++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Benchmarks.yml b/.github/workflows/Benchmarks.yml index 5d6d2f4a..b0e51948 100644 --- a/.github/workflows/Benchmarks.yml +++ b/.github/workflows/Benchmarks.yml @@ -7,7 +7,8 @@ on: permissions: contents: read - packages: read + # dep_build_wasm_examples.yml publishes the shared builder cache to GHCR. + packages: write # The reason for default shell bash is because on our self-hosted windows runners, # the default shell is powershell, which doesn't work correctly together with `just` commands. @@ -27,7 +28,9 @@ jobs: needs: - build-wasm-examples strategy: - fail-fast: true + # A lost self-hosted runner should not cancel benchmark artifacts from + # otherwise healthy matrix jobs. + fail-fast: false matrix: hypervisor: [hyperv, mshv3, kvm] # hyperv is windows, mshv and kvm are linux cpu: [amd, intel] diff --git a/.github/workflows/CreateRelease.yml b/.github/workflows/CreateRelease.yml index 69170030..4c78503e 100644 --- a/.github/workflows/CreateRelease.yml +++ b/.github/workflows/CreateRelease.yml @@ -17,7 +17,7 @@ jobs: publish: # see https://github.com/orgs/community/discussions/26286#discussioncomment-3251208 for why we need to check the ref - if: ${{ contains(github.ref, 'refs/heads/release/') }} || ${{ github.ref=='refs/heads/main' }} + if: ${{ contains(github.ref, 'refs/heads/release/') || github.ref == 'refs/heads/main' }} needs: [ benchmarks ] runs-on: ubuntu-latest env: diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index d9c7b5b9..b7ac2bbb 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -35,7 +35,9 @@ jobs: build: if: ${{ inputs.docs_only == 'false' }} strategy: - fail-fast: true + # Preserve completed shards when an ephemeral runner disconnects so a + # failed-job rerun does not need to repeat the entire matrix. + fail-fast: false matrix: hypervisor: [hyperv, mshv3, kvm] # hyperv is windows, mshv and kvm are linux cpu: [amd, intel] From 2075bb4a8bd8e437872fb2dcda2c2b5fd56e1541 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 4 Sep 2026 11:05:05 -0700 Subject: [PATCH 3/6] Isolate long-running CI workloads Signed-off-by: James Sturtevant --- .github/workflows/dep_rust.yml | 58 ++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index b7ac2bbb..8715a5b0 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -49,11 +49,12 @@ jobs: config: debug runs-on: ${{ fromJson( - format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}", "JobId=build-{3}-{4}-{5}-{6}"]', + format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}", "JobId=build-{3}-{4}-{5}-{6}-{7}"]', matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', matrix.hypervisor == 'hyperv' && 'win2025' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor, matrix.cpu, matrix.config, + matrix.wasmtime, github.run_id, github.run_number, github.run_attempt)) }} @@ -64,7 +65,7 @@ jobs: # On Linux, always pass the hypervisor feature (kvm/mshv3). # On Linux latest builds, also add wasmtime_latest. FEATURES: ${{ matrix.hypervisor != 'hyperv' && format('{0}{1}', matrix.hypervisor, matrix.wasmtime == 'latest' && ',wasmtime_latest' || '') || (matrix.wasmtime == 'latest' && 'wasmtime_latest' || '') }} - RUN_BENCHMARKS: ${{ matrix.config == 'release' && (matrix.wasmtime == 'lts' || (matrix.hypervisor == 'kvm' && matrix.cpu == 'amd')) }} + RUN_EXAMPLES: ${{ (matrix.config == 'debug' && matrix.wasmtime == 'lts') || (matrix.config == 'release' && matrix.wasmtime == 'latest' && matrix.hypervisor == 'kvm' && matrix.cpu == 'amd') }} steps: - uses: actions/checkout@v5 @@ -133,25 +134,64 @@ jobs: shell: pwsh - name: Test Examples + if: ${{ env.RUN_EXAMPLES == 'true' }} run: just examples-ci ${{ matrix.config }} ${{ env.FEATURES }} working-directory: ./src/hyperlight_wasm env: # required for gh cli when downloading GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - ### Benchmarks ### + + benchmark: + if: ${{ inputs.docs_only == 'false' }} + strategy: + fail-fast: false + matrix: + hypervisor: [hyperv, mshv3, kvm] + cpu: [amd, intel] + wasmtime: [latest, lts] + exclude: + - wasmtime: latest + hypervisor: hyperv + - wasmtime: latest + hypervisor: mshv3 + - wasmtime: latest + cpu: intel + + runs-on: ${{ fromJson( + format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}", "JobId=benchmark-{3}-{4}-{5}-{6}"]', + matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', + matrix.hypervisor == 'hyperv' && 'win2025' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor, + matrix.cpu, + matrix.wasmtime, + github.run_id, + github.run_number, + github.run_attempt)) }} + env: + FEATURES: ${{ matrix.hypervisor != 'hyperv' && format('{0}{1}', matrix.hypervisor, matrix.wasmtime == 'latest' && ',wasmtime_latest' || '') || (matrix.wasmtime == 'latest' && 'wasmtime_latest' || '') }} + steps: + - uses: actions/checkout@v5 + + - name: Hyperlight setup + uses: hyperlight-dev/ci-setup-workflow@v1.8.0 + with: + rust-toolchain: "1.94" + + - name: Download Wasm Modules + uses: actions/download-artifact@v5 + with: + name: guest-modules-${{ matrix.wasmtime }} + path: ./x64/release - name: Download benchmarks from "latest" - run: | - just bench-download ${{ runner.os }} ${{ matrix.hypervisor }} ${{ matrix.cpu }} dev-latest + run: just bench-download ${{ runner.os }} ${{ matrix.hypervisor }} ${{ matrix.cpu }} dev-latest env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} continue-on-error: true working-directory: ./src/hyperlight_wasm - if: ${{ env.RUN_BENCHMARKS == 'true' }} - name: Run benchmarks run: | - just bench-ci dev ${{ matrix.config }} ${{ env.FEATURES }} + just ensure-tools + just compile-wit + just bench-ci dev release ${{ env.FEATURES }} working-directory: ./src/hyperlight_wasm - if: ${{ env.RUN_BENCHMARKS == 'true' }} From 98f00ada017ff7f0bae776e1ed0cd68a54de994e Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 4 Sep 2026 12:44:58 -0700 Subject: [PATCH 4/6] Keep full example matrix coverage Signed-off-by: James Sturtevant --- .github/workflows/CreateRelease.yml | 2 +- .github/workflows/dep_rust.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/CreateRelease.yml b/.github/workflows/CreateRelease.yml index 4c78503e..69170030 100644 --- a/.github/workflows/CreateRelease.yml +++ b/.github/workflows/CreateRelease.yml @@ -17,7 +17,7 @@ jobs: publish: # see https://github.com/orgs/community/discussions/26286#discussioncomment-3251208 for why we need to check the ref - if: ${{ contains(github.ref, 'refs/heads/release/') || github.ref == 'refs/heads/main' }} + if: ${{ contains(github.ref, 'refs/heads/release/') }} || ${{ github.ref=='refs/heads/main' }} needs: [ benchmarks ] runs-on: ubuntu-latest env: diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 8715a5b0..c2419cc8 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -65,7 +65,6 @@ jobs: # On Linux, always pass the hypervisor feature (kvm/mshv3). # On Linux latest builds, also add wasmtime_latest. FEATURES: ${{ matrix.hypervisor != 'hyperv' && format('{0}{1}', matrix.hypervisor, matrix.wasmtime == 'latest' && ',wasmtime_latest' || '') || (matrix.wasmtime == 'latest' && 'wasmtime_latest' || '') }} - RUN_EXAMPLES: ${{ (matrix.config == 'debug' && matrix.wasmtime == 'lts') || (matrix.config == 'release' && matrix.wasmtime == 'latest' && matrix.hypervisor == 'kvm' && matrix.cpu == 'amd') }} steps: - uses: actions/checkout@v5 @@ -134,7 +133,6 @@ jobs: shell: pwsh - name: Test Examples - if: ${{ env.RUN_EXAMPLES == 'true' }} run: just examples-ci ${{ matrix.config }} ${{ env.FEATURES }} working-directory: ./src/hyperlight_wasm env: From c776a4d7cf5c8b0ac09aff61df60e238eec71199 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 4 Sep 2026 12:55:29 -0700 Subject: [PATCH 5/6] Keep benchmark permissions unchanged Signed-off-by: James Sturtevant --- .github/workflows/Benchmarks.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/Benchmarks.yml b/.github/workflows/Benchmarks.yml index b0e51948..7149163b 100644 --- a/.github/workflows/Benchmarks.yml +++ b/.github/workflows/Benchmarks.yml @@ -7,8 +7,7 @@ on: permissions: contents: read - # dep_build_wasm_examples.yml publishes the shared builder cache to GHCR. - packages: write + packages: read # The reason for default shell bash is because on our self-hosted windows runners, # the default shell is powershell, which doesn't work correctly together with `just` commands. From 7fd7489b65e0651972a8f91766434ef16bdecf43 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 4 Sep 2026 13:14:05 -0700 Subject: [PATCH 6/6] Clarify benchmark matrix selection Signed-off-by: James Sturtevant --- .github/workflows/dep_rust.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index c2419cc8..98bc3db5 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -146,14 +146,11 @@ jobs: matrix: hypervisor: [hyperv, mshv3, kvm] cpu: [amd, intel] - wasmtime: [latest, lts] - exclude: - - wasmtime: latest - hypervisor: hyperv - - wasmtime: latest - hypervisor: mshv3 - - wasmtime: latest - cpu: intel + wasmtime: [lts] + include: + - hypervisor: kvm + cpu: amd + wasmtime: latest runs-on: ${{ fromJson( format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}", "JobId=benchmark-{3}-{4}-{5}-{6}"]',