ci: balance runner pools across vendors and persist fuzz corpus

.github/workflows/CargoPublish.yml

@@ -22,7 +22,12 @@ permissions:
 
 jobs:
   publish-hyperlight-packages:
-    runs-on: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd", "JobId=publish-hyperlight-packages-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"]
+    # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise.
+    runs-on: ${{ fromJSON(format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=publish-hyperlight-packages-{1}-{2}-{3}"]',
+        (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel',
+        github.run_id,
+        github.run_number,
+        github.run_attempt)) }}
 
     if: ${{ startsWith(github.ref, 'refs/heads/release/v') || inputs.dry_run }}
 

.github/workflows/Coverage.yml

@@ -29,7 +29,8 @@ jobs:
       fail-fast: false
       matrix:
         hypervisor: [kvm]
-        cpu: [amd]
+        # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise.
+        cpu: ${{ fromJSON((endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && '["amd"]' || '["intel"]') }}
     runs-on: ${{ fromJson(
         format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-{0}-{1}"]',
           matrix.hypervisor,

.github/workflows/CreateRelease.yml

@@ -24,7 +24,12 @@ jobs:
   build-rust-ubuntu:
     # see https://github.com/orgs/community/discussions/26286#discussioncomment-3251208 for why we need to check the ref
     if: ${{ contains(github.ref, 'refs/heads/release/') }} ||  ${{ github.ref=='refs/heads/main' }}
-    runs-on: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd", "JobId=build-rust-ubuntu-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"]
+    # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise.
+    runs-on: ${{ fromJSON(format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=build-rust-ubuntu-{1}-{2}-{3}"]',
+        (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel',
+        github.run_id,
+        github.run_number,
+        github.run_attempt)) }}
     needs: [release-blocker-check]
 
     steps:
@@ -101,7 +106,12 @@ jobs:
   publish:
     # see https://github.com/orgs/community/discussions/26286#discussioncomment-3251208 for why we need to check the ref
     if: ${{ contains(github.ref, 'refs/heads/release/') }} ||  ${{ github.ref=='refs/heads/main' }}
-    runs-on: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd", "JobId=create-release-publish-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"]
+    # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise.
+    runs-on: ${{ fromJSON(format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=create-release-publish-{1}-{2}-{3}"]',
+        (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel',
+        github.run_id,
+        github.run_number,
+        github.run_attempt)) }}
     needs:
       [
         build-rust-ubuntu,

.github/workflows/DailyArm64.yml

@@ -90,6 +90,7 @@ jobs:
       target: ${{ matrix.target }}
       arch: arm64
       max_total_time: 300 # 5 minutes in seconds
+      timeout_minutes: 35 # 5 minutes fuzzing plus setup and build on slower arm64 runners
 
   # File (or update) a release-blocking GitHub issue if any job fails. The first
   # label (area/ci-periodics-aarch64) is the de-duplication key used by

.github/workflows/Fuzzing.yml

@@ -28,12 +28,15 @@ jobs:
       target: ${{ matrix.target }}
       arch: X64
       max_total_time: 18000 # 5 hours in seconds
+      timeout_minutes: 330 # 300 minutes fuzzing plus 30 minutes for setup, build, cmin and corpus save
+      persist_corpus: true
     secrets: inherit
 
   notify-failure:
     runs-on: ubuntu-latest
     needs: [build-guests, fuzzing]
-    if: always() && (needs.build-guests.result == 'failure' || needs.fuzzing.result == 'failure')
+    # Timed-out jobs report 'cancelled'. A cancelled workflow stays quiet.
+    if: ${{ !cancelled() && (needs.build-guests.result == 'failure' || needs.build-guests.result == 'cancelled' || needs.fuzzing.result == 'failure' || needs.fuzzing.result == 'cancelled') }}
     permissions:
       issues: write
     steps:

.github/workflows/PrimeCaches.yml

@@ -81,10 +81,12 @@ jobs:
             hypervisor: kvm
             config: release
     timeout-minutes: ${{ fromJSON(vars.PRIME_CACHES_JOB_TIMEOUT || '30') }}
+    # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise.
     runs-on: ${{ fromJson(matrix.arch == 'X64'
-        && format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-amd", "JobId=prime-cache-{2}-{3}-{4}-{5}-{6}"]',
+        && format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}", "JobId=prime-cache-{3}-{4}-{5}-{6}-{7}"]',
             matrix.hypervisor == 'hyperv-ws2025' && 'Windows' || 'Linux',
             matrix.hypervisor == 'hyperv-ws2025' && 'win2025' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor,
+            (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel',
             matrix.hypervisor,
             matrix.config,
             github.run_id,

.github/workflows/ValidatePullRequest.yml

@@ -223,6 +223,7 @@ jobs:
     with:
       docs_only: ${{ needs.docs-pr.outputs.docs-only }}
       max_total_time: 300 # 5 minutes in seconds
+      timeout_minutes: 25 # 5 minutes fuzzing plus setup and build. Observed max 14.5 minutes.
       arch: ${{ matrix.arch }}
       target: ${{ matrix.target }}
 

.github/workflows/dep_build_guests.yml

@@ -33,8 +33,10 @@ jobs:
   build-guests:
     if: ${{ inputs.docs_only == 'false' }}
     timeout-minutes: ${{ fromJSON(vars.BUILD_GUESTS_JOB_TIMEOUT || '15') }}
+    # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise.
     runs-on: ${{ fromJson(inputs.arch == 'X64'
-        && format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-amd", "JobId=build-guests-{0}-{1}-{2}-{3}"]',
+        && format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=build-guests-{1}-{2}-{3}-{4}"]',
+            (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel',
             inputs.config,
             github.run_id,
             github.run_number,

.github/workflows/dep_code_checks.yml

@@ -26,7 +26,12 @@ jobs:
   linux-checks:
     if: ${{ inputs.docs_only == 'false' }}
     timeout-minutes: ${{ fromJSON(vars.CODE_CHECKS_JOB_TIMEOUT || '30') }}
-    runs-on: ["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-amd", "JobId=linux-checks-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"]
+    # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise.
+    runs-on: ${{ fromJSON(format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=linux-checks-{1}-{2}-{3}"]',
+        (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel',
+        github.run_id,
+        github.run_number,
+        github.run_attempt)) }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
 
@@ -99,7 +104,12 @@ jobs:
   windows-checks:
     if: ${{ inputs.docs_only == 'false' }}
     timeout-minutes: ${{ fromJSON(vars.CODE_CHECKS_JOB_TIMEOUT || '30') }}
-    runs-on: ["self-hosted", "Windows", "X64", "1ES.Pool=hld-win2025-amd", "JobId=windows-checks-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"]
+    # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise.
+    runs-on: ${{ fromJSON(format('["self-hosted", "Windows", "X64", "1ES.Pool=hld-win2025-{0}", "JobId=windows-checks-{1}-{2}-{3}"]',
+        (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel',
+        github.run_id,
+        github.run_number,
+        github.run_attempt)) }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
 

.github/workflows/dep_fuzzing.yml

@@ -7,6 +7,15 @@ on:
         description: Maximum total time for the fuzz run in seconds
         required: true
         type: number
+      timeout_minutes:
+        description: Maximum total time for the fuzz job in minutes
+        required: true
+        type: number
+      persist_corpus:
+        description: Cache and minimise the corpus across runs. For long scheduled runs only.
+        required: false
+        type: boolean
+        default: false
       target:
         description: Fuzz targets to run
         required: true
@@ -27,8 +36,12 @@ permissions:
 jobs:
   fuzz:
     if: ${{ inputs.docs_only == 'false' }}
+    # Precomputed by callers because Actions expressions do not support arithmetic.
+    timeout-minutes: ${{ inputs.timeout_minutes }}
+    # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise.
     runs-on: ${{ fromJson(inputs.arch == 'X64'
-        && format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-amd", "JobId=fuzz-{0}-{1}-{2}-{3}"]',
+        && format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=fuzz-{1}-{2}-{3}-{4}"]',
+            (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel',
             inputs.target,
             github.run_id,
             github.run_number,
@@ -56,10 +69,43 @@ jobs:
       - name: Install cargo-fuzz
         run: command -v cargo-fuzz >/dev/null 2>&1 || cargo install cargo-fuzz
 
+      # cargo-fuzz resolves the fuzz project by walking up from the working
+      # directory, so the corpus lives at <repo root>/fuzz/corpus/<target>.
+      # Runners are single-use, so without this each run starts from an empty
+      # corpus and coverage-guided fuzzing never accumulates progress.
+      - name: Restore fuzz corpus
+        if: ${{ inputs.persist_corpus }}
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: fuzz/corpus/${{ inputs.target }}
+          key: fuzz-corpus-${{ inputs.target }}-${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: |
+            fuzz-corpus-${{ inputs.target }}-
+
       - name: Run Fuzzing
         run: just fuzz-timed ${{ inputs.target }} ${{ inputs.max_total_time }}
         working-directory: src/hyperlight_host
 
+      # The corpus grows every week and the repo shares a 10 GB cache budget
+      # with the rust-cache entries PrimeCaches.yml maintains. cmin keeps only
+      # the minimal input set preserving coverage, bounding that growth.
+      # Housekeeping, so a failure here must not fail the job.
+      - name: Minimise fuzz corpus
+        if: ${{ always() && inputs.persist_corpus }}
+        timeout-minutes: 20
+        continue-on-error: true
+        run: just fuzz-cmin ${{ inputs.target }}
+        working-directory: src/hyperlight_host
+
+      # always() so the evolved corpus survives a timeout or a crash, not just
+      # a clean run.
+      - name: Save fuzz corpus
+        if: ${{ always() && inputs.persist_corpus }}
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: fuzz/corpus/${{ inputs.target }}
+          key: fuzz-corpus-${{ inputs.target }}-${{ github.run_id }}-${{ github.run_attempt }}
+
       - name: Upload Crash Artifacts
         if: failure() # This ensures artifacts are only uploaded on failure
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1

.github/workflows/dep_update_guest_locks.yml

@@ -23,7 +23,12 @@ defaults:
 
 jobs:
   update-guest-locks:
-    runs-on: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd", "JobId=update-guest-locks-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"]
+    # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise.
+    runs-on: ${{ fromJSON(format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=update-guest-locks-{1}-{2}-{3}"]',
+        (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel',
+        github.run_id,
+        github.run_number,
+        github.run_attempt)) }}
     timeout-minutes: ${{ fromJSON(vars.UPDATE_GUEST_LOCKS_JOB_TIMEOUT || '15') }}
     steps:
       # Get GitHub App token for pushing commits back to the PR

Justfile

@@ -456,6 +456,11 @@ fuzz fuzz-target:
 fuzz-timed fuzz-target max_time:
     case "{{ fuzz-target }}" in *trace*) just fuzz-trace-timed {{ max_time }} {{ fuzz-target }} ;; *) cargo +nightly fuzz run {{ fuzz-target }} --release -- -rss_limit_mb={{ fuzz_memory_limit }} -max_total_time={{ max_time }} ;; esac
 
+# Minimises the corpus of the given target to the smallest set preserving coverage
+# Uses *case* for compatibility to determine if the target is a tracing fuzzer or not
+fuzz-cmin fuzz-target:
+    case "{{ fuzz-target }}" in *trace*) just fuzz-trace-cmin {{ fuzz-target }} ;; *) cargo +nightly fuzz cmin {{ fuzz-target }} --release -- -rss_limit_mb={{ fuzz_memory_limit }} ;; esac
+
 # Builds fuzzers for submission to external fuzzing services
 build-fuzzers: (build-fuzzer "fuzz_guest_call") (build-fuzzer "fuzz_host_call") (build-fuzzer "fuzz_host_print")
 
@@ -483,6 +488,17 @@ fuzz-trace-timed max_time fuzz-target="fuzz_guest_trace":
     just build-rust-guests release
     just move-rust-guests release
 
+# Minimises the corpus of a tracing fuzzer
+# cmin re-executes every input, so the trace guest must be in place for it
+fuzz-trace-cmin fuzz-target="fuzz_guest_trace":
+    # We need to build the trace guest with the trace feature enabled
+    just build-rust-guests release trace_guest
+    just move-rust-guests release
+    RUST_LOG="trace,hyperlight_guest=trace,hyperlight_guest_bin=trace" cargo +nightly fuzz cmin {{ fuzz-target }} --features trace --release -- -rss_limit_mb={{ fuzz_memory_limit }}
+    # Rebuild the trace guest without the trace feature to avoid affecting other tests
+    just build-rust-guests release
+    just move-rust-guests release
+
 build-trace-fuzzers:
     cargo +nightly fuzz build fuzz_guest_trace --features trace