diff --git a/.github/workflows/CargoPublish.yml b/.github/workflows/CargoPublish.yml index 4b91fffad..68b89ef31 100644 --- a/.github/workflows/CargoPublish.yml +++ b/.github/workflows/CargoPublish.yml @@ -22,7 +22,12 @@ permissions: jobs: publish-hyperlight-packages: - runs-on: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd", "JobId=publish-hyperlight-packages-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"] + # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise. + runs-on: ${{ fromJSON(format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=publish-hyperlight-packages-{1}-{2}-{3}"]', + (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel', + github.run_id, + github.run_number, + github.run_attempt)) }} if: ${{ startsWith(github.ref, 'refs/heads/release/v') || inputs.dry_run }} diff --git a/.github/workflows/Coverage.yml b/.github/workflows/Coverage.yml index 8e91f7561..253c9141e 100644 --- a/.github/workflows/Coverage.yml +++ b/.github/workflows/Coverage.yml @@ -29,7 +29,8 @@ jobs: fail-fast: false matrix: hypervisor: [kvm] - cpu: [amd] + # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise. + cpu: ${{ fromJSON((endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && '["amd"]' || '["intel"]') }} runs-on: ${{ fromJson( format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-{0}-{1}"]', matrix.hypervisor, diff --git a/.github/workflows/CreateRelease.yml b/.github/workflows/CreateRelease.yml index 769a52c36..63eb834e1 100644 --- a/.github/workflows/CreateRelease.yml +++ b/.github/workflows/CreateRelease.yml @@ -24,7 +24,12 @@ jobs: build-rust-ubuntu: # see https://github.com/orgs/community/discussions/26286#discussioncomment-3251208 for why we need to check the ref if: ${{ contains(github.ref, 'refs/heads/release/') }} || ${{ github.ref=='refs/heads/main' }} - runs-on: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd", "JobId=build-rust-ubuntu-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"] + # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise. + runs-on: ${{ fromJSON(format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=build-rust-ubuntu-{1}-{2}-{3}"]', + (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel', + github.run_id, + github.run_number, + github.run_attempt)) }} needs: [release-blocker-check] steps: @@ -101,7 +106,12 @@ jobs: publish: # see https://github.com/orgs/community/discussions/26286#discussioncomment-3251208 for why we need to check the ref if: ${{ contains(github.ref, 'refs/heads/release/') }} || ${{ github.ref=='refs/heads/main' }} - runs-on: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd", "JobId=create-release-publish-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"] + # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise. + runs-on: ${{ fromJSON(format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=create-release-publish-{1}-{2}-{3}"]', + (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel', + github.run_id, + github.run_number, + github.run_attempt)) }} needs: [ build-rust-ubuntu, diff --git a/.github/workflows/DailyArm64.yml b/.github/workflows/DailyArm64.yml index 14e4d80f4..088baf666 100644 --- a/.github/workflows/DailyArm64.yml +++ b/.github/workflows/DailyArm64.yml @@ -90,6 +90,7 @@ jobs: target: ${{ matrix.target }} arch: arm64 max_total_time: 300 # 5 minutes in seconds + timeout_minutes: 35 # 5 minutes fuzzing plus setup and build on slower arm64 runners # File (or update) a release-blocking GitHub issue if any job fails. The first # label (area/ci-periodics-aarch64) is the de-duplication key used by diff --git a/.github/workflows/Fuzzing.yml b/.github/workflows/Fuzzing.yml index e9f95c56e..3cceeb68d 100644 --- a/.github/workflows/Fuzzing.yml +++ b/.github/workflows/Fuzzing.yml @@ -28,12 +28,15 @@ jobs: target: ${{ matrix.target }} arch: X64 max_total_time: 18000 # 5 hours in seconds + timeout_minutes: 330 # 300 minutes fuzzing plus 30 minutes for setup, build, cmin and corpus save + persist_corpus: true secrets: inherit notify-failure: runs-on: ubuntu-latest needs: [build-guests, fuzzing] - if: always() && (needs.build-guests.result == 'failure' || needs.fuzzing.result == 'failure') + # Timed-out jobs report 'cancelled'. A cancelled workflow stays quiet. + if: ${{ !cancelled() && (needs.build-guests.result == 'failure' || needs.build-guests.result == 'cancelled' || needs.fuzzing.result == 'failure' || needs.fuzzing.result == 'cancelled') }} permissions: issues: write steps: diff --git a/.github/workflows/PrimeCaches.yml b/.github/workflows/PrimeCaches.yml index bec441206..d12d5e4f8 100644 --- a/.github/workflows/PrimeCaches.yml +++ b/.github/workflows/PrimeCaches.yml @@ -81,10 +81,12 @@ jobs: hypervisor: kvm config: release timeout-minutes: ${{ fromJSON(vars.PRIME_CACHES_JOB_TIMEOUT || '30') }} + # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise. runs-on: ${{ fromJson(matrix.arch == 'X64' - && format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-amd", "JobId=prime-cache-{2}-{3}-{4}-{5}-{6}"]', + && format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}", "JobId=prime-cache-{3}-{4}-{5}-{6}-{7}"]', matrix.hypervisor == 'hyperv-ws2025' && 'Windows' || 'Linux', matrix.hypervisor == 'hyperv-ws2025' && 'win2025' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor, + (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel', matrix.hypervisor, matrix.config, github.run_id, diff --git a/.github/workflows/ValidatePullRequest.yml b/.github/workflows/ValidatePullRequest.yml index 4c16a78b7..de8a2259f 100644 --- a/.github/workflows/ValidatePullRequest.yml +++ b/.github/workflows/ValidatePullRequest.yml @@ -223,6 +223,7 @@ jobs: with: docs_only: ${{ needs.docs-pr.outputs.docs-only }} max_total_time: 300 # 5 minutes in seconds + timeout_minutes: 25 # 5 minutes fuzzing plus setup and build. Observed max 14.5 minutes. arch: ${{ matrix.arch }} target: ${{ matrix.target }} diff --git a/.github/workflows/dep_build_guests.yml b/.github/workflows/dep_build_guests.yml index feac4ea52..b65b89b73 100644 --- a/.github/workflows/dep_build_guests.yml +++ b/.github/workflows/dep_build_guests.yml @@ -33,8 +33,10 @@ jobs: build-guests: if: ${{ inputs.docs_only == 'false' }} timeout-minutes: ${{ fromJSON(vars.BUILD_GUESTS_JOB_TIMEOUT || '15') }} + # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise. runs-on: ${{ fromJson(inputs.arch == 'X64' - && format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-amd", "JobId=build-guests-{0}-{1}-{2}-{3}"]', + && format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=build-guests-{1}-{2}-{3}-{4}"]', + (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel', inputs.config, github.run_id, github.run_number, diff --git a/.github/workflows/dep_code_checks.yml b/.github/workflows/dep_code_checks.yml index aedc4ca17..f75a52610 100644 --- a/.github/workflows/dep_code_checks.yml +++ b/.github/workflows/dep_code_checks.yml @@ -26,7 +26,12 @@ jobs: linux-checks: if: ${{ inputs.docs_only == 'false' }} timeout-minutes: ${{ fromJSON(vars.CODE_CHECKS_JOB_TIMEOUT || '30') }} - runs-on: ["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-amd", "JobId=linux-checks-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"] + # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise. + runs-on: ${{ fromJSON(format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=linux-checks-{1}-{2}-{3}"]', + (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel', + github.run_id, + github.run_number, + github.run_attempt)) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -99,7 +104,12 @@ jobs: windows-checks: if: ${{ inputs.docs_only == 'false' }} timeout-minutes: ${{ fromJSON(vars.CODE_CHECKS_JOB_TIMEOUT || '30') }} - runs-on: ["self-hosted", "Windows", "X64", "1ES.Pool=hld-win2025-amd", "JobId=windows-checks-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"] + # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise. + runs-on: ${{ fromJSON(format('["self-hosted", "Windows", "X64", "1ES.Pool=hld-win2025-{0}", "JobId=windows-checks-{1}-{2}-{3}"]', + (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel', + github.run_id, + github.run_number, + github.run_attempt)) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/dep_fuzzing.yml b/.github/workflows/dep_fuzzing.yml index 471d12de9..f52a4f8da 100644 --- a/.github/workflows/dep_fuzzing.yml +++ b/.github/workflows/dep_fuzzing.yml @@ -7,6 +7,15 @@ on: description: Maximum total time for the fuzz run in seconds required: true type: number + timeout_minutes: + description: Maximum total time for the fuzz job in minutes + required: true + type: number + persist_corpus: + description: Cache and minimise the corpus across runs. For long scheduled runs only. + required: false + type: boolean + default: false target: description: Fuzz targets to run required: true @@ -27,8 +36,12 @@ permissions: jobs: fuzz: if: ${{ inputs.docs_only == 'false' }} + # Precomputed by callers because Actions expressions do not support arithmetic. + timeout-minutes: ${{ inputs.timeout_minutes }} + # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise. runs-on: ${{ fromJson(inputs.arch == 'X64' - && format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-amd", "JobId=fuzz-{0}-{1}-{2}-{3}"]', + && format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=fuzz-{1}-{2}-{3}-{4}"]', + (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel', inputs.target, github.run_id, github.run_number, @@ -56,10 +69,43 @@ jobs: - name: Install cargo-fuzz run: command -v cargo-fuzz >/dev/null 2>&1 || cargo install cargo-fuzz + # cargo-fuzz resolves the fuzz project by walking up from the working + # directory, so the corpus lives at /fuzz/corpus/. + # Runners are single-use, so without this each run starts from an empty + # corpus and coverage-guided fuzzing never accumulates progress. + - name: Restore fuzz corpus + if: ${{ inputs.persist_corpus }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: fuzz/corpus/${{ inputs.target }} + key: fuzz-corpus-${{ inputs.target }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + fuzz-corpus-${{ inputs.target }}- + - name: Run Fuzzing run: just fuzz-timed ${{ inputs.target }} ${{ inputs.max_total_time }} working-directory: src/hyperlight_host + # The corpus grows every week and the repo shares a 10 GB cache budget + # with the rust-cache entries PrimeCaches.yml maintains. cmin keeps only + # the minimal input set preserving coverage, bounding that growth. + # Housekeeping, so a failure here must not fail the job. + - name: Minimise fuzz corpus + if: ${{ always() && inputs.persist_corpus }} + timeout-minutes: 20 + continue-on-error: true + run: just fuzz-cmin ${{ inputs.target }} + working-directory: src/hyperlight_host + + # always() so the evolved corpus survives a timeout or a crash, not just + # a clean run. + - name: Save fuzz corpus + if: ${{ always() && inputs.persist_corpus }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: fuzz/corpus/${{ inputs.target }} + key: fuzz-corpus-${{ inputs.target }}-${{ github.run_id }}-${{ github.run_attempt }} + - name: Upload Crash Artifacts if: failure() # This ensures artifacts are only uploaded on failure uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/dep_update_guest_locks.yml b/.github/workflows/dep_update_guest_locks.yml index de39a5d5b..1e4716d09 100644 --- a/.github/workflows/dep_update_guest_locks.yml +++ b/.github/workflows/dep_update_guest_locks.yml @@ -23,7 +23,12 @@ defaults: jobs: update-guest-locks: - runs-on: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd", "JobId=update-guest-locks-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"] + # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise. + runs-on: ${{ fromJSON(format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-kvm-{0}", "JobId=update-guest-locks-{1}-{2}-{3}"]', + (endsWith(github.run_id, '0') || endsWith(github.run_id, '2') || endsWith(github.run_id, '4') || endsWith(github.run_id, '6') || endsWith(github.run_id, '8')) && 'amd' || 'intel', + github.run_id, + github.run_number, + github.run_attempt)) }} timeout-minutes: ${{ fromJSON(vars.UPDATE_GUEST_LOCKS_JOB_TIMEOUT || '15') }} steps: # Get GitHub App token for pushing commits back to the PR diff --git a/Justfile b/Justfile index c5524bab6..ad82ce87d 100644 --- a/Justfile +++ b/Justfile @@ -456,6 +456,11 @@ fuzz fuzz-target: fuzz-timed fuzz-target max_time: case "{{ fuzz-target }}" in *trace*) just fuzz-trace-timed {{ max_time }} {{ fuzz-target }} ;; *) cargo +nightly fuzz run {{ fuzz-target }} --release -- -rss_limit_mb={{ fuzz_memory_limit }} -max_total_time={{ max_time }} ;; esac +# Minimises the corpus of the given target to the smallest set preserving coverage +# Uses *case* for compatibility to determine if the target is a tracing fuzzer or not +fuzz-cmin fuzz-target: + case "{{ fuzz-target }}" in *trace*) just fuzz-trace-cmin {{ fuzz-target }} ;; *) cargo +nightly fuzz cmin {{ fuzz-target }} --release -- -rss_limit_mb={{ fuzz_memory_limit }} ;; esac + # Builds fuzzers for submission to external fuzzing services build-fuzzers: (build-fuzzer "fuzz_guest_call") (build-fuzzer "fuzz_host_call") (build-fuzzer "fuzz_host_print") @@ -483,6 +488,17 @@ fuzz-trace-timed max_time fuzz-target="fuzz_guest_trace": just build-rust-guests release just move-rust-guests release +# Minimises the corpus of a tracing fuzzer +# cmin re-executes every input, so the trace guest must be in place for it +fuzz-trace-cmin fuzz-target="fuzz_guest_trace": + # We need to build the trace guest with the trace feature enabled + just build-rust-guests release trace_guest + just move-rust-guests release + RUST_LOG="trace,hyperlight_guest=trace,hyperlight_guest_bin=trace" cargo +nightly fuzz cmin {{ fuzz-target }} --features trace --release -- -rss_limit_mb={{ fuzz_memory_limit }} + # Rebuild the trace guest without the trace feature to avoid affecting other tests + just build-rust-guests release + just move-rust-guests release + build-trace-fuzzers: cargo +nightly fuzz build fuzz_guest_trace --features trace