From 9cf09a90788202fb2e944ee237288996e02cd585 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Fri, 31 Jul 2026 01:22:19 -0400 Subject: [PATCH 01/13] ci: activate dependency-aware package builds and tests --- .github/workflows/ci.yml | 436 ++++++++++++++++++++++++++++++------ ci/tools/compute_ci_plan.py | 147 ++++++++++++ 2 files changed, 515 insertions(+), 68 deletions(-) create mode 100644 ci/tools/compute_ci_plan.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2aadf222306..0be7f9445b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,24 +72,31 @@ jobs: echo "skip=${skip}" >> "$GITHUB_OUTPUT" echo "doc_only=${doc_only}" >> "$GITHUB_OUTPUT" - # Detect which top-level modules were touched by the PR so downstream build - # and test jobs can avoid rebuilding/retesting modules unaffected by the - # change. See issue #299. + # Detect which packages were touched by the PR so downstream build and test + # jobs can avoid rebuilding/retesting packages unaffected by the change. + # See issue #299. # # Dependency graph (verified in pyproject.toml files): # cuda_pathfinder -> (no internal deps) # cuda_bindings -> cuda_pathfinder # cuda_core -> cuda_pathfinder, cuda_bindings - # cuda_python -> cuda_bindings (meta package) + # cuda_python -> cuda_pathfinder, cuda_bindings, cuda_core (meta package) # # A change to cuda_pathfinder (or shared infra) forces a rebuild of every # downstream module. A change to cuda_bindings forces rebuild of cuda_core. - # A change to cuda_core alone skips rebuilding/retesting cuda_bindings. + # A change to cuda_core alone skips rebuilding/retesting cuda_bindings and + # cuda_pathfinder, but still retests the downstream cuda-python metapackage. + # CI/planner changes are shared by design, so this implementation runs the + # full pipeline; exercise selective cases in follow-up package-only PRs. # On push to main, tag refs, schedule, or workflow_dispatch events we # unconditionally run everything because there is no meaningful "changed # paths" baseline for those events. detect-changes: runs-on: ubuntu-latest + permissions: + actions: read + contents: read + pull-requests: read outputs: bindings: ${{ steps.compose.outputs.bindings }} core: ${{ steps.compose.outputs.core }} @@ -100,10 +107,14 @@ jobs: build_bindings: ${{ steps.compose.outputs.build_bindings }} build_core: ${{ steps.compose.outputs.build_core }} build_pathfinder: ${{ steps.compose.outputs.build_pathfinder }} + build_python: ${{ steps.compose.outputs.build_python }} test_bindings: ${{ steps.compose.outputs.test_bindings }} test_core: ${{ steps.compose.outputs.test_core }} test_pathfinder: ${{ steps.compose.outputs.test_pathfinder }} pr_merge_base: ${{ steps.filter.outputs.merge_base }} + test_python: ${{ steps.compose.outputs.test_python }} + baseline_run_id: ${{ steps.compose.outputs.baseline_run_id }} + baseline_sha: ${{ steps.compose.outputs.baseline_sha }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -136,52 +147,177 @@ jobs: # off by `if:`, so `BASE_REF` is never consumed there. BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} run: | - # Diff against the merge base with the PR's actual target branch. - # Uses merge-base so diverged branches only show files changed on - # the PR side, not upstream commits. + set -euo pipefail if [[ -z "${BASE_REF}" ]]; then echo "Could not resolve PR base branch from get-pr-info output" >&2 exit 1 fi + + # Diff against the merge base with the PR's actual target branch. + # Disabling rename detection reports both sides of a cross-package + # move, which prevents the source package from being skipped. base=$(git merge-base HEAD "origin/${BASE_REF}") - changed=$(git diff --name-only "$base"...HEAD) + git diff --no-renames --name-only -z "$base"...HEAD > changed-paths + python ci/tools/compute_ci_plan.py changed-paths >> "$GITHUB_OUTPUT" + echo "merge_base=${base}" >> "$GITHUB_OUTPUT" + + { + echo "### Selective CI changed paths" + echo + tr '\0' '\n' < changed-paths | sed 's/^/- `/' | sed 's/$/`/' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Resolve reusable base artifacts + id: baseline + if: ${{ startsWith(github.ref_name, 'pull-request/') }} + env: + BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + + unavailable() { + echo "available=false" >> "$GITHUB_OUTPUT" + echo "No complete reusable artifact set was found; this run will build and test everything." >> "$GITHUB_STEP_SUMMARY" + exit 0 + } + + if [[ -z "${BASE_REF}" ]]; then + unavailable + fi + + merge_base=$(git merge-base HEAD "origin/${BASE_REF}") + if ! runs=$(gh run list \ + --repo "${{ github.repository }}" \ + --branch "${BASE_REF}" \ + --commit "${merge_base}" \ + --event push \ + --workflow ci.yml \ + --status success \ + --limit 1 \ + --json databaseId,headSha,createdAt); then + unavailable + fi + + # Reuse only artifacts produced from the exact commit used as the + # PR diff base. Using the latest base-branch run is unsafe for a PR + # that was opened before newer changes landed on that branch. + run_id=$(jq -r '.[0].databaseId // empty' <<< "$runs") + baseline_sha=$(jq -r '.[0].headSha // empty' <<< "$runs") + if [[ -z "${run_id}" || "${baseline_sha}" != "${merge_base}" ]]; then + unavailable + fi + + if ! artifact_names=$(gh api \ + "repos/${{ github.repository }}/actions/runs/${run_id}/artifacts?per_page=100" \ + --paginate \ + --jq '.artifacts[] | select(.expired == false) | .name'); then + unavailable + fi - has_match() { - grep -qE "$1" <<< "$changed" && echo true || echo false + has_artifact() { + grep -Fxq "$1" <<< "$artifact_names" } + missing=() + for name in cuda-pathfinder-wheel cuda-python-wheel; do + has_artifact "$name" || missing+=("$name") + done + + cuda_version=$(yq '.cuda.build.version' ci/versions.yml) + if ! python_versions=$(yq -r '.jobs.build.strategy.matrix."python-version"[]' .github/workflows/build-wheel.yml); then + unavailable + fi + if [[ -z "${python_versions}" ]]; then + unavailable + fi + while IFS= read -r python_version; do + python=${python_version//./} + for platform in linux-64 linux-aarch64 win-64; do + binding="cuda-bindings-python${python}-cuda${cuda_version}-${platform}-${baseline_sha}" + core="cuda-core-python${python}-${platform}-${baseline_sha}" + has_artifact "$binding" || missing+=("$binding") + has_artifact "$core" || missing+=("$core") + done + done <<< "${python_versions}" + + if (( ${#missing[@]} != 0 )); then + printf 'Missing reusable artifact: %s\n' "${missing[@]}" >&2 + unavailable + fi + { - echo "bindings=$(has_match '^cuda_bindings/')" - echo "core=$(has_match '^cuda_core/')" - echo "pathfinder=$(has_match '^cuda_pathfinder/')" - echo "python_meta=$(has_match '^cuda_python/')" - echo "test_helpers=$(has_match '^cuda_python_test_helpers/')" - echo "shared=$(has_match '^(\.github/|ci/|scripts/|toolshed/|conftest\.py$|pyproject\.toml$|pixi\.(toml|lock)$|pytest\.ini$|ruff\.toml$)')" - echo "merge_base=${base}" + echo "available=true" + echo "run_id=${run_id}" + echo "sha=${baseline_sha}" } >> "$GITHUB_OUTPUT" + { + echo + echo "Reusable artifacts: run \`${run_id}\` at \`${baseline_sha}\` on \`${BASE_REF}\`." + } >> "$GITHUB_STEP_SUMMARY" - name: Compose gating outputs id: compose env: IS_PR: ${{ startsWith(github.ref_name, 'pull-request/') }} - BINDINGS: ${{ steps.filter.outputs.bindings || 'false' }} - CORE: ${{ steps.filter.outputs.core || 'false' }} - PATHFINDER: ${{ steps.filter.outputs.pathfinder || 'false' }} - PYTHON_META: ${{ steps.filter.outputs.python_meta || 'false' }} - TEST_HELPERS: ${{ steps.filter.outputs.test_helpers || 'false' }} - SHARED: ${{ steps.filter.outputs.shared || 'false' }} + BASELINE_AVAILABLE: ${{ steps.baseline.outputs.available || 'false' }} + BASELINE_RUN_ID: ${{ steps.baseline.outputs.run_id }} + BASELINE_SHA: ${{ steps.baseline.outputs.sha }} + BINDINGS: ${{ steps.filter.outputs.bindings_source }} + CORE: ${{ steps.filter.outputs.core_source }} + PATHFINDER: ${{ steps.filter.outputs.pathfinder_source }} + PYTHON_META: ${{ steps.filter.outputs.python_source }} + TEST_HELPERS: ${{ steps.filter.outputs.test_helpers }} + SHARED: ${{ steps.filter.outputs.shared }} + BUILD_BINDINGS: ${{ steps.filter.outputs.build_bindings }} + BUILD_CORE: ${{ steps.filter.outputs.build_core }} + BUILD_PATHFINDER: ${{ steps.filter.outputs.build_pathfinder }} + BUILD_PYTHON: ${{ steps.filter.outputs.build_python }} + TEST_BINDINGS: ${{ steps.filter.outputs.test_bindings }} + TEST_CORE: ${{ steps.filter.outputs.test_core }} + TEST_PATHFINDER: ${{ steps.filter.outputs.test_pathfinder }} + TEST_PYTHON: ${{ steps.filter.outputs.test_python }} run: | set -euxo pipefail - # Non-PR events (push to main, tag push, schedule, workflow_dispatch) - # always exercise the full pipeline because there is no baseline for - # a meaningful diff. - if [[ "${IS_PR}" != "true" ]]; then + planner_valid=true + if [[ "${IS_PR}" == "true" ]]; then + for value in \ + "${BINDINGS}" "${CORE}" "${PATHFINDER}" "${PYTHON_META}" \ + "${TEST_HELPERS}" "${SHARED}" \ + "${BUILD_BINDINGS}" "${BUILD_CORE}" "${BUILD_PATHFINDER}" "${BUILD_PYTHON}" \ + "${TEST_BINDINGS}" "${TEST_CORE}" "${TEST_PATHFINDER}" "${TEST_PYTHON}"; do + if [[ "${value}" != "true" && "${value}" != "false" ]]; then + planner_valid=false + fi + done + if [[ "${BASELINE_AVAILABLE}" == "true" && + ( -z "${BASELINE_RUN_ID}" || -z "${BASELINE_SHA}" ) ]]; then + planner_valid=false + fi + fi + + # Non-PR events produce the complete trusted artifact set. PRs also + # run everything when the trusted base artifact inventory is absent + # or the planner did not emit a complete boolean result. + if [[ "${IS_PR}" != "true" || + "${BASELINE_AVAILABLE}" != "true" || + "${planner_valid}" != "true" ]]; then bindings=true core=true pathfinder=true python_meta=true test_helpers=true shared=true + build_bindings=true + build_core=true + build_pathfinder=true + build_python=true + test_bindings=true + test_core=true + test_pathfinder=true + test_python=true + baseline_run_id="" + baseline_sha="" else bindings="${BINDINGS}" core="${CORE}" @@ -189,32 +325,18 @@ jobs: python_meta="${PYTHON_META}" test_helpers="${TEST_HELPERS}" shared="${SHARED}" + build_bindings="${BUILD_BINDINGS}" + build_core="${BUILD_CORE}" + build_pathfinder="${BUILD_PATHFINDER}" + build_python="${BUILD_PYTHON}" + test_bindings="${TEST_BINDINGS}" + test_core="${TEST_CORE}" + test_pathfinder="${TEST_PATHFINDER}" + test_python="${TEST_PYTHON}" + baseline_run_id="${BASELINE_RUN_ID}" + baseline_sha="${BASELINE_SHA}" fi - or_flag() { - for v in "$@"; do - if [[ "${v}" == "true" ]]; then - echo "true" - return - fi - done - echo "false" - } - - # Build gating: pathfinder change forces rebuild of bindings and - # core; bindings change forces rebuild of core. shared changes force - # a full rebuild. - build_pathfinder="$(or_flag "${shared}" "${pathfinder}")" - build_bindings="$(or_flag "${shared}" "${pathfinder}" "${bindings}")" - build_core="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${core}")" - - # Test gating: tests for a module must run whenever that module, any - # of its runtime dependencies, the shared test helper package, or - # shared infra changes. pathfinder tests are cheap and always run. - test_pathfinder=true - test_bindings="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${test_helpers}")" - test_core="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${core}" "${test_helpers}")" - { echo "bindings=${bindings}" echo "core=${core}" @@ -225,11 +347,27 @@ jobs: echo "build_bindings=${build_bindings}" echo "build_core=${build_core}" echo "build_pathfinder=${build_pathfinder}" + echo "build_python=${build_python}" echo "test_bindings=${test_bindings}" echo "test_core=${test_core}" echo "test_pathfinder=${test_pathfinder}" + echo "test_python=${test_python}" + echo "baseline_run_id=${baseline_run_id}" + echo "baseline_sha=${baseline_sha}" } >> "$GITHUB_OUTPUT" + { + echo + echo "### Effective package plan" + echo + echo "| Package | Build | Test |" + echo "| --- | --- | --- |" + echo "| cuda-pathfinder | ${build_pathfinder} | ${test_pathfinder} |" + echo "| cuda-bindings | ${build_bindings} | ${test_bindings} |" + echo "| cuda-core | ${build_core} | ${test_core} |" + echo "| cuda-python | ${build_python} | ${test_python} |" + } >> "$GITHUB_STEP_SUMMARY" + api-check-core-vs-release: name: API check (cuda_core vs. latest release) if: >- @@ -314,15 +452,17 @@ jobs: merge-base: ${{ needs.detect-changes.outputs.pr_merge_base }} # NOTE: Build jobs are intentionally split by platform rather than using a single - # matrix. This allows each test job to depend only on its corresponding build, - # so faster platforms can proceed through build & test without waiting for slower - # ones. Keep these job definitions textually identical except for: + # matrix. This lets each test job consume its platform-specific artifacts as + # soon as they are ready. ARM64 and Windows tests also wait for linux-64, + # which produces the universal pathfinder and cuda-python wheels. Keep these + # job definitions textually identical except for: # - host-platform value # - if: condition (build-linux-64 omits doc-only check since it's needed for docs) build-linux-64: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: @@ -330,50 +470,105 @@ jobs: - linux-64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} + baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} # See build-linux-64 for why build jobs are split by platform. build-linux-aarch64: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: host-platform: - linux-aarch64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python) || + fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} + baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} # See build-linux-64 for why build jobs are split by platform. build-windows: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: host-platform: - win-64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python) || + fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} + baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} # NOTE: test-sdist jobs are split by platform (mirroring build-* and test-wheel-*) # so platform-specific sources (e.g. cuda_bindings/*_windows.pyx selected by @@ -385,26 +580,57 @@ jobs: needs: - ci-vars - should-skip + - detect-changes + - build-linux-64 name: Test sdist linux-64 - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/test-sdist-linux.yml with: host-platform: linux-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} # See test-sdist-linux for why sdist test jobs are split by platform. test-sdist-windows: needs: - ci-vars - should-skip + - detect-changes + - build-linux-64 + - build-windows name: Test sdist win-64 - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/test-sdist-windows.yml with: host-platform: win-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} # NOTE: Test jobs are split by platform for the same reason as build jobs (see # build-linux-64). Keep these job definitions textually identical except for: @@ -418,8 +644,14 @@ jobs: host-platform: - linux-64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars @@ -433,7 +665,10 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} + test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} # See test-linux-64 for why test jobs are split by platform. test-linux-aarch64: @@ -443,13 +678,20 @@ jobs: host-platform: - linux-aarch64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars - should-skip - detect-changes + - build-linux-64 - build-linux-aarch64 secrets: inherit uses: ./.github/workflows/test-wheel-linux.yml @@ -458,7 +700,10 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} + test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} # See test-linux-64 for why test jobs are split by platform. test-windows: @@ -468,13 +713,20 @@ jobs: host-platform: - win-64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars - should-skip - detect-changes + - build-linux-64 - build-windows secrets: inherit uses: ./.github/workflows/test-wheel-windows.yml @@ -483,7 +735,10 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} + test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} doc: name: Docs @@ -538,13 +793,19 @@ jobs: if: always() runs-on: ubuntu-latest needs: + - ci-vars - should-skip - detect-changes + - build-linux-64 + - build-linux-aarch64 + - build-windows - test-sdist-linux - test-sdist-windows - test-linux-64 - test-linux-aarch64 - test-windows + - api-check-core-vs-release + - api-check-core-vs-base - doc - precommit-windows steps: @@ -565,6 +826,16 @@ jobs: fi doc_only="${{ needs.should-skip.outputs.doc-only }}" + build_selected="${{ needs.detect-changes.outputs.build_pathfinder == 'true' || + needs.detect-changes.outputs.build_bindings == 'true' || + needs.detect-changes.outputs.build_core == 'true' || + needs.detect-changes.outputs.build_python == 'true' }}" + test_selected="${{ needs.detect-changes.outputs.test_pathfinder == 'true' || + needs.detect-changes.outputs.test_bindings == 'true' || + needs.detect-changes.outputs.test_core == 'true' || + needs.detect-changes.outputs.test_python == 'true' }}" + core_changed="${{ needs.detect-changes.outputs.core == 'true' }}" + is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { name=$1; expected=$2; result=$3 @@ -575,18 +846,47 @@ jobs: fi } - # always expected to succeed (even in [doc-only] mode) - check_result "should-skip" "success" "${{ needs.should-skip.result }}" - check_result "detect-changes" "success" "${{ needs.detect-changes.result }}" - check_result "doc" "success" "${{ needs.doc.result }}" + # Control jobs, the universal linux build, docs, and Windows + # pre-commit checks always run. + check_result "ci-vars" "success" "${{ needs.ci-vars.result }}" + check_result "should-skip" "success" "${{ needs.should-skip.result }}" + check_result "detect-changes" "success" "${{ needs.detect-changes.result }}" + check_result "build-linux-64" "success" "${{ needs.build-linux-64.result }}" + check_result "doc" "success" "${{ needs.doc.result }}" check_result "precommit-windows" "success" "${{ needs.precommit-windows.result }}" - # [doc-only] flips these from 'success' to 'skipped' - if [[ "$doc_only" == "true" ]]; then expected="skipped"; else expected="success"; fi + # Platform builds run whenever any package needs to be built or tested. + expected="skipped" + if [[ "$doc_only" != "true" && + ( "$build_selected" == "true" || "$test_selected" == "true" ) ]]; then + expected="success" + fi + check_result "build-linux-aarch64" "$expected" "${{ needs.build-linux-aarch64.result }}" + check_result "build-windows" "$expected" "${{ needs.build-windows.result }}" + + # Sdist and wheel tests are independently gated by the effective plan. + expected="skipped" + if [[ "$doc_only" != "true" && "$build_selected" == "true" ]]; then + expected="success" + fi check_result "test-sdist-linux" "$expected" "${{ needs.test-sdist-linux.result }}" check_result "test-sdist-windows" "$expected" "${{ needs.test-sdist-windows.result }}" + + expected="skipped" + if [[ "$doc_only" != "true" && "$test_selected" == "true" ]]; then + expected="success" + fi check_result "test-linux-64" "$expected" "${{ needs.test-linux-64.result }}" check_result "test-linux-aarch64" "$expected" "${{ needs.test-linux-aarch64.result }}" check_result "test-windows" "$expected" "${{ needs.test-windows.result }}" + # API compatibility checks only run for cuda_core source changes. + expected="skipped" + if [[ "$core_changed" == "true" ]]; then expected="success"; fi + check_result "api-check-core-vs-release" "$expected" "${{ needs.api-check-core-vs-release.result }}" + + expected="skipped" + if [[ "$is_pr" == "true" && "$core_changed" == "true" ]]; then expected="success"; fi + check_result "api-check-core-vs-base" "$expected" "${{ needs.api-check-core-vs-base.result }}" + [[ "$status" == "success" ]] diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py new file mode 100644 index 00000000000..7095514b73e --- /dev/null +++ b/ci/tools/compute_ci_plan.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Compute the package build and test closure for a set of changed paths.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +PACKAGES = { + "cuda_pathfinder": "pathfinder", + "cuda_bindings": "bindings", + "cuda_core": "core", + "cuda_python": "python", +} + +SHARED_PREFIXES = ( + ".github/", + "ci/", + "scripts/", + "toolshed/", +) + +SHARED_FILES = { + ".pre-commit-config.yaml", + "conftest.py", + "pixi.lock", + "pixi.toml", + "pytest.ini", + "ruff.toml", +} + +KNOWN_REPOSITORY_FILES = { + ".git-blame-ignore-revs", + ".gitignore", + "AGENTS.md", + "CHANGELOG.md", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "README.md", + "SECURITY.md", +} + + +def _bool(value: bool) -> str: + return str(value).lower() + + +def _read_paths(path: Path) -> list[str]: + return [value.decode("utf-8", errors="surrogateescape") for value in path.read_bytes().split(b"\0") if value] + + +def compute_plan(paths: list[str]) -> dict[str, bool]: + source = dict.fromkeys(PACKAGES.values(), False) + tests = dict.fromkeys(PACKAGES.values(), False) + docs = False + test_helpers = False + shared = False + unknown = False + + for path in paths: + package_dir, separator, relative_path = path.partition("/") + package = PACKAGES.get(package_dir) + if package is not None and separator: + if relative_path.startswith("docs/"): + docs = True + elif relative_path.startswith(("tests/", "examples/")): + tests[package] = True + else: + source[package] = True + continue + + if path.startswith("cuda_python_test_helpers/"): + test_helpers = True + elif path.startswith("benchmarks/cuda_bindings/"): + tests["bindings"] = True + elif path.startswith(SHARED_PREFIXES) or path in SHARED_FILES: + shared = True + elif path in KNOWN_REPOSITORY_FILES: + # Repository policy and prose files do not affect package artifacts. + continue + else: + unknown = True + + full = shared or unknown + + build_pathfinder = full or source["pathfinder"] + # Development cuda-python wheels exactly pin cuda-bindings, so a + # metapackage change needs a matching bindings artifact for its smoke test. + build_bindings = full or source["pathfinder"] or source["bindings"] or source["python"] + build_core = full or source["pathfinder"] or source["bindings"] or source["core"] + # A core-only change can reuse the baseline cuda-python wheel: rebuilding + # it would also require rebuilding the exact-version cuda-bindings pin. + build_python = full or source["pathfinder"] or source["bindings"] or source["python"] + + test_pathfinder = full or source["pathfinder"] or tests["pathfinder"] + test_bindings = full or source["pathfinder"] or source["bindings"] or tests["bindings"] or test_helpers + test_core = full or source["pathfinder"] or source["bindings"] or source["core"] or tests["core"] or test_helpers + test_python = ( + full or source["pathfinder"] or source["bindings"] or source["core"] or source["python"] or tests["python"] + ) + + return { + "shared": shared, + "unknown": unknown, + "docs": docs, + "test_helpers": test_helpers, + "pathfinder_source": source["pathfinder"], + "bindings_source": source["bindings"], + "core_source": source["core"], + "python_source": source["python"], + "pathfinder_tests": tests["pathfinder"], + "bindings_tests": tests["bindings"], + "core_tests": tests["core"], + "python_tests": tests["python"], + "build_pathfinder": build_pathfinder, + "build_bindings": build_bindings, + "build_core": build_core, + "build_python": build_python, + "test_pathfinder": test_pathfinder, + "test_bindings": test_bindings, + "test_core": test_core, + "test_python": test_python, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "paths_file", + type=Path, + help="NUL-separated changed-path list produced by git diff --name-only -z", + ) + args = parser.parse_args() + + plan = compute_plan(_read_paths(args.paths_file)) + for key, value in plan.items(): + print(f"{key}={_bool(value)}") + + +if __name__ == "__main__": + main() From da7ebee702759b45c0a81fa0c1a9f7f233fbf874 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Thu, 13 Aug 2026 22:45:31 -0400 Subject: [PATCH 02/13] ci: use paths-filter for selective CI planning --- .github/workflows/ci.yml | 307 ++++++++++++++++++------------------ ci/tools/compute_ci_plan.py | 147 ----------------- 2 files changed, 156 insertions(+), 298 deletions(-) delete mode 100644 ci/tools/compute_ci_plan.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0be7f9445b5..16ab65b3db2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,7 @@ jobs: # downstream module. A change to cuda_bindings forces rebuild of cuda_core. # A change to cuda_core alone skips rebuilding/retesting cuda_bindings and # cuda_pathfinder, but still retests the downstream cuda-python metapackage. - # CI/planner changes are shared by design, so this implementation runs the + # CI/filter changes are shared by design, so this implementation runs the # full pipeline; exercise selective cases in follow-up package-only PRs. # On push to main, tag refs, schedule, or workflow_dispatch events we # unconditionally run everything because there is no meaningful "changed @@ -98,43 +98,98 @@ jobs: contents: read pull-requests: read outputs: - bindings: ${{ steps.compose.outputs.bindings }} - core: ${{ steps.compose.outputs.core }} - pathfinder: ${{ steps.compose.outputs.pathfinder }} - python_meta: ${{ steps.compose.outputs.python_meta }} - test_helpers: ${{ steps.compose.outputs.test_helpers }} - shared: ${{ steps.compose.outputs.shared }} - build_bindings: ${{ steps.compose.outputs.build_bindings }} - build_core: ${{ steps.compose.outputs.build_core }} - build_pathfinder: ${{ steps.compose.outputs.build_pathfinder }} - build_python: ${{ steps.compose.outputs.build_python }} - test_bindings: ${{ steps.compose.outputs.test_bindings }} - test_core: ${{ steps.compose.outputs.test_core }} - test_pathfinder: ${{ steps.compose.outputs.test_pathfinder }} - pr_merge_base: ${{ steps.filter.outputs.merge_base }} - test_python: ${{ steps.compose.outputs.test_python }} - baseline_run_id: ${{ steps.compose.outputs.baseline_run_id }} - baseline_sha: ${{ steps.compose.outputs.baseline_sha }} + # Missing base artifacts or a skipped path filter fail open to the full pipeline. + core: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.core_source == 'true' }} + build_pathfinder: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' }} + build_bindings: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.python_source == 'true' }} + build_core: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.core_source == 'true' }} + build_python: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.python_source == 'true' }} + test_pathfinder: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.pathfinder_tests == 'true' }} + test_bindings: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.bindings_tests == 'true' || + steps.filter.outputs.test_helpers == 'true' }} + test_core: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.core_source == 'true' || + steps.filter.outputs.core_tests == 'true' || + steps.filter.outputs.test_helpers == 'true' }} + pr_merge_base: ${{ steps.merge-base.outputs.sha }} + test_python: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.core_source == 'true' || + steps.filter.outputs.python_source == 'true' || + steps.filter.outputs.python_tests == 'true' }} + baseline_run_id: >- + ${{ steps.filter.outputs.changes != '' && + steps.baseline.outputs.available == 'true' && + steps.baseline.outputs.run_id || '' }} + baseline_sha: >- + ${{ steps.filter.outputs.changes != '' && + steps.baseline.outputs.available == 'true' && + steps.baseline.outputs.sha || '' }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - # Treeless clone: commit graph is needed for `git merge-base` and - # `git diff --name-only` below, but historical blobs aren't. + # Treeless clone: the commit graph is needed to resolve the PR merge + # base and classify its changed paths, but historical blobs aren't. fetch-depth: 0 filter: blob:none # copy-pr-bot pushes every PR (whether it targets main or a backport # branch such as 12.9.x) to pull-request/, so the base branch # cannot be inferred from github.ref_name. Look it up via the - # upstream PR metadata so the diff below is rooted at the right place. + # upstream PR metadata so change detection is rooted at the right place. - name: Resolve PR base branch id: pr-info if: ${{ startsWith(github.ref_name, 'pull-request/') }} uses: nv-gha-runners/get-pr-info@main - - name: Detect changed paths - id: filter + - name: Resolve PR merge base + id: merge-base if: ${{ startsWith(github.ref_name, 'pull-request/') }} env: # GitHub Actions evaluates step-level `env:` expressions eagerly — @@ -153,25 +208,83 @@ jobs: exit 1 fi - # Diff against the merge base with the PR's actual target branch. - # Disabling rename detection reports both sides of a cross-package - # move, which prevents the source package from being skipped. base=$(git merge-base HEAD "origin/${BASE_REF}") - git diff --no-renames --name-only -z "$base"...HEAD > changed-paths - python ci/tools/compute_ci_plan.py changed-paths >> "$GITHUB_OUTPUT" - echo "merge_base=${base}" >> "$GITHUB_OUTPUT" + echo "sha=${base}" >> "$GITHUB_OUTPUT" - { - echo "### Selective CI changed paths" - echo - tr '\0' '\n' < changed-paths | sed 's/^/- `/' | sed 's/$/`/' - } >> "$GITHUB_STEP_SUMMARY" + - name: Classify changed paths + id: filter + if: ${{ startsWith(github.ref_name, 'pull-request/') }} + uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 + with: + # The workflow runs on copy-pr-bot push branches, so compare the + # checked-out PR head against the base resolved from PR metadata. + base: ${{ steps.merge-base.outputs.sha }} + ref: ${{ github.sha }} + token: '' + predicate-quantifier: some-with-excludes + filters: | + pathfinder_source: + - 'cuda_pathfinder/**' + - '!cuda_pathfinder/docs/**' + - '!cuda_pathfinder/tests/**' + - '!cuda_pathfinder/examples/**' + bindings_source: + - 'cuda_bindings/**' + - '!cuda_bindings/docs/**' + - '!cuda_bindings/tests/**' + - '!cuda_bindings/examples/**' + core_source: + - 'cuda_core/**' + - '!cuda_core/docs/**' + - '!cuda_core/tests/**' + - '!cuda_core/examples/**' + python_source: + - 'cuda_python/**' + - '!cuda_python/docs/**' + - '!cuda_python/tests/**' + - '!cuda_python/examples/**' + pathfinder_tests: + - 'cuda_pathfinder/tests/**' + - 'cuda_pathfinder/examples/**' + bindings_tests: + - 'cuda_bindings/tests/**' + - 'cuda_bindings/examples/**' + - 'benchmarks/cuda_bindings/**' + core_tests: + - 'cuda_core/tests/**' + - 'cuda_core/examples/**' + python_tests: + - 'cuda_python/tests/**' + - 'cuda_python/examples/**' + test_helpers: + - 'cuda_python_test_helpers/**' + # Shared infrastructure and unknown paths run the full pipeline. + # Exclude only paths whose narrower behavior is defined above or + # repository policy/prose files known not to affect artifacts. + force_all: + - '**' + - '!cuda_pathfinder/**' + - '!cuda_bindings/**' + - '!cuda_core/**' + - '!cuda_python/**' + - '!cuda_python_test_helpers/**' + - '!benchmarks/cuda_bindings/**' + - '!.git-blame-ignore-revs' + - '!.gitignore' + - '!AGENTS.md' + - '!CHANGELOG.md' + - '!CODE_OF_CONDUCT.md' + - '!CONTRIBUTING.md' + - '!LICENSE' + - '!README.md' + - '!SECURITY.md' - name: Resolve reusable base artifacts id: baseline if: ${{ startsWith(github.ref_name, 'pull-request/') }} env: BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} + MERGE_BASE: ${{ steps.merge-base.outputs.sha }} GH_TOKEN: ${{ github.token }} run: | set -uo pipefail @@ -186,7 +299,10 @@ jobs: unavailable fi - merge_base=$(git merge-base HEAD "origin/${BASE_REF}") + merge_base="${MERGE_BASE}" + if [[ -z "${merge_base}" ]]; then + unavailable + fi if ! runs=$(gh run list \ --repo "${{ github.repository }}" \ --branch "${BASE_REF}" \ @@ -256,118 +372,6 @@ jobs: echo "Reusable artifacts: run \`${run_id}\` at \`${baseline_sha}\` on \`${BASE_REF}\`." } >> "$GITHUB_STEP_SUMMARY" - - name: Compose gating outputs - id: compose - env: - IS_PR: ${{ startsWith(github.ref_name, 'pull-request/') }} - BASELINE_AVAILABLE: ${{ steps.baseline.outputs.available || 'false' }} - BASELINE_RUN_ID: ${{ steps.baseline.outputs.run_id }} - BASELINE_SHA: ${{ steps.baseline.outputs.sha }} - BINDINGS: ${{ steps.filter.outputs.bindings_source }} - CORE: ${{ steps.filter.outputs.core_source }} - PATHFINDER: ${{ steps.filter.outputs.pathfinder_source }} - PYTHON_META: ${{ steps.filter.outputs.python_source }} - TEST_HELPERS: ${{ steps.filter.outputs.test_helpers }} - SHARED: ${{ steps.filter.outputs.shared }} - BUILD_BINDINGS: ${{ steps.filter.outputs.build_bindings }} - BUILD_CORE: ${{ steps.filter.outputs.build_core }} - BUILD_PATHFINDER: ${{ steps.filter.outputs.build_pathfinder }} - BUILD_PYTHON: ${{ steps.filter.outputs.build_python }} - TEST_BINDINGS: ${{ steps.filter.outputs.test_bindings }} - TEST_CORE: ${{ steps.filter.outputs.test_core }} - TEST_PATHFINDER: ${{ steps.filter.outputs.test_pathfinder }} - TEST_PYTHON: ${{ steps.filter.outputs.test_python }} - run: | - set -euxo pipefail - planner_valid=true - if [[ "${IS_PR}" == "true" ]]; then - for value in \ - "${BINDINGS}" "${CORE}" "${PATHFINDER}" "${PYTHON_META}" \ - "${TEST_HELPERS}" "${SHARED}" \ - "${BUILD_BINDINGS}" "${BUILD_CORE}" "${BUILD_PATHFINDER}" "${BUILD_PYTHON}" \ - "${TEST_BINDINGS}" "${TEST_CORE}" "${TEST_PATHFINDER}" "${TEST_PYTHON}"; do - if [[ "${value}" != "true" && "${value}" != "false" ]]; then - planner_valid=false - fi - done - if [[ "${BASELINE_AVAILABLE}" == "true" && - ( -z "${BASELINE_RUN_ID}" || -z "${BASELINE_SHA}" ) ]]; then - planner_valid=false - fi - fi - - # Non-PR events produce the complete trusted artifact set. PRs also - # run everything when the trusted base artifact inventory is absent - # or the planner did not emit a complete boolean result. - if [[ "${IS_PR}" != "true" || - "${BASELINE_AVAILABLE}" != "true" || - "${planner_valid}" != "true" ]]; then - bindings=true - core=true - pathfinder=true - python_meta=true - test_helpers=true - shared=true - build_bindings=true - build_core=true - build_pathfinder=true - build_python=true - test_bindings=true - test_core=true - test_pathfinder=true - test_python=true - baseline_run_id="" - baseline_sha="" - else - bindings="${BINDINGS}" - core="${CORE}" - pathfinder="${PATHFINDER}" - python_meta="${PYTHON_META}" - test_helpers="${TEST_HELPERS}" - shared="${SHARED}" - build_bindings="${BUILD_BINDINGS}" - build_core="${BUILD_CORE}" - build_pathfinder="${BUILD_PATHFINDER}" - build_python="${BUILD_PYTHON}" - test_bindings="${TEST_BINDINGS}" - test_core="${TEST_CORE}" - test_pathfinder="${TEST_PATHFINDER}" - test_python="${TEST_PYTHON}" - baseline_run_id="${BASELINE_RUN_ID}" - baseline_sha="${BASELINE_SHA}" - fi - - { - echo "bindings=${bindings}" - echo "core=${core}" - echo "pathfinder=${pathfinder}" - echo "python_meta=${python_meta}" - echo "test_helpers=${test_helpers}" - echo "shared=${shared}" - echo "build_bindings=${build_bindings}" - echo "build_core=${build_core}" - echo "build_pathfinder=${build_pathfinder}" - echo "build_python=${build_python}" - echo "test_bindings=${test_bindings}" - echo "test_core=${test_core}" - echo "test_pathfinder=${test_pathfinder}" - echo "test_python=${test_python}" - echo "baseline_run_id=${baseline_run_id}" - echo "baseline_sha=${baseline_sha}" - } >> "$GITHUB_OUTPUT" - - { - echo - echo "### Effective package plan" - echo - echo "| Package | Build | Test |" - echo "| --- | --- | --- |" - echo "| cuda-pathfinder | ${build_pathfinder} | ${test_pathfinder} |" - echo "| cuda-bindings | ${build_bindings} | ${test_bindings} |" - echo "| cuda-core | ${build_core} | ${test_core} |" - echo "| cuda-python | ${build_python} | ${test_python} |" - } >> "$GITHUB_STEP_SUMMARY" - api-check-core-vs-release: name: API check (cuda_core vs. latest release) if: >- @@ -834,7 +838,7 @@ jobs: needs.detect-changes.outputs.test_bindings == 'true' || needs.detect-changes.outputs.test_core == 'true' || needs.detect-changes.outputs.test_python == 'true' }}" - core_changed="${{ needs.detect-changes.outputs.core == 'true' }}" + run_core_api_check="${{ needs.detect-changes.outputs.core == 'true' }}" is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { @@ -880,13 +884,14 @@ jobs: check_result "test-linux-aarch64" "$expected" "${{ needs.test-linux-aarch64.result }}" check_result "test-windows" "$expected" "${{ needs.test-windows.result }}" - # API compatibility checks only run for cuda_core source changes. + # API compatibility checks run for cuda_core source changes and for + # conservative full runs when reusable base artifacts are unavailable. expected="skipped" - if [[ "$core_changed" == "true" ]]; then expected="success"; fi + if [[ "$run_core_api_check" == "true" ]]; then expected="success"; fi check_result "api-check-core-vs-release" "$expected" "${{ needs.api-check-core-vs-release.result }}" expected="skipped" - if [[ "$is_pr" == "true" && "$core_changed" == "true" ]]; then expected="success"; fi + if [[ "$is_pr" == "true" && "$run_core_api_check" == "true" ]]; then expected="success"; fi check_result "api-check-core-vs-base" "$expected" "${{ needs.api-check-core-vs-base.result }}" [[ "$status" == "success" ]] diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py deleted file mode 100644 index 7095514b73e..00000000000 --- a/ci/tools/compute_ci_plan.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Compute the package build and test closure for a set of changed paths.""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -PACKAGES = { - "cuda_pathfinder": "pathfinder", - "cuda_bindings": "bindings", - "cuda_core": "core", - "cuda_python": "python", -} - -SHARED_PREFIXES = ( - ".github/", - "ci/", - "scripts/", - "toolshed/", -) - -SHARED_FILES = { - ".pre-commit-config.yaml", - "conftest.py", - "pixi.lock", - "pixi.toml", - "pytest.ini", - "ruff.toml", -} - -KNOWN_REPOSITORY_FILES = { - ".git-blame-ignore-revs", - ".gitignore", - "AGENTS.md", - "CHANGELOG.md", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "README.md", - "SECURITY.md", -} - - -def _bool(value: bool) -> str: - return str(value).lower() - - -def _read_paths(path: Path) -> list[str]: - return [value.decode("utf-8", errors="surrogateescape") for value in path.read_bytes().split(b"\0") if value] - - -def compute_plan(paths: list[str]) -> dict[str, bool]: - source = dict.fromkeys(PACKAGES.values(), False) - tests = dict.fromkeys(PACKAGES.values(), False) - docs = False - test_helpers = False - shared = False - unknown = False - - for path in paths: - package_dir, separator, relative_path = path.partition("/") - package = PACKAGES.get(package_dir) - if package is not None and separator: - if relative_path.startswith("docs/"): - docs = True - elif relative_path.startswith(("tests/", "examples/")): - tests[package] = True - else: - source[package] = True - continue - - if path.startswith("cuda_python_test_helpers/"): - test_helpers = True - elif path.startswith("benchmarks/cuda_bindings/"): - tests["bindings"] = True - elif path.startswith(SHARED_PREFIXES) or path in SHARED_FILES: - shared = True - elif path in KNOWN_REPOSITORY_FILES: - # Repository policy and prose files do not affect package artifacts. - continue - else: - unknown = True - - full = shared or unknown - - build_pathfinder = full or source["pathfinder"] - # Development cuda-python wheels exactly pin cuda-bindings, so a - # metapackage change needs a matching bindings artifact for its smoke test. - build_bindings = full or source["pathfinder"] or source["bindings"] or source["python"] - build_core = full or source["pathfinder"] or source["bindings"] or source["core"] - # A core-only change can reuse the baseline cuda-python wheel: rebuilding - # it would also require rebuilding the exact-version cuda-bindings pin. - build_python = full or source["pathfinder"] or source["bindings"] or source["python"] - - test_pathfinder = full or source["pathfinder"] or tests["pathfinder"] - test_bindings = full or source["pathfinder"] or source["bindings"] or tests["bindings"] or test_helpers - test_core = full or source["pathfinder"] or source["bindings"] or source["core"] or tests["core"] or test_helpers - test_python = ( - full or source["pathfinder"] or source["bindings"] or source["core"] or source["python"] or tests["python"] - ) - - return { - "shared": shared, - "unknown": unknown, - "docs": docs, - "test_helpers": test_helpers, - "pathfinder_source": source["pathfinder"], - "bindings_source": source["bindings"], - "core_source": source["core"], - "python_source": source["python"], - "pathfinder_tests": tests["pathfinder"], - "bindings_tests": tests["bindings"], - "core_tests": tests["core"], - "python_tests": tests["python"], - "build_pathfinder": build_pathfinder, - "build_bindings": build_bindings, - "build_core": build_core, - "build_python": build_python, - "test_pathfinder": test_pathfinder, - "test_bindings": test_bindings, - "test_core": test_core, - "test_python": test_python, - } - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument( - "paths_file", - type=Path, - help="NUL-separated changed-path list produced by git diff --name-only -z", - ) - args = parser.parse_args() - - plan = compute_plan(_read_paths(args.paths_file)) - for key, value in plan.items(): - print(f"{key}={_bool(value)}") - - -if __name__ == "__main__": - main() From 42fe412d683de6f3b2cf7becc72e06d0ddca2d57 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Fri, 14 Aug 2026 00:25:40 -0400 Subject: [PATCH 03/13] ci: refine selective path classification --- .github/workflows/ci.yml | 113 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 105 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16ab65b3db2..9120b2e062f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,8 +86,8 @@ jobs: # downstream module. A change to cuda_bindings forces rebuild of cuda_core. # A change to cuda_core alone skips rebuilding/retesting cuda_bindings and # cuda_pathfinder, but still retests the downstream cuda-python metapackage. - # CI/filter changes are shared by design, so this implementation runs the - # full pipeline; exercise selective cases in follow-up package-only PRs. + # Shared build/orchestration changes run the full pipeline; test-only CI + # infrastructure runs every test suite without rebuilding package wheels. # On push to main, tag refs, schedule, or workflow_dispatch events we # unconditionally run everything because there is no meaningful "changed # paths" baseline for those events. @@ -102,6 +102,7 @@ jobs: core: >- ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || steps.filter.outputs.core_source == 'true' }} build_pathfinder: >- ${{ steps.baseline.outputs.available != 'true' || @@ -133,12 +134,14 @@ jobs: ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || steps.filter.outputs.pathfinder_source == 'true' || steps.filter.outputs.pathfinder_tests == 'true' }} test_bindings: >- ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || steps.filter.outputs.pathfinder_source == 'true' || steps.filter.outputs.bindings_source == 'true' || steps.filter.outputs.bindings_tests == 'true' || @@ -147,6 +150,7 @@ jobs: ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || steps.filter.outputs.pathfinder_source == 'true' || steps.filter.outputs.bindings_source == 'true' || steps.filter.outputs.core_source == 'true' || @@ -157,6 +161,7 @@ jobs: ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || steps.filter.outputs.pathfinder_source == 'true' || steps.filter.outputs.bindings_source == 'true' || steps.filter.outputs.core_source == 'true' || @@ -228,39 +233,79 @@ jobs: - '!cuda_pathfinder/docs/**' - '!cuda_pathfinder/tests/**' - '!cuda_pathfinder/examples/**' + - '!cuda_pathfinder/pixi.lock' + - '!cuda_pathfinder/pixi.toml' + - '!cuda_pathfinder/AGENTS.md' + - '!cuda_pathfinder/CLAUDE.md' bindings_source: - 'cuda_bindings/**' - '!cuda_bindings/docs/**' - '!cuda_bindings/tests/**' - '!cuda_bindings/examples/**' + - '!cuda_bindings/pixi.lock' + - '!cuda_bindings/pixi.toml' + - '!cuda_bindings/AGENTS.md' + - '!cuda_bindings/CLAUDE.md' core_source: - 'cuda_core/**' - '!cuda_core/docs/**' - '!cuda_core/tests/**' - '!cuda_core/examples/**' + - '!cuda_core/pixi.lock' + - '!cuda_core/pixi.toml' + - '!cuda_core/pytest.ini' + - '!cuda_core/AGENTS.md' + - '!cuda_core/CLAUDE.md' python_source: - 'cuda_python/**' - '!cuda_python/docs/**' - '!cuda_python/tests/**' - '!cuda_python/examples/**' + - '!cuda_python/AGENTS.md' + - '!cuda_python/CLAUDE.md' + # cuda_python/README.md is a symlink to this packaging input. + - 'README.md' pathfinder_tests: - 'cuda_pathfinder/tests/**' - 'cuda_pathfinder/examples/**' + - '!cuda_pathfinder/**/AGENTS.md' + - '!cuda_pathfinder/**/CLAUDE.md' bindings_tests: - 'cuda_bindings/tests/**' - 'cuda_bindings/examples/**' - 'benchmarks/cuda_bindings/**' + - '!benchmarks/cuda_bindings/pixi.lock' + - '!benchmarks/cuda_bindings/pixi.toml' + - '!cuda_bindings/**/AGENTS.md' + - '!cuda_bindings/**/CLAUDE.md' core_tests: - 'cuda_core/tests/**' - 'cuda_core/examples/**' + - 'cuda_core/pytest.ini' + - '!cuda_core/**/AGENTS.md' + - '!cuda_core/**/CLAUDE.md' python_tests: - 'cuda_python/tests/**' - 'cuda_python/examples/**' + - '!cuda_python/**/AGENTS.md' + - '!cuda_python/**/CLAUDE.md' test_helpers: - - 'cuda_python_test_helpers/**' + - 'cuda_python_test_helpers/cuda_python_test_helpers/**' + # These files configure or implement wheel tests, but do not + # change any package artifact. + all_tests: + - '.github/workflows/test-wheel-linux.yml' + - '.github/workflows/test-wheel-windows.yml' + - 'ci/test-matrix.yml' + - 'ci/tools/configure_driver_mode.ps1' + - 'ci/tools/guess_latest.sh' + - 'ci/tools/install_gpu_driver.ps1' + - 'ci/tools/install_gpu_driver.sh' + - 'ci/tools/run-tests' + - 'ci/tools/setup-sanitizer' # Shared infrastructure and unknown paths run the full pipeline. - # Exclude only paths whose narrower behavior is defined above or - # repository policy/prose files known not to affect artifacts. + # Exclude paths classified above and paths consumed only by an + # independent or unconditional CI job. force_all: - '**' - '!cuda_pathfinder/**' @@ -269,15 +314,67 @@ jobs: - '!cuda_python/**' - '!cuda_python_test_helpers/**' - '!benchmarks/cuda_bindings/**' - - '!.git-blame-ignore-revs' + - '!benchmarks/cuda_core/**' + - '!.agents/**' + - '!.coveragerc' - '!.gitignore' + - '!.pre-commit-config.yaml' + - '!.spdx-ignore' - '!AGENTS.md' - - '!CHANGELOG.md' - - '!CODE_OF_CONDUCT.md' + - '!CLAUDE.md' - '!CONTRIBUTING.md' - '!LICENSE' - '!README.md' - '!SECURITY.md' + - '!context7.json' + - '!greptile.json' + - '!pixi.lock' + - '!pixi.toml' + - '!pytest.ini' + - '!ruff.toml' + - '!toolshed/**' + - '!.github/ISSUE_TEMPLATE/**' + - '!.github/PULL_REQUEST_TEMPLATE.md' + - '!.github/RELEASE-core.md' + - '!.github/actionlint.yaml' + - '!.github/actions/doc_preview/**' + - '!.github/actions/get_pr_number/**' + - '!.github/copy-pr-bot.yaml' + - '!.github/dependabot.yml' + - '!.github/labeler.yml' + - '!.github/workflows/backport.yml' + - '!.github/workflows/bandit.yml' + - '!.github/workflows/build-docs.yml' + - '!.github/workflows/ci-nightly.yml' + - '!.github/workflows/ci-pixi-source-test.yml' + - '!.github/workflows/cleanup-pr-previews.yml' + - '!.github/workflows/coverage.yml' + - '!.github/workflows/pr-auto-label.yml' + - '!.github/workflows/pr-metadata-check.yml' + - '!.github/workflows/release-cuda-pathfinder.yml' + - '!.github/workflows/release-upload.yml' + - '!.github/workflows/release.yml' + - '!.github/workflows/security-suite.yml' + - '!.github/workflows/test-wheel-linux.yml' + - '!.github/workflows/test-wheel-windows.yml' + - '!.github/workflows/triagelabel.yml' + - '!ci/.ci-pipeline-regen.md' + - '!ci/ci-pipeline.svg' + - '!ci/cleanup-pr-previews' + - '!ci/test-matrix.yml' + - '!ci/tools/check_mempool_hygiene.py' + - '!ci/tools/check_pixi_cuda_version.py' + - '!ci/tools/check_release_notes.py' + - '!ci/tools/configure_driver_mode.ps1' + - '!ci/tools/download-wheels' + - '!ci/tools/guess_latest.sh' + - '!ci/tools/install_gpu_driver.ps1' + - '!ci/tools/install_gpu_driver.sh' + - '!ci/tools/run-tests' + - '!ci/tools/run_pytest_with_stack.py' + - '!ci/tools/setup-sanitizer' + - '!ci/tools/tests/**' + - '!ci/tools/validate-release-wheels' - name: Resolve reusable base artifacts id: baseline From eaed263c78ef65bf401a3607ec46e6cb734d0616 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Fri, 14 Aug 2026 13:16:43 -0400 Subject: [PATCH 04/13] ci: compute selective workplan in Python --- .github/workflows/build-wheel.yml | 134 ++++---- .github/workflows/ci.yml | 389 ++++------------------- .github/workflows/test-sdist-linux.yml | 52 ++- .github/workflows/test-sdist-windows.yml | 46 ++- .github/workflows/test-wheel-linux.yml | 75 +++-- .github/workflows/test-wheel-windows.yml | 69 ++-- ci/tools/compute_ci_plan.py | 241 ++++++++++++++ ci/tools/tests/test_compute_ci_plan.py | 100 ++++++ 8 files changed, 567 insertions(+), 539 deletions(-) create mode 100644 ci/tools/compute_ci_plan.py create mode 100644 ci/tools/tests/test_compute_ci_plan.py diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 390d6f88ae2..15e84819f1d 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -14,35 +14,8 @@ on: prev-cuda-version: required: true type: string - build-pathfinder: - required: false - type: boolean - default: true - build-bindings: - required: false - type: boolean - default: true - build-core: - required: false - type: boolean - default: true - build-python: - required: false - type: boolean - default: true - test-bindings: - required: false - type: boolean - default: true - test-core: - required: false - type: boolean - default: true - baseline-run-id: - required: false - type: string - default: "" - baseline-sha: + workplan: + description: JSON workplan. An empty value builds and tests everything. required: false type: string default: "" @@ -57,6 +30,15 @@ permissions: jobs: build: + env: + BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} + BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} + BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} + BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} + TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} + TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} + BASELINE_RUN_ID: ${{ inputs.workplan != '' && fromJSON(inputs.workplan).baseline.run_id || '' }} + BASELINE_SHA: ${{ inputs.workplan != '' && fromJSON(inputs.workplan).baseline.sha || '' }} strategy: fail-fast: false matrix: @@ -83,7 +65,7 @@ jobs: filter: blob:none - name: Install latest rapidsai/sccache - if: ${{ startsWith(inputs.host-platform, 'linux') && (inputs.build-bindings || inputs.build-core) }} + if: ${{ startsWith(inputs.host-platform, 'linux') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} run: | curl -fsSL "https://github.com/rapidsai/sccache/releases/latest/download/sccache-$(uname -m)-unknown-linux-musl.tar.gz" \ | sudo tar -C /usr/local/bin -xvzf - --wildcards --strip-components=1 -x '*/sccache' @@ -91,7 +73,7 @@ jobs: # xref: https://github.com/orgs/community/discussions/42856#discussioncomment-7678867 - name: Adding addtional GHA cache-related env vars - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: actions/github-script@v9 with: script: | @@ -118,13 +100,13 @@ jobs: python-version: "3.12" - name: Set up MSVC - if: ${{ startsWith(inputs.host-platform, 'win') && (inputs.build-bindings || inputs.build-core || inputs.test-bindings || inputs.test-core) }} + if: ${{ startsWith(inputs.host-platform, 'win') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') }} uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 - name: Set up yq # GitHub made an unprofessional decision to not provide it in their Windows VMs, # see https://github.com/actions/runner-images/issues/7443. - if: ${{ startsWith(inputs.host-platform, 'win') && inputs.build-core }} + if: ${{ startsWith(inputs.host-platform, 'win') && env.BUILD_CORE == 'true' }} env: YQ_VERSION: v4.52.5 YQ_SHA256: 47594981f3848a4b4447494adeca9555f908f7cf0a89c4da3fd0243a4631da1c @@ -162,20 +144,20 @@ jobs: # To keep the build workflow simple, all matrix jobs will build a wheel for later use within this workflow. - name: Build and check cuda.pathfinder wheel - if: ${{ inputs.build-pathfinder }} + if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | pushd cuda_pathfinder pip wheel -v --no-deps . popd - name: Download reusable cuda.pathfinder wheel - if: ${{ !inputs.build-pathfinder }} + if: ${{ env.BUILD_PATHFINDER != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel path: cuda_pathfinder github-token: ${{ github.token }} - run-id: ${{ inputs.baseline-run-id }} + run-id: ${{ env.BASELINE_RUN_ID }} - name: List the cuda.pathfinder artifacts directory run: | @@ -190,12 +172,12 @@ jobs: # We only need/want a single pure python wheel, pick linux-64 index 0. # This is what we will use for testing & releasing. - name: Check cuda.pathfinder wheel - if: ${{ inputs.build-pathfinder && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} + if: ${{ env.BUILD_PATHFINDER == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} run: | twine check --strict cuda_pathfinder/*.whl - name: Constrain builds to the local cuda.pathfinder wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 @@ -217,7 +199,7 @@ jobs: if-no-files-found: error - name: Set up mini CTK - if: ${{ inputs.build-bindings || inputs.build-core || inputs.test-bindings || inputs.test-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -225,7 +207,7 @@ jobs: cuda-version: ${{ inputs.cuda-version }} - name: Build cuda.bindings wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 with: package-dir: ./cuda_bindings/ @@ -271,7 +253,7 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.bindings) - if: ${{ inputs.build-bindings && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_BINDINGS == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_bindings.json @@ -279,13 +261,13 @@ jobs: build-step: "Build cuda.bindings wheel" - name: Download reusable cuda.bindings wheel - if: ${{ !inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ env.CUDA_BINDINGS_ARTIFACT_BASENAME }}-${{ inputs.baseline-sha }} + name: ${{ env.CUDA_BINDINGS_ARTIFACT_BASENAME }}-${{ env.BASELINE_SHA }} path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} github-token: ${{ github.token }} - run-id: ${{ inputs.baseline-run-id }} + run-id: ${{ env.BASELINE_RUN_ID }} - name: List the cuda.bindings artifacts directory run: | @@ -298,12 +280,12 @@ jobs: ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} - name: Check cuda.bindings wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | twine check --strict ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl - name: Constrain cuda.core to the local cuda.bindings wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) bindings_wheels=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-"${BUILD_CUDA_MAJOR}".*.whl) @@ -332,7 +314,7 @@ jobs: if-no-files-found: error - name: Build cuda.core wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 with: package-dir: ./cuda_core/ @@ -380,7 +362,7 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.core) - if: ${{ inputs.build-core && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_CORE == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_core.json @@ -388,7 +370,7 @@ jobs: build-step: "Build cuda.core wheel" - name: List the cuda.core artifacts directory and rename - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == win* ]]; then export CHOWN=chown @@ -411,17 +393,17 @@ jobs: ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - name: Download reusable cuda.core wheel - if: ${{ !inputs.build-core }} + if: ${{ env.BUILD_CORE != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ env.CUDA_CORE_ARTIFACT_BASENAME }}-${{ inputs.baseline-sha }} + name: ${{ env.CUDA_CORE_ARTIFACT_BASENAME }}-${{ env.BASELINE_SHA }} path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} github-token: ${{ github.token }} - run-id: ${{ inputs.baseline-run-id }} + run-id: ${{ env.BASELINE_RUN_ID }} # We only need/want a single pure python wheel, pick linux-64 index 0. - name: Build and check cuda-python wheel - if: ${{ inputs.build-python && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} + if: ${{ env.BUILD_PYTHON == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} run: | pushd cuda_python pip wheel -v --no-deps . @@ -429,13 +411,13 @@ jobs: popd - name: Download reusable cuda-python wheel - if: ${{ !inputs.build-python && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} + if: ${{ env.BUILD_PYTHON != 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel path: cuda_python github-token: ${{ github.token }} - run-id: ${{ inputs.baseline-run-id }} + run-id: ${{ env.BASELINE_RUN_ID }} - name: List the cuda-python artifacts directory if: ${{ strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} @@ -458,7 +440,7 @@ jobs: - name: Set up Python id: setup-python2 - if: ${{ inputs.test-bindings || inputs.test-core }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} @@ -466,17 +448,17 @@ jobs: allow-prereleases: ${{ startsWith(matrix.python-version, '3.15') }} - name: Enable Scientific Python Nightly Wheels for Python 3.15 - if: ${{ (inputs.test-bindings || inputs.test-core) && startsWith(matrix.python-version, '3.15') }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && startsWith(matrix.python-version, '3.15') }} run: | echo "PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" >> "$GITHUB_ENV" echo "PIP_ONLY_BINARY=numpy" >> "$GITHUB_ENV" - name: verify free-threaded build - if: ${{ (inputs.test-bindings || inputs.test-core) && endsWith(matrix.python-version, 't') }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && endsWith(matrix.python-version, 't') }} run: python -c 'import sys; assert not sys._is_gil_enabled()' - name: Set up Python include paths - if: ${{ inputs.test-bindings || inputs.test-core }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == linux* ]]; then echo "CPLUS_INCLUDE_PATH=${Python3_ROOT_DIR}/include/python${{ matrix.python-version }}" >> $GITHUB_ENV @@ -487,19 +469,19 @@ jobs: echo "PY_EXT_SUFFIX=$(python -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")" >> $GITHUB_ENV - name: Install cuda.pathfinder (required for next step) - if: ${{ inputs.test-bindings || inputs.test-core }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} run: | pip install cuda_pathfinder/*.whl - name: Hide GNU link.exe so Meson finds MSVC link.exe - if: ${{ startsWith(inputs.host-platform, 'win') && (inputs.test-bindings || inputs.test-core) }} + if: ${{ startsWith(inputs.host-platform, 'win') && (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') }} run: | if [ -f "/c/Program Files/Git/usr/bin/link.exe" ]; then mv "/c/Program Files/Git/usr/bin/link.exe" "/c/Program Files/Git/usr/bin/link.exe.bak" fi - name: Build cuda.bindings Cython tests - if: ${{ inputs.test-bindings }} + if: ${{ env.TEST_BINDINGS == 'true' }} run: | pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl --group ./cuda_bindings/pyproject.toml:test pushd ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }} @@ -507,7 +489,7 @@ jobs: popd - name: Upload cuda.bindings Cython tests - if: ${{ inputs.test-bindings }} + if: ${{ env.TEST_BINDINGS == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests @@ -515,10 +497,10 @@ jobs: if-no-files-found: error - name: Build cuda.core Cython tests - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl - if ${{ inputs.build-core }}; then + if ${{ env.BUILD_CORE == 'true' }}; then core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}" -maxdepth 1 -type f -name '*.whl' -print -quit) else core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" -maxdepth 1 -type f -name '*.whl' -print -quit) @@ -533,7 +515,7 @@ jobs: popd - name: Upload cuda.core Cython tests - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests @@ -542,7 +524,7 @@ jobs: # Note: This overwrites CUDA_PATH etc - name: Set up mini CTK - if: ${{ inputs.build-core || inputs.test-core }} + if: ${{ env.BUILD_CORE == 'true' || env.TEST_CORE == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -551,13 +533,13 @@ jobs: cuda-path: "./cuda_toolkit_prev" - name: Build cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | nvcc --version python "${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/build_test_binaries.py" - name: Upload cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries @@ -568,7 +550,7 @@ jobs: if-no-files-found: error - name: Download cuda.bindings build artifacts from the prior branch - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -597,7 +579,7 @@ jobs: rmdir $OLD_BASENAME - name: Constrain previous cuda.core to the downloaded cuda.bindings wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) bindings_wheels=(cuda_bindings/dist-prev/cuda_bindings-"${BUILD_PREV_CUDA_MAJOR}".*.whl) @@ -619,7 +601,7 @@ jobs: } | tee wheel-constraints/cuda-core-prev.txt - name: Build cuda.core wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 with: package-dir: ./cuda_core/ @@ -667,7 +649,7 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.core prev) - if: ${{ inputs.build-core && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_CORE == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_core_prev.json @@ -675,7 +657,7 @@ jobs: build-step: "Build cuda.core wheel" - name: List the cuda.core artifacts directory and rename - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == win* ]]; then export CHOWN=chown @@ -699,7 +681,7 @@ jobs: ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - name: Merge cuda.core wheels - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | pip install wheel python ci/tools/merge_cuda_core_wheels.py \ @@ -708,7 +690,7 @@ jobs: --output-dir "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" - name: Check cuda.core wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | twine check --strict ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9120b2e062f..99a8904a1b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,83 +98,7 @@ jobs: contents: read pull-requests: read outputs: - # Missing base artifacts or a skipped path filter fail open to the full pipeline. - core: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.core_source == 'true' }} - build_pathfinder: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.pathfinder_source == 'true' }} - build_bindings: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.bindings_source == 'true' || - steps.filter.outputs.python_source == 'true' }} - build_core: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.bindings_source == 'true' || - steps.filter.outputs.core_source == 'true' }} - build_python: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.bindings_source == 'true' || - steps.filter.outputs.python_source == 'true' }} - test_pathfinder: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.all_tests == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.pathfinder_tests == 'true' }} - test_bindings: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.all_tests == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.bindings_source == 'true' || - steps.filter.outputs.bindings_tests == 'true' || - steps.filter.outputs.test_helpers == 'true' }} - test_core: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.all_tests == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.bindings_source == 'true' || - steps.filter.outputs.core_source == 'true' || - steps.filter.outputs.core_tests == 'true' || - steps.filter.outputs.test_helpers == 'true' }} - pr_merge_base: ${{ steps.merge-base.outputs.sha }} - test_python: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.all_tests == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.bindings_source == 'true' || - steps.filter.outputs.core_source == 'true' || - steps.filter.outputs.python_source == 'true' || - steps.filter.outputs.python_tests == 'true' }} - baseline_run_id: >- - ${{ steps.filter.outputs.changes != '' && - steps.baseline.outputs.available == 'true' && - steps.baseline.outputs.run_id || '' }} - baseline_sha: >- - ${{ steps.filter.outputs.changes != '' && - steps.baseline.outputs.available == 'true' && - steps.baseline.outputs.sha || '' }} + workplan: ${{ steps.workplan.outputs.workplan }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -216,166 +140,6 @@ jobs: base=$(git merge-base HEAD "origin/${BASE_REF}") echo "sha=${base}" >> "$GITHUB_OUTPUT" - - name: Classify changed paths - id: filter - if: ${{ startsWith(github.ref_name, 'pull-request/') }} - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 - with: - # The workflow runs on copy-pr-bot push branches, so compare the - # checked-out PR head against the base resolved from PR metadata. - base: ${{ steps.merge-base.outputs.sha }} - ref: ${{ github.sha }} - token: '' - predicate-quantifier: some-with-excludes - filters: | - pathfinder_source: - - 'cuda_pathfinder/**' - - '!cuda_pathfinder/docs/**' - - '!cuda_pathfinder/tests/**' - - '!cuda_pathfinder/examples/**' - - '!cuda_pathfinder/pixi.lock' - - '!cuda_pathfinder/pixi.toml' - - '!cuda_pathfinder/AGENTS.md' - - '!cuda_pathfinder/CLAUDE.md' - bindings_source: - - 'cuda_bindings/**' - - '!cuda_bindings/docs/**' - - '!cuda_bindings/tests/**' - - '!cuda_bindings/examples/**' - - '!cuda_bindings/pixi.lock' - - '!cuda_bindings/pixi.toml' - - '!cuda_bindings/AGENTS.md' - - '!cuda_bindings/CLAUDE.md' - core_source: - - 'cuda_core/**' - - '!cuda_core/docs/**' - - '!cuda_core/tests/**' - - '!cuda_core/examples/**' - - '!cuda_core/pixi.lock' - - '!cuda_core/pixi.toml' - - '!cuda_core/pytest.ini' - - '!cuda_core/AGENTS.md' - - '!cuda_core/CLAUDE.md' - python_source: - - 'cuda_python/**' - - '!cuda_python/docs/**' - - '!cuda_python/tests/**' - - '!cuda_python/examples/**' - - '!cuda_python/AGENTS.md' - - '!cuda_python/CLAUDE.md' - # cuda_python/README.md is a symlink to this packaging input. - - 'README.md' - pathfinder_tests: - - 'cuda_pathfinder/tests/**' - - 'cuda_pathfinder/examples/**' - - '!cuda_pathfinder/**/AGENTS.md' - - '!cuda_pathfinder/**/CLAUDE.md' - bindings_tests: - - 'cuda_bindings/tests/**' - - 'cuda_bindings/examples/**' - - 'benchmarks/cuda_bindings/**' - - '!benchmarks/cuda_bindings/pixi.lock' - - '!benchmarks/cuda_bindings/pixi.toml' - - '!cuda_bindings/**/AGENTS.md' - - '!cuda_bindings/**/CLAUDE.md' - core_tests: - - 'cuda_core/tests/**' - - 'cuda_core/examples/**' - - 'cuda_core/pytest.ini' - - '!cuda_core/**/AGENTS.md' - - '!cuda_core/**/CLAUDE.md' - python_tests: - - 'cuda_python/tests/**' - - 'cuda_python/examples/**' - - '!cuda_python/**/AGENTS.md' - - '!cuda_python/**/CLAUDE.md' - test_helpers: - - 'cuda_python_test_helpers/cuda_python_test_helpers/**' - # These files configure or implement wheel tests, but do not - # change any package artifact. - all_tests: - - '.github/workflows/test-wheel-linux.yml' - - '.github/workflows/test-wheel-windows.yml' - - 'ci/test-matrix.yml' - - 'ci/tools/configure_driver_mode.ps1' - - 'ci/tools/guess_latest.sh' - - 'ci/tools/install_gpu_driver.ps1' - - 'ci/tools/install_gpu_driver.sh' - - 'ci/tools/run-tests' - - 'ci/tools/setup-sanitizer' - # Shared infrastructure and unknown paths run the full pipeline. - # Exclude paths classified above and paths consumed only by an - # independent or unconditional CI job. - force_all: - - '**' - - '!cuda_pathfinder/**' - - '!cuda_bindings/**' - - '!cuda_core/**' - - '!cuda_python/**' - - '!cuda_python_test_helpers/**' - - '!benchmarks/cuda_bindings/**' - - '!benchmarks/cuda_core/**' - - '!.agents/**' - - '!.coveragerc' - - '!.gitignore' - - '!.pre-commit-config.yaml' - - '!.spdx-ignore' - - '!AGENTS.md' - - '!CLAUDE.md' - - '!CONTRIBUTING.md' - - '!LICENSE' - - '!README.md' - - '!SECURITY.md' - - '!context7.json' - - '!greptile.json' - - '!pixi.lock' - - '!pixi.toml' - - '!pytest.ini' - - '!ruff.toml' - - '!toolshed/**' - - '!.github/ISSUE_TEMPLATE/**' - - '!.github/PULL_REQUEST_TEMPLATE.md' - - '!.github/RELEASE-core.md' - - '!.github/actionlint.yaml' - - '!.github/actions/doc_preview/**' - - '!.github/actions/get_pr_number/**' - - '!.github/copy-pr-bot.yaml' - - '!.github/dependabot.yml' - - '!.github/labeler.yml' - - '!.github/workflows/backport.yml' - - '!.github/workflows/bandit.yml' - - '!.github/workflows/build-docs.yml' - - '!.github/workflows/ci-nightly.yml' - - '!.github/workflows/ci-pixi-source-test.yml' - - '!.github/workflows/cleanup-pr-previews.yml' - - '!.github/workflows/coverage.yml' - - '!.github/workflows/pr-auto-label.yml' - - '!.github/workflows/pr-metadata-check.yml' - - '!.github/workflows/release-cuda-pathfinder.yml' - - '!.github/workflows/release-upload.yml' - - '!.github/workflows/release.yml' - - '!.github/workflows/security-suite.yml' - - '!.github/workflows/test-wheel-linux.yml' - - '!.github/workflows/test-wheel-windows.yml' - - '!.github/workflows/triagelabel.yml' - - '!ci/.ci-pipeline-regen.md' - - '!ci/ci-pipeline.svg' - - '!ci/cleanup-pr-previews' - - '!ci/test-matrix.yml' - - '!ci/tools/check_mempool_hygiene.py' - - '!ci/tools/check_pixi_cuda_version.py' - - '!ci/tools/check_release_notes.py' - - '!ci/tools/configure_driver_mode.ps1' - - '!ci/tools/download-wheels' - - '!ci/tools/guess_latest.sh' - - '!ci/tools/install_gpu_driver.ps1' - - '!ci/tools/install_gpu_driver.sh' - - '!ci/tools/run-tests' - - '!ci/tools/run_pytest_with_stack.py' - - '!ci/tools/setup-sanitizer' - - '!ci/tools/tests/**' - - '!ci/tools/validate-release-wheels' - - name: Resolve reusable base artifacts id: baseline if: ${{ startsWith(github.ref_name, 'pull-request/') }} @@ -408,7 +172,7 @@ jobs: --workflow ci.yml \ --status success \ --limit 1 \ - --json databaseId,headSha,createdAt); then + --json databaseId,headSha); then unavailable fi @@ -469,11 +233,41 @@ jobs: echo "Reusable artifacts: run \`${run_id}\` at \`${baseline_sha}\` on \`${BASE_REF}\`." } >> "$GITHUB_STEP_SUMMARY" + - name: Test CI workplan planner + run: python3 -m unittest ci/tools/tests/test_compute_ci_plan.py + + - name: Compute CI workplan + id: workplan + env: + MERGE_BASE: ${{ steps.merge-base.outputs.sha }} + BASELINE_AVAILABLE: ${{ steps.baseline.outputs.available }} + BASELINE_RUN_ID: ${{ steps.baseline.outputs.run_id }} + BASELINE_SHA: ${{ steps.baseline.outputs.sha }} + run: | + set -euo pipefail + args=(--head "$GITHUB_SHA") + if [[ -n "$MERGE_BASE" ]]; then + args+=(--merge-base "$MERGE_BASE") + fi + if [[ "$BASELINE_AVAILABLE" == "true" ]]; then + args+=(--baseline-run-id "$BASELINE_RUN_ID" --baseline-sha "$BASELINE_SHA") + fi + + workplan=$(python3 ci/tools/compute_ci_plan.py "${args[@]}") + echo "workplan=$workplan" >> "$GITHUB_OUTPUT" + { + echo + echo "### CI workplan" + echo '```json' + jq . <<< "$workplan" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + api-check-core-vs-release: name: API check (cuda_core vs. latest release) if: >- ${{ !fromJSON(needs.should-skip.outputs.skip) && - fromJSON(needs.detect-changes.outputs.core) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks }} runs-on: ubuntu-latest needs: - should-skip @@ -524,7 +318,7 @@ jobs: if: >- ${{ startsWith(github.ref_name, 'pull-request/') && !fromJSON(needs.should-skip.outputs.skip) && - fromJSON(needs.detect-changes.outputs.core) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks }} runs-on: ubuntu-latest needs: - should-skip @@ -542,7 +336,7 @@ jobs: shell: bash --noprofile --norc -euo pipefail {0} run: | git fetch --depth=1 --filter=blob:none origin \ - "${{ needs.detect-changes.outputs.pr_merge_base }}" + "${{ fromJSON(needs.detect-changes.outputs.workplan).merge_base }}" - name: Check cuda_core public API id: griffe @@ -550,7 +344,7 @@ jobs: with: package-name: cuda.core package-dir: cuda_core - merge-base: ${{ needs.detect-changes.outputs.pr_merge_base }} + merge-base: ${{ fromJSON(needs.detect-changes.outputs.workplan).merge_base }} # NOTE: Build jobs are intentionally split by platform rather than using a single # matrix. This lets each test job consume its platform-specific artifacts as @@ -580,14 +374,7 @@ jobs: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} - build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} - build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} - build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} - build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} - test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} - baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} - baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See build-linux-64 for why build jobs are split by platform. build-linux-aarch64: @@ -604,14 +391,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.build_pathfinder) || - fromJSON(needs.detect-changes.outputs.build_bindings) || - fromJSON(needs.detect-changes.outputs.build_core) || - fromJSON(needs.detect-changes.outputs.build_python) || - fromJSON(needs.detect-changes.outputs.test_pathfinder) || - fromJSON(needs.detect-changes.outputs.test_bindings) || - fromJSON(needs.detect-changes.outputs.test_core) || - fromJSON(needs.detect-changes.outputs.test_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platform_builds }} permissions: actions: read contents: read @@ -621,14 +401,7 @@ jobs: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} - build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} - build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} - build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} - build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} - test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} - baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} - baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See build-linux-64 for why build jobs are split by platform. build-windows: @@ -645,14 +418,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.build_pathfinder) || - fromJSON(needs.detect-changes.outputs.build_bindings) || - fromJSON(needs.detect-changes.outputs.build_core) || - fromJSON(needs.detect-changes.outputs.build_python) || - fromJSON(needs.detect-changes.outputs.test_pathfinder) || - fromJSON(needs.detect-changes.outputs.test_bindings) || - fromJSON(needs.detect-changes.outputs.test_core) || - fromJSON(needs.detect-changes.outputs.test_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platform_builds }} permissions: actions: read contents: read @@ -662,14 +428,7 @@ jobs: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} - build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} - build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} - build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} - build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} - test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} - baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} - baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # NOTE: test-sdist jobs are split by platform (mirroring build-* and test-wheel-*) # so platform-specific sources (e.g. cuda_bindings/*_windows.pyx selected by @@ -687,10 +446,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.build_pathfinder) || - fromJSON(needs.detect-changes.outputs.build_bindings) || - fromJSON(needs.detect-changes.outputs.build_core) || - fromJSON(needs.detect-changes.outputs.build_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests }} permissions: actions: read contents: read @@ -699,10 +455,7 @@ jobs: with: host-platform: linux-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} - build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} - build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} - build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See test-sdist-linux for why sdist test jobs are split by platform. test-sdist-windows: @@ -716,10 +469,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.build_pathfinder) || - fromJSON(needs.detect-changes.outputs.build_bindings) || - fromJSON(needs.detect-changes.outputs.build_core) || - fromJSON(needs.detect-changes.outputs.build_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests }} permissions: actions: read contents: read @@ -728,10 +478,7 @@ jobs: with: host-platform: win-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} - build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} - build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} - build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # NOTE: Test jobs are split by platform for the same reason as build jobs (see # build-linux-64). Keep these job definitions textually identical except for: @@ -747,10 +494,7 @@ jobs: name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.test_pathfinder) || - fromJSON(needs.detect-changes.outputs.test_bindings) || - fromJSON(needs.detect-changes.outputs.test_core) || - fromJSON(needs.detect-changes.outputs.test_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.wheel_tests }} permissions: actions: read contents: read # This is required for actions/checkout @@ -766,10 +510,7 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} - test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} - test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See test-linux-64 for why test jobs are split by platform. test-linux-aarch64: @@ -781,10 +522,7 @@ jobs: name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.test_pathfinder) || - fromJSON(needs.detect-changes.outputs.test_bindings) || - fromJSON(needs.detect-changes.outputs.test_core) || - fromJSON(needs.detect-changes.outputs.test_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.wheel_tests }} permissions: actions: read contents: read # This is required for actions/checkout @@ -801,10 +539,7 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} - test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} - test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See test-linux-64 for why test jobs are split by platform. test-windows: @@ -816,10 +551,7 @@ jobs: name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.test_pathfinder) || - fromJSON(needs.detect-changes.outputs.test_bindings) || - fromJSON(needs.detect-changes.outputs.test_core) || - fromJSON(needs.detect-changes.outputs.test_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.wheel_tests }} permissions: actions: read contents: read # This is required for actions/checkout @@ -836,10 +568,7 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} - test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} - test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} doc: name: Docs @@ -927,15 +656,10 @@ jobs: fi doc_only="${{ needs.should-skip.outputs.doc-only }}" - build_selected="${{ needs.detect-changes.outputs.build_pathfinder == 'true' || - needs.detect-changes.outputs.build_bindings == 'true' || - needs.detect-changes.outputs.build_core == 'true' || - needs.detect-changes.outputs.build_python == 'true' }}" - test_selected="${{ needs.detect-changes.outputs.test_pathfinder == 'true' || - needs.detect-changes.outputs.test_bindings == 'true' || - needs.detect-changes.outputs.test_core == 'true' || - needs.detect-changes.outputs.test_python == 'true' }}" - run_core_api_check="${{ needs.detect-changes.outputs.core == 'true' }}" + platform_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.platform_builds || false }}" + build_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests || false }}" + test_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.wheel_tests || false }}" + run_core_api_check="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks || false }}" is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { @@ -958,8 +682,7 @@ jobs: # Platform builds run whenever any package needs to be built or tested. expected="skipped" - if [[ "$doc_only" != "true" && - ( "$build_selected" == "true" || "$test_selected" == "true" ) ]]; then + if [[ "$doc_only" != "true" && "$platform_selected" == "true" ]]; then expected="success" fi check_result "build-linux-aarch64" "$expected" "${{ needs.build-linux-aarch64.result }}" diff --git a/.github/workflows/test-sdist-linux.yml b/.github/workflows/test-sdist-linux.yml index f0f64492f2d..ba7cdfc6ef1 100644 --- a/.github/workflows/test-sdist-linux.yml +++ b/.github/workflows/test-sdist-linux.yml @@ -11,22 +11,11 @@ on: cuda-version: required: true type: string - build-pathfinder: + workplan: + description: JSON workplan. An empty value builds everything. required: false - default: true - type: boolean - build-bindings: - required: false - default: true - type: boolean - build-core: - required: false - default: true - type: boolean - build-python: - required: false - default: true - type: boolean + default: "" + type: string defaults: run: @@ -39,7 +28,12 @@ permissions: jobs: test-sdist: name: Test sdist builds - if: ${{ inputs.build-pathfinder || inputs.build-bindings || inputs.build-core || inputs.build-python }} + if: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).jobs.sdist_tests }} + env: + BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} + BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} + BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} + BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} timeout-minutes: 60 runs-on: linux-amd64-cpu8 steps: @@ -61,26 +55,26 @@ jobs: # Pure Python packages -- no CTK needed. - name: Build cuda.pathfinder sdist and wheel-from-sdist - if: ${{ inputs.build-pathfinder }} + if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | python -m build --sdist cuda_pathfinder/ pip wheel --no-deps --wheel-dir cuda_pathfinder/dist cuda_pathfinder/dist/*.tar.gz - name: Build cuda-python sdist and wheel-from-sdist - if: ${{ inputs.build-python }} + if: ${{ env.BUILD_PYTHON == 'true' }} run: | python -m build --sdist cuda_python/ pip wheel --no-deps --wheel-dir cuda_python/dist cuda_python/dist/*.tar.gz - name: Download cuda.pathfinder wheel - if: ${{ !inputs.build-pathfinder && (inputs.build-bindings || inputs.build-core) }} + if: ${{ env.BUILD_PATHFINDER != 'true' && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel path: cuda_pathfinder/dist - name: Constrain builds to the local cuda.pathfinder wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 @@ -93,14 +87,14 @@ jobs: # The env vars ACTIONS_CACHE_SERVICE_V2, ACTIONS_RESULTS_URL, and ACTIONS_RUNTIME_TOKEN # are exposed by this action. - name: Enable sccache - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # 0.0.10 with: disable_annotations: 'true' # xref: https://github.com/orgs/community/discussions/42856#discussioncomment-7678867 - name: Adding additional GHA cache-related env vars - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: actions/github-script@v9 with: script: | @@ -108,14 +102,14 @@ jobs: core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL']) - name: Setup proxy cache - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: nv-gha-runners/setup-proxy-cache@main continue-on-error: true with: enable-apt: true - name: Set up mini CTK - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -125,7 +119,7 @@ jobs: # cuda_bindings/setup.py parses CUDA headers at import time, so CUDA_PATH # (set by fetch_ctk) must be available for both sdist and wheel builds. - name: Build cuda.bindings sdist and wheel-from-sdist - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export CC="sccache cc" @@ -136,14 +130,14 @@ jobs: pip wheel --no-deps --wheel-dir cuda_bindings/dist cuda_bindings/dist/*.tar.gz - name: Download cuda.bindings wheel - if: ${{ !inputs.build-bindings && inputs.build-core }} + if: ${{ env.BUILD_BINDINGS != 'true' && env.BUILD_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-bindings-python312-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ github.sha }} path: cuda_bindings/dist - name: Constrain cuda.core to the local cuda.bindings wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | CUDA_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) @@ -164,7 +158,7 @@ jobs: # wheel-from-sdist needs CTK and cuda-bindings (dynamic build dep via # get_requires_for_build_wheel in build_hooks.py). - name: Build cuda.core sdist and wheel-from-sdist - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" @@ -176,5 +170,5 @@ jobs: pip wheel --no-deps --wheel-dir cuda_core/dist cuda_core/dist/*.tar.gz - name: Show sccache stats - if: ${{ always() && (inputs.build-bindings || inputs.build-core) }} + if: ${{ always() && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} run: sccache --show-stats diff --git a/.github/workflows/test-sdist-windows.yml b/.github/workflows/test-sdist-windows.yml index 5451d20429e..a0594800eba 100644 --- a/.github/workflows/test-sdist-windows.yml +++ b/.github/workflows/test-sdist-windows.yml @@ -17,22 +17,11 @@ on: cuda-version: required: true type: string - build-pathfinder: + workplan: + description: JSON workplan. An empty value builds everything. required: false - default: true - type: boolean - build-bindings: - required: false - default: true - type: boolean - build-core: - required: false - default: true - type: boolean - build-python: - required: false - default: true - type: boolean + default: "" + type: string defaults: run: @@ -45,7 +34,12 @@ permissions: jobs: test-sdist: name: Test sdist builds - if: ${{ inputs.build-pathfinder || inputs.build-bindings || inputs.build-core || inputs.build-python }} + if: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).jobs.sdist_tests }} + env: + BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} + BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} + BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} + BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} timeout-minutes: 60 runs-on: windows-2022 steps: @@ -63,7 +57,7 @@ jobs: python-version: "3.12" - name: Set up MSVC - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 - name: Install build tools @@ -71,26 +65,26 @@ jobs: # Pure Python packages -- no CTK needed. - name: Build cuda.pathfinder sdist and wheel-from-sdist - if: ${{ inputs.build-pathfinder }} + if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | python -m build --sdist cuda_pathfinder/ pip wheel --no-deps --wheel-dir cuda_pathfinder/dist cuda_pathfinder/dist/*.tar.gz - name: Build cuda-python sdist and wheel-from-sdist - if: ${{ inputs.build-python }} + if: ${{ env.BUILD_PYTHON == 'true' }} run: | python -m build --sdist cuda_python/ pip wheel --no-deps --wheel-dir cuda_python/dist cuda_python/dist/*.tar.gz - name: Download cuda.pathfinder wheel - if: ${{ !inputs.build-pathfinder && (inputs.build-bindings || inputs.build-core) }} + if: ${{ env.BUILD_PATHFINDER != 'true' && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel path: cuda_pathfinder/dist - name: Constrain builds to the local cuda.pathfinder wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 @@ -103,7 +97,7 @@ jobs: # smoke test, not a production build; see build-wheel.yml which also # limits sccache to Linux). - name: Set up mini CTK - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -115,7 +109,7 @@ jobs: # Constraint paths are passed as native Windows paths because the pip # subprocesses run outside Git Bash. - name: Build cuda.bindings sdist and wheel-from-sdist - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-bindings.txt")" @@ -124,14 +118,14 @@ jobs: pip wheel --no-deps --wheel-dir cuda_bindings/dist cuda_bindings/dist/*.tar.gz - name: Download cuda.bindings wheel - if: ${{ !inputs.build-bindings && inputs.build-core }} + if: ${{ env.BUILD_BINDINGS != 'true' && env.BUILD_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-bindings-python312-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ github.sha }} path: cuda_bindings/dist - name: Constrain cuda.core to the local cuda.bindings wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | CUDA_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) @@ -152,7 +146,7 @@ jobs: # wheel-from-sdist needs CTK and cuda-bindings (dynamic build dep via # get_requires_for_build_wheel in build_hooks.py). - name: Build cuda.core sdist and wheel-from-sdist - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 8134a6844fd..4b7f2d64f8e 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -22,18 +22,10 @@ on: nruns: type: number default: 1 - test-pathfinder: - type: boolean - default: true - test-bindings: - type: boolean - default: true - test-core: - type: boolean - default: true - test-python: - type: boolean - default: true + workplan: + description: JSON workplan. An empty value tests everything. + type: string + default: "" run-id: description: > Workflow run ID to download artifacts from. @@ -106,6 +98,11 @@ jobs: echo "OLD_BRANCH=${OLD_BRANCH}" >> "$GITHUB_OUTPUT" test: + env: + TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }} + TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} + TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} + TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }} name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }}${{ matrix.FLAVOR && format(', {0}', matrix.FLAVOR) || '' }}${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }} timeout-minutes: 60 needs: compute-matrix @@ -164,7 +161,7 @@ jobs: LOCAL_CTK: ${{ matrix.LOCAL_CTK }} PY_VER: ${{ matrix.PY_VER }} SHA: ${{ inputs.sha || github.sha }} - SKIP_BINDINGS_TEST_OVERRIDE: ${{ !inputs.test-bindings && '1' || '0' }} + SKIP_BINDINGS_TEST_OVERRIDE: ${{ env.TEST_BINDINGS != 'true' && '1' || '0' }} run: ./ci/tools/env-vars test - name: Apply extra matrix environment variables @@ -174,7 +171,7 @@ jobs: run: echo "$MATRIX_ENV" | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> "$GITHUB_ENV" - name: Download cuda-pathfinder build artifacts - if: ${{ inputs.test-pathfinder || inputs.test-bindings || inputs.test-core || inputs.test-python }} + if: ${{ env.TEST_PATHFINDER == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel @@ -183,7 +180,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python build artifacts - if: ${{ inputs.test-python && env.BINDINGS_SOURCE == 'main' }} + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel @@ -192,7 +189,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda.bindings build artifacts - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'main' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -202,7 +199,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python & cuda.bindings build artifacts from the prior branch - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'backport' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -228,7 +225,7 @@ jobs: mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ rmdir $OLD_BASENAME - if ${{ inputs.test-python }}; then + if ${{ env.TEST_PYTHON == 'true' }}; then gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python ls -al cuda-python-wheel mv cuda-python-wheel/*.whl . @@ -236,20 +233,20 @@ jobs: fi - name: Display structure of downloaded cuda-python artifacts - if: ${{ inputs.test-python && env.BINDINGS_SOURCE != 'published' }} + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'published' }} run: | pwd ls -lah cuda_python*.whl cuda_pathfinder/ - name: Display structure of downloaded cuda.bindings artifacts - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE != 'published' }} run: | pwd ls -lahR $CUDA_BINDINGS_ARTIFACTS_DIR - name: Download cuda.bindings Cython tests - if: ${{ inputs.test-bindings && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests @@ -258,13 +255,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.bindings Cython tests - if: ${{ inputs.test-bindings && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} run: | pwd ls -lahR $CUDA_BINDINGS_CYTHON_TESTS_DIR - name: Download cuda.core build artifacts - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} @@ -273,13 +270,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core build artifacts - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | pwd ls -lahR $CUDA_CORE_ARTIFACTS_DIR - name: Download cuda.core Cython tests - if: ${{ inputs.test-core && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests @@ -288,13 +285,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core Cython tests - if: ${{ inputs.test-core && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} run: | pwd ls -lahR $CUDA_CORE_CYTHON_TESTS_DIR - name: Download cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries @@ -303,7 +300,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | pwd ls -lahR $CUDA_CORE_TEST_BINARIES_DIR @@ -319,7 +316,7 @@ jobs: AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache" - name: Enable Scientific Python Nightly Wheels for Python 3.15 - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && startsWith(matrix.PY_VER, '3.15') }} run: | echo "PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" >> "$GITHUB_ENV" @@ -334,7 +331,7 @@ jobs: cuda-version: ${{ matrix.CUDA_VER }} - name: Set up latest cuda_sanitizer_api - if: ${{ (inputs.test-bindings || inputs.test-core) && env.SETUP_SANITIZER == '1' }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && env.SETUP_SANITIZER == '1' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -343,7 +340,7 @@ jobs: cuda-components: "cuda_sanitizer_api" - name: Set up compute-sanitizer - if: ${{ inputs.test-bindings || inputs.test-core }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} run: setup-sanitizer - name: Set up test repetition on nightly runs @@ -351,7 +348,7 @@ jobs: # ── Standard test steps (skipped for nightly modes) ── - name: Run cuda.pathfinder tests with see_what_works - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works @@ -359,14 +356,14 @@ jobs: run: run-tests pathfinder - name: Run cuda.bindings tests - if: ${{ inputs.test-mode == 'standard' && inputs.test-bindings && env.SKIP_CUDA_BINDINGS_TEST == '0' }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} run: run-tests bindings - name: Run cuda.bindings benchmarks (smoke test) - if: ${{ inputs.test-mode == 'standard' && inputs.test-bindings && env.SKIP_CUDA_BINDINGS_TEST == '0' }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} run: | pip install pyperf pushd benchmarks/cuda_bindings @@ -374,20 +371,20 @@ jobs: popd - name: Run cuda.core tests - if: ${{ inputs.test-mode == 'standard' && inputs.test-core }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_CORE == 'true' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} run: run-tests core - name: Ensure cuda-python installable - if: ${{ inputs.test-mode == 'standard' && inputs.test-python && env.BINDINGS_SOURCE == 'main' }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} run: | # Package suites install their own dependencies. A metapackage-only # run has no preceding suite, so install the exact local internal # wheels in one transaction while resolving released dependencies # such as cuda-core from the package index. - if ${{ inputs.test-bindings || inputs.test-core }}; then + if ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }}; then dependency_args=(--no-deps) else dependency_args=( @@ -402,7 +399,7 @@ jobs: pip install --only-binary=:all: "${dependency_args[@]}" "${python_requirements[@]}" - name: Install cuda.pathfinder extra wheels for testing - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} run: | set -euo pipefail pushd cuda_pathfinder @@ -411,7 +408,7 @@ jobs: popd - name: Run cuda.pathfinder tests with all_must_work - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 04b290b1cd0..eb9d7ce9764 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -22,18 +22,10 @@ on: nruns: type: number default: 1 - test-pathfinder: - type: boolean - default: true - test-bindings: - type: boolean - default: true - test-core: - type: boolean - default: true - test-python: - type: boolean - default: true + workplan: + description: JSON workplan. An empty value tests everything. + type: string + default: "" run-id: description: > Workflow run ID to download artifacts from. @@ -96,6 +88,11 @@ jobs: echo "MATRIX=${MATRIX}" | tee --append "${GITHUB_OUTPUT}" test: + env: + TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }} + TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} + TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} + TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }} name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }} (${{ matrix.DRIVER_MODE }})${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }} timeout-minutes: 60 # The build stage could fail but we want the CI to keep moving. @@ -151,7 +148,7 @@ jobs: LOCAL_CTK: ${{ matrix.LOCAL_CTK }} PY_VER: ${{ matrix.PY_VER }} SHA: ${{ inputs.sha || github.sha }} - SKIP_BINDINGS_TEST_OVERRIDE: ${{ !inputs.test-bindings && '1' || '0' }} + SKIP_BINDINGS_TEST_OVERRIDE: ${{ env.TEST_BINDINGS != 'true' && '1' || '0' }} shell: bash --noprofile --norc -xeuo pipefail {0} run: ./ci/tools/env-vars test @@ -163,7 +160,7 @@ jobs: run: echo "$MATRIX_ENV" | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> "$GITHUB_ENV" - name: Download cuda-pathfinder build artifacts - if: ${{ inputs.test-pathfinder || inputs.test-bindings || inputs.test-core || inputs.test-python }} + if: ${{ env.TEST_PATHFINDER == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel @@ -172,7 +169,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python build artifacts - if: ${{ inputs.test-python && env.BINDINGS_SOURCE == 'main' }} + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel @@ -181,7 +178,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda.bindings build artifacts - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'main' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -191,7 +188,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python & cuda.bindings build artifacts from the prior branch - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'backport' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -208,7 +205,7 @@ jobs: mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ rmdir $OLD_BASENAME - if ${{ inputs.test-python }}; then + if ${{ env.TEST_PYTHON == 'true' }}; then gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python ls -al cuda-python-wheel mv cuda-python-wheel/*.whl . @@ -216,20 +213,20 @@ jobs: fi - name: Display structure of downloaded cuda-python artifacts - if: ${{ inputs.test-python && env.BINDINGS_SOURCE != 'published' }} + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'published' }} run: | Get-Location Get-ChildItem cuda_python*.whl | Select-Object Mode, LastWriteTime, Length, FullName - name: Display structure of downloaded cuda.bindings artifacts - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE != 'published' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_BINDINGS_ARTIFACTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.bindings Cython tests - if: ${{ inputs.test-bindings && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests @@ -238,13 +235,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.bindings Cython tests - if: ${{ inputs.test-bindings && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_BINDINGS_CYTHON_TESTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.core build artifacts - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} @@ -253,13 +250,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core build artifacts - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_CORE_ARTIFACTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.core Cython tests - if: ${{ inputs.test-core && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests @@ -268,13 +265,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core Cython tests - if: ${{ inputs.test-core && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_CORE_CYTHON_TESTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries @@ -283,7 +280,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_CORE_TEST_BINARIES_DIR | Select-Object Mode, LastWriteTime, Length, FullName @@ -296,7 +293,7 @@ jobs: allow-prereleases: ${{ startsWith(matrix.PY_VER, '3.15') }} - name: Enable Scientific Python Nightly Wheels for Python 3.15 - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && startsWith(matrix.PY_VER, '3.15') }} shell: bash --noprofile --norc -xeuo pipefail {0} run: | @@ -326,7 +323,7 @@ jobs: # ── Standard test steps (skipped for nightly modes) ── - name: Run cuda.pathfinder tests with see_what_works - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works @@ -335,7 +332,7 @@ jobs: run: run-tests pathfinder - name: Run cuda.bindings tests - if: ${{ inputs.test-mode == 'standard' && inputs.test-bindings && env.SKIP_CUDA_BINDINGS_TEST == '0' }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} @@ -343,7 +340,7 @@ jobs: run: run-tests bindings - name: Run cuda.core tests - if: ${{ inputs.test-mode == 'standard' && inputs.test-core }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_CORE == 'true' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} @@ -351,13 +348,13 @@ jobs: run: run-tests core - name: Ensure cuda-python installable - if: ${{ inputs.test-mode == 'standard' && inputs.test-python && env.BINDINGS_SOURCE == 'main' }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} run: | # Package suites install their own dependencies. A metapackage-only # run has no preceding suite, so install the exact local internal # wheels in one transaction while resolving released dependencies # such as cuda-core from the package index. - if ('${{ inputs.test-bindings || inputs.test-core }}' -eq 'true') { + if ('${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }}' -eq 'true') { $dependencyArgs = @('--no-deps') } else { $dependencyArgs = @( @@ -372,7 +369,7 @@ jobs: pip install --only-binary=:all: @dependencyArgs @pythonRequirements - name: Install cuda.pathfinder extra wheels for testing - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} shell: bash --noprofile --norc -xeuo pipefail {0} run: | pushd cuda_pathfinder @@ -381,7 +378,7 @@ jobs: popd - name: Run cuda.pathfinder tests with all_must_work - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py new file mode 100644 index 00000000000..07d57570cea --- /dev/null +++ b/ci/tools/compute_ci_plan.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Compute the CI build and test workplan for a pull request.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import PurePosixPath + +MODULES = ("pathfinder", "bindings", "core", "python") +PACKAGE_MODULES = { + "cuda_pathfinder": "pathfinder", + "cuda_bindings": "bindings", + "cuda_core": "core", + "cuda_python": "python", +} + +# Source changes have different build and test consumers. In particular, +# cuda-python source needs a same-version bindings wheel, while a core-only +# change can reuse the baseline cuda-python wheel. +SOURCE_IMPACT = { + "pathfinder": (set(MODULES), set(MODULES)), + "bindings": ({"bindings", "core", "python"}, {"bindings", "core", "python"}), + "core": ({"core"}, {"core", "python"}), + "python": ({"bindings", "python"}, {"python"}), +} + +IGNORED_BASENAMES = {"AGENTS.md", "CLAUDE.md"} +IGNORED_PATHS = { + ".coveragerc", + ".gitignore", + ".pre-commit-config.yaml", + ".spdx-ignore", + "CONTRIBUTING.md", + "LICENSE", + "SECURITY.md", + "context7.json", + "greptile.json", + "pixi.lock", + "pixi.toml", + "pytest.ini", + "ruff.toml", + "benchmarks/cuda_bindings/pixi.lock", + "benchmarks/cuda_bindings/pixi.toml", + "ci/.ci-pipeline-regen.md", + "ci/ci-pipeline.svg", + "ci/cleanup-pr-previews", + "ci/tools/check_mempool_hygiene.py", + "ci/tools/check_pixi_cuda_version.py", + "ci/tools/check_release_notes.py", + "ci/tools/download-wheels", + "ci/tools/run_pytest_with_stack.py", + "ci/tools/validate-release-wheels", + "cuda_bindings/pixi.lock", + "cuda_bindings/pixi.toml", + "cuda_core/pixi.lock", + "cuda_core/pixi.toml", + "cuda_pathfinder/pixi.lock", + "cuda_pathfinder/pixi.toml", +} +IGNORED_PREFIXES = ( + ".agents/", + "benchmarks/cuda_core/", + "ci/tools/tests/", + "cuda_python_test_helpers/", + "toolshed/", +) + +ALL_TEST_PATHS = { + ".github/workflows/test-wheel-linux.yml", + ".github/workflows/test-wheel-windows.yml", + "ci/test-matrix.yml", + "ci/tools/configure_driver_mode.ps1", + "ci/tools/guess_latest.sh", + "ci/tools/install_gpu_driver.ps1", + "ci/tools/install_gpu_driver.sh", + "ci/tools/run-tests", + "ci/tools/setup-sanitizer", +} + +INDEPENDENT_GITHUB_PATHS = { + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/RELEASE-core.md", + ".github/actionlint.yaml", + ".github/copy-pr-bot.yaml", + ".github/dependabot.yml", + ".github/labeler.yml", +} +INDEPENDENT_WORKFLOWS = { + "backport.yml", + "bandit.yml", + "build-docs.yml", + "ci-nightly.yml", + "ci-pixi-source-test.yml", + "cleanup-pr-previews.yml", + "coverage.yml", + "pr-auto-label.yml", + "pr-metadata-check.yml", + "release-cuda-pathfinder.yml", + "release-upload.yml", + "release.yml", + "security-suite.yml", + "triagelabel.yml", +} +INDEPENDENT_ACTIONS = {"doc_preview", "get_pr_number"} + + +def _is_independent(path: str) -> bool: + if path.startswith(IGNORED_PREFIXES): + return True + + if path in INDEPENDENT_GITHUB_PATHS or path.startswith(".github/ISSUE_TEMPLATE/"): + return True + + parts = PurePosixPath(path).parts + if len(parts) >= 3 and parts[:2] == (".github", "workflows"): + return parts[2] in INDEPENDENT_WORKFLOWS + if len(parts) >= 3 and parts[:2] == (".github", "actions"): + return parts[2] in INDEPENDENT_ACTIONS + return False + + +def compute_workplan( + paths: list[str], + *, + merge_base: str, + baseline_run_id: str, + baseline_sha: str, +) -> dict[str, object]: + """Return the final CI decisions for the supplied changed paths.""" + source_changes: set[str] = set() + test_changes: set[str] = set() + all_tests = False + force_all = not merge_base or not baseline_run_id or not baseline_sha + + if not force_all: + for path in paths: + path_parts = PurePosixPath(path).parts + if not path_parts or path in IGNORED_PATHS or path_parts[-1] in IGNORED_BASENAMES: + continue + + if path == "README.md": + # cuda_python/README.md is a tracked symlink to this sdist input. + source_changes.add("python") + continue + + module = PACKAGE_MODULES.get(path_parts[0]) + if module is not None and len(path_parts) > 1: + relative = path_parts[1:] + if relative[0] == "docs": + continue + if relative[0] in {"tests", "examples"} or (module == "core" and relative == ("pytest.ini",)): + test_changes.add(module) + else: + source_changes.add(module) + continue + + if path.startswith("cuda_python_test_helpers/cuda_python_test_helpers/"): + test_changes.update(("bindings", "core")) + elif path.startswith("benchmarks/cuda_bindings/"): + test_changes.add("bindings") + elif path in ALL_TEST_PATHS: + all_tests = True + elif not _is_independent(path): + force_all = True + + if force_all: + builds = set(MODULES) + tests = set(MODULES) + else: + builds: set[str] = set() + tests = set(MODULES) if all_tests else set(test_changes) + for module in source_changes: + build_impact, test_impact = SOURCE_IMPACT[module] + builds.update(build_impact) + tests.update(test_impact) + + modules = { + module: { + "needs_build": module in builds, + "needs_test": module in tests, + } + for module in MODULES + } + return { + "modules": modules, + "jobs": { + "platform_builds": bool(builds or tests), + "sdist_tests": bool(builds), + "wheel_tests": bool(tests), + "core_api_checks": force_all or "core" in source_changes, + }, + "merge_base": merge_base, + "baseline": { + "run_id": baseline_run_id if not force_all else "", + "sha": baseline_sha if not force_all else "", + }, + } + + +def _changed_paths(merge_base: str, head: str) -> list[str]: + result = subprocess.run( # noqa: S603 - argv is passed directly to git without a shell. + ["git", "diff", "--no-renames", "--name-only", "-z", f"{merge_base}...{head}"], # noqa: S607 + check=True, + stdout=subprocess.PIPE, + ) + return [path.decode("utf-8", errors="surrogateescape") for path in result.stdout.split(b"\0") if path] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--merge-base", default="") + parser.add_argument("--head", default="HEAD") + parser.add_argument("--baseline-run-id", default="") + parser.add_argument("--baseline-sha", default="") + args = parser.parse_args() + + if bool(args.baseline_run_id) != bool(args.baseline_sha): + parser.error("baseline run ID and SHA must be supplied together") + if args.baseline_sha and args.baseline_sha != args.merge_base: + parser.error("baseline SHA must match the merge base") + + reusable_baseline = bool(args.merge_base and args.baseline_run_id) + paths = _changed_paths(args.merge_base, args.head) if reusable_baseline else [] + plan = compute_workplan( + paths, + merge_base=args.merge_base, + baseline_run_id=args.baseline_run_id, + baseline_sha=args.baseline_sha, + ) + print(json.dumps(plan, separators=(",", ":"), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/ci/tools/tests/test_compute_ci_plan.py b/ci/tools/tests/test_compute_ci_plan.py new file mode 100644 index 00000000000..9be450d98e6 --- /dev/null +++ b/ci/tools/tests/test_compute_ci_plan.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import unittest + +from ci.tools.compute_ci_plan import compute_workplan + +ALL_MODULES = {"pathfinder", "bindings", "core", "python"} + + +def plan_for(*paths: str, baseline: bool = True) -> dict[str, object]: + return compute_workplan( + list(paths), + merge_base="base", + baseline_run_id="123" if baseline else "", + baseline_sha="base" if baseline else "", + ) + + +def selected(plan: dict[str, object], key: str) -> set[str]: + modules = plan["modules"] + assert isinstance(modules, dict) + return {name for name, decision in modules.items() if decision[key]} + + +class ComputeWorkplanTest(unittest.TestCase): + def test_path_impacts(self) -> None: + cases = { + "cuda_pathfinder/cuda/pathfinder/_loader.py": (ALL_MODULES, ALL_MODULES, False), + "cuda_bindings/cuda/bindings/driver.pyx": ( + {"bindings", "core", "python"}, + {"bindings", "core", "python"}, + False, + ), + "cuda_core/cuda/core/_device.py": ({"core"}, {"core", "python"}, True), + "cuda_python/pyproject.toml": ({"bindings", "python"}, {"python"}, False), + "README.md": ({"bindings", "python"}, {"python"}, False), + "cuda_pathfinder/tests/test_loader.py": (set(), {"pathfinder"}, False), + "cuda_bindings/examples/0_Introduction/vectorAddDrv.py": (set(), {"bindings"}, False), + "cuda_core/pytest.ini": (set(), {"core"}, False), + "cuda_core/tests/fixtures/pixi.toml": (set(), {"core"}, False), + "cuda_python/tests/test_import.py": (set(), {"python"}, False), + "cuda_python/pixi.toml": ({"bindings", "python"}, {"python"}, False), + "cuda_python_test_helpers/cuda_python_test_helpers/cuda_utils.py": ( + set(), + {"bindings", "core"}, + False, + ), + "ci/tools/run-tests": (set(), ALL_MODULES, False), + "ci/versions.yml": (ALL_MODULES, ALL_MODULES, True), + } + + for path, (builds, tests, core_api) in cases.items(): + with self.subTest(path=path): + plan = plan_for(path) + assert selected(plan, "needs_build") == builds + assert selected(plan, "needs_test") == tests + assert plan["jobs"]["core_api_checks"] == core_api + + def test_ignored_paths_select_no_work(self) -> None: + for path in ( + "cuda_core/docs/index.rst", + "cuda_core/pixi.toml", + "benchmarks/cuda_bindings/pixi.toml", + "benchmarks/cuda_bindings/AGENTS.md", + ".github/workflows/ci-pixi-source-test.yml", + "benchmarks/cuda_core/benchmark.py", + ): + with self.subTest(path=path): + plan = plan_for(path) + assert not selected(plan, "needs_build") + assert not selected(plan, "needs_test") + + def test_unknown_path_and_missing_baseline_force_all(self) -> None: + for plan in ( + plan_for("new-top-level-file"), + plan_for("new-area/pixi.toml"), + plan_for(".github/workflows/new-main-ci-workflow.yml"), + plan_for("cuda_core/docs/index.rst", baseline=False), + compute_workplan([], merge_base="base", baseline_run_id="123", baseline_sha=""), + ): + assert selected(plan, "needs_build") == ALL_MODULES + assert selected(plan, "needs_test") == ALL_MODULES + assert plan["jobs"]["core_api_checks"] + assert plan["baseline"] == {"run_id": "", "sha": ""} + + def test_mixed_changes_are_combined(self) -> None: + plan = plan_for("cuda_core/tests/test_device.py", "cuda_python/pyproject.toml") + assert selected(plan, "needs_build") == {"bindings", "python"} + assert selected(plan, "needs_test") == {"core", "python"} + assert plan["jobs"]["platform_builds"] + assert plan["jobs"]["sdist_tests"] + assert plan["jobs"]["wheel_tests"] + + +if __name__ == "__main__": + unittest.main() From 5c798a7c83c6b8f1e7ad1bd47840003750a20837 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Mon, 17 Aug 2026 11:06:21 -0400 Subject: [PATCH 05/13] ci: split selective tests by platform --- .github/workflows/ci.yml | 42 ++++++++++++------------ ci/tools/compute_ci_plan.py | 36 ++++++++++++--------- ci/tools/tests/test_compute_ci_plan.py | 45 ++++++++++++++++++++++++-- 3 files changed, 84 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99a8904a1b2..3de756b1ce9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -391,7 +391,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.platform_builds }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }} permissions: actions: read contents: read @@ -418,7 +418,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.platform_builds }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows }} permissions: actions: read contents: read @@ -494,7 +494,7 @@ jobs: name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.wheel_tests }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }} permissions: actions: read contents: read # This is required for actions/checkout @@ -522,7 +522,7 @@ jobs: name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.wheel_tests }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }} permissions: actions: read contents: read # This is required for actions/checkout @@ -551,7 +551,7 @@ jobs: name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.wheel_tests }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows }} permissions: actions: read contents: read # This is required for actions/checkout @@ -656,9 +656,9 @@ jobs: fi doc_only="${{ needs.should-skip.outputs.doc-only }}" - platform_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.platform_builds || false }}" + linux_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux || false }}" + windows_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows || false }}" build_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests || false }}" - test_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.wheel_tests || false }}" run_core_api_check="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks || false }}" is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" @@ -680,15 +680,19 @@ jobs: check_result "doc" "success" "${{ needs.doc.result }}" check_result "precommit-windows" "success" "${{ needs.precommit-windows.result }}" - # Platform builds run whenever any package needs to be built or tested. - expected="skipped" - if [[ "$doc_only" != "true" && "$platform_selected" == "true" ]]; then - expected="success" + # Optional platform builds and wheel tests share the platform plan. + linux_expected="skipped" + if [[ "$doc_only" != "true" && "$linux_selected" == "true" ]]; then + linux_expected="success" + fi + windows_expected="skipped" + if [[ "$doc_only" != "true" && "$windows_selected" == "true" ]]; then + windows_expected="success" fi - check_result "build-linux-aarch64" "$expected" "${{ needs.build-linux-aarch64.result }}" - check_result "build-windows" "$expected" "${{ needs.build-windows.result }}" + check_result "build-linux-aarch64" "$linux_expected" "${{ needs.build-linux-aarch64.result }}" + check_result "build-windows" "$windows_expected" "${{ needs.build-windows.result }}" - # Sdist and wheel tests are independently gated by the effective plan. + # Sdist tests follow build selection; wheel tests follow the platform plan. expected="skipped" if [[ "$doc_only" != "true" && "$build_selected" == "true" ]]; then expected="success" @@ -696,13 +700,9 @@ jobs: check_result "test-sdist-linux" "$expected" "${{ needs.test-sdist-linux.result }}" check_result "test-sdist-windows" "$expected" "${{ needs.test-sdist-windows.result }}" - expected="skipped" - if [[ "$doc_only" != "true" && "$test_selected" == "true" ]]; then - expected="success" - fi - check_result "test-linux-64" "$expected" "${{ needs.test-linux-64.result }}" - check_result "test-linux-aarch64" "$expected" "${{ needs.test-linux-aarch64.result }}" - check_result "test-windows" "$expected" "${{ needs.test-windows.result }}" + check_result "test-linux-64" "$linux_expected" "${{ needs.test-linux-64.result }}" + check_result "test-linux-aarch64" "$linux_expected" "${{ needs.test-linux-aarch64.result }}" + check_result "test-windows" "$windows_expected" "${{ needs.test-windows.result }}" # API compatibility checks run for cuda_core source changes and for # conservative full runs when reusable base artifacts are unavailable. diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py index 07d57570cea..d459a227071 100644 --- a/ci/tools/compute_ci_plan.py +++ b/ci/tools/compute_ci_plan.py @@ -14,6 +14,7 @@ from pathlib import PurePosixPath MODULES = ("pathfinder", "bindings", "core", "python") +PLATFORMS = ("linux", "windows") PACKAGE_MODULES = { "cuda_pathfinder": "pathfinder", "cuda_bindings": "bindings", @@ -72,16 +73,16 @@ "toolshed/", ) -ALL_TEST_PATHS = { - ".github/workflows/test-wheel-linux.yml", - ".github/workflows/test-wheel-windows.yml", - "ci/test-matrix.yml", - "ci/tools/configure_driver_mode.ps1", - "ci/tools/guess_latest.sh", - "ci/tools/install_gpu_driver.ps1", - "ci/tools/install_gpu_driver.sh", - "ci/tools/run-tests", - "ci/tools/setup-sanitizer", +TEST_INFRA_PLATFORMS = { + ".github/workflows/test-wheel-linux.yml": {"linux"}, + ".github/workflows/test-wheel-windows.yml": {"windows"}, + "ci/test-matrix.yml": set(PLATFORMS), + "ci/tools/configure_driver_mode.ps1": {"windows"}, + "ci/tools/guess_latest.sh": {"linux"}, + "ci/tools/install_gpu_driver.ps1": {"windows"}, + "ci/tools/install_gpu_driver.sh": {"linux"}, + "ci/tools/run-tests": set(PLATFORMS), + "ci/tools/setup-sanitizer": {"linux"}, } INDEPENDENT_GITHUB_PATHS = { @@ -136,7 +137,7 @@ def compute_workplan( """Return the final CI decisions for the supplied changed paths.""" source_changes: set[str] = set() test_changes: set[str] = set() - all_tests = False + test_platforms: set[str] = set() force_all = not merge_base or not baseline_run_id or not baseline_sha if not force_all: @@ -165,21 +166,24 @@ def compute_workplan( test_changes.update(("bindings", "core")) elif path.startswith("benchmarks/cuda_bindings/"): test_changes.add("bindings") - elif path in ALL_TEST_PATHS: - all_tests = True + elif platforms := TEST_INFRA_PLATFORMS.get(path): + test_platforms.update(platforms) elif not _is_independent(path): force_all = True if force_all: builds = set(MODULES) tests = set(MODULES) + test_platforms = set(PLATFORMS) else: builds: set[str] = set() - tests = set(MODULES) if all_tests else set(test_changes) + tests = set(MODULES) if test_platforms else set(test_changes) for module in source_changes: build_impact, test_impact = SOURCE_IMPACT[module] builds.update(build_impact) tests.update(test_impact) + if source_changes or test_changes: + test_platforms.update(PLATFORMS) modules = { module: { @@ -191,9 +195,9 @@ def compute_workplan( return { "modules": modules, "jobs": { - "platform_builds": bool(builds or tests), + # These gates cover both optional artifact builds and wheel tests. + "platforms": {platform: platform in test_platforms for platform in PLATFORMS}, "sdist_tests": bool(builds), - "wheel_tests": bool(tests), "core_api_checks": force_all or "core" in source_changes, }, "merge_base": merge_base, diff --git a/ci/tools/tests/test_compute_ci_plan.py b/ci/tools/tests/test_compute_ci_plan.py index 9be450d98e6..303b4c8ac89 100644 --- a/ci/tools/tests/test_compute_ci_plan.py +++ b/ci/tools/tests/test_compute_ci_plan.py @@ -9,6 +9,7 @@ from ci.tools.compute_ci_plan import compute_workplan ALL_MODULES = {"pathfinder", "bindings", "core", "python"} +ALL_PLATFORMS = {"linux", "windows"} def plan_for(*paths: str, baseline: bool = True) -> dict[str, object]: @@ -26,6 +27,15 @@ def selected(plan: dict[str, object], key: str) -> set[str]: return {name for name, decision in modules.items() if decision[key]} +def selected_platforms(plan: dict[str, object]) -> set[str]: + jobs = plan["jobs"] + assert isinstance(jobs, dict) + platforms = jobs["platforms"] + assert isinstance(platforms, dict) + assert set(platforms) == ALL_PLATFORMS + return {name for name, enabled in platforms.items() if enabled} + + class ComputeWorkplanTest(unittest.TestCase): def test_path_impacts(self) -> None: cases = { @@ -58,8 +68,38 @@ def test_path_impacts(self) -> None: plan = plan_for(path) assert selected(plan, "needs_build") == builds assert selected(plan, "needs_test") == tests + assert selected_platforms(plan) == ALL_PLATFORMS + assert plan["jobs"]["sdist_tests"] == bool(builds) assert plan["jobs"]["core_api_checks"] == core_api + def test_test_infrastructure_platforms(self) -> None: + cases = { + ".github/workflows/test-wheel-linux.yml": {"linux"}, + ".github/workflows/test-wheel-windows.yml": {"windows"}, + "ci/test-matrix.yml": ALL_PLATFORMS, + "ci/tools/configure_driver_mode.ps1": {"windows"}, + "ci/tools/guess_latest.sh": {"linux"}, + "ci/tools/install_gpu_driver.ps1": {"windows"}, + "ci/tools/install_gpu_driver.sh": {"linux"}, + "ci/tools/run-tests": ALL_PLATFORMS, + "ci/tools/setup-sanitizer": {"linux"}, + } + + for path, platforms in cases.items(): + with self.subTest(path=path): + plan = plan_for(path) + assert not selected(plan, "needs_build") + assert selected(plan, "needs_test") == ALL_MODULES + assert selected_platforms(plan) == platforms + assert not plan["jobs"]["sdist_tests"] + assert not plan["jobs"]["core_api_checks"] + + mixed_plan = plan_for("ci/tools/install_gpu_driver.sh", "ci/tools/install_gpu_driver.ps1") + assert selected_platforms(mixed_plan) == ALL_PLATFORMS + + source_plan = plan_for("ci/tools/install_gpu_driver.sh", "cuda_python/pyproject.toml") + assert selected_platforms(source_plan) == ALL_PLATFORMS + def test_ignored_paths_select_no_work(self) -> None: for path in ( "cuda_core/docs/index.rst", @@ -73,6 +113,7 @@ def test_ignored_paths_select_no_work(self) -> None: plan = plan_for(path) assert not selected(plan, "needs_build") assert not selected(plan, "needs_test") + assert not selected_platforms(plan) def test_unknown_path_and_missing_baseline_force_all(self) -> None: for plan in ( @@ -84,6 +125,7 @@ def test_unknown_path_and_missing_baseline_force_all(self) -> None: ): assert selected(plan, "needs_build") == ALL_MODULES assert selected(plan, "needs_test") == ALL_MODULES + assert selected_platforms(plan) == ALL_PLATFORMS assert plan["jobs"]["core_api_checks"] assert plan["baseline"] == {"run_id": "", "sha": ""} @@ -91,9 +133,8 @@ def test_mixed_changes_are_combined(self) -> None: plan = plan_for("cuda_core/tests/test_device.py", "cuda_python/pyproject.toml") assert selected(plan, "needs_build") == {"bindings", "python"} assert selected(plan, "needs_test") == {"core", "python"} - assert plan["jobs"]["platform_builds"] + assert selected_platforms(plan) == ALL_PLATFORMS assert plan["jobs"]["sdist_tests"] - assert plan["jobs"]["wheel_tests"] if __name__ == "__main__": From e5c3610ab0025e0a4e4ec1cd927e8fe6dc98fe35 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Mon, 17 Aug 2026 14:44:43 -0400 Subject: [PATCH 06/13] ci: harden prior artifact downloads --- .github/workflows/build-wheel.yml | 16 +++++++++------ .github/workflows/test-wheel-linux.yml | 21 +++++++++++++------- .github/workflows/test-wheel-windows.yml | 21 +++++++++++++------- ci/tools/retry-gh-run-download | 25 ++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 20 deletions(-) create mode 100755 ci/tools/retry-gh-run-download diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 15e84819f1d..a71f71c3e5f 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -567,16 +567,20 @@ jobs: fi OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml) - OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" + OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]" LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI") PREV_BINDINGS_DIR="cuda_bindings/dist-prev" - gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python - rm -rf ${OLD_BASENAME}-tests # exclude cython test artifacts - ls -al $OLD_BASENAME + ./ci/tools/retry-gh-run-download \ + "${LATEST_PRIOR_RUN_ID}" \ + -p "${OLD_ARTIFACT_PATTERN}" \ + -R NVIDIA/cuda-python + OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}") + test -d "${OLD_ARTIFACT_DIR}" + ls -al "${OLD_ARTIFACT_DIR}" mkdir -p "${PREV_BINDINGS_DIR}" - mv $OLD_BASENAME/*.whl "${PREV_BINDINGS_DIR}" - rmdir $OLD_BASENAME + mv "${OLD_ARTIFACT_DIR}"/*.whl "${PREV_BINDINGS_DIR}" + rmdir "${OLD_ARTIFACT_DIR}" - name: Constrain previous cuda.core to the downloaded cuda.bindings wheel if: ${{ env.BUILD_CORE == 'true' }} diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 4b7f2d64f8e..aced9e93ea5 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -215,18 +215,25 @@ jobs: && apt install gh -y OLD_BRANCH=${{ needs.compute-matrix.outputs.OLD_BRANCH }} - OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" + OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]" LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI") - gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python - rm -rf ${OLD_BASENAME}-tests # exclude cython test artifacts - ls -al $OLD_BASENAME + ./ci/tools/retry-gh-run-download \ + "${LATEST_PRIOR_RUN_ID}" \ + -p "${OLD_ARTIFACT_PATTERN}" \ + -R NVIDIA/cuda-python + OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}") + test -d "${OLD_ARTIFACT_DIR}" + ls -al "${OLD_ARTIFACT_DIR}" mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}" - mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ - rmdir $OLD_BASENAME + mv "${OLD_ARTIFACT_DIR}"/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ + rmdir "${OLD_ARTIFACT_DIR}" if ${{ env.TEST_PYTHON == 'true' }}; then - gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python + ./ci/tools/retry-gh-run-download \ + "${LATEST_PRIOR_RUN_ID}" \ + -p cuda-python-wheel \ + -R NVIDIA/cuda-python ls -al cuda-python-wheel mv cuda-python-wheel/*.whl . rmdir cuda-python-wheel diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index eb9d7ce9764..b6e163117db 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -195,18 +195,25 @@ jobs: shell: bash --noprofile --norc -xeuo pipefail {0} run: | OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml) - OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" + OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]" LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI") - gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python - rm -rf ${OLD_BASENAME}-tests # exclude cython test artifacts - ls -al $OLD_BASENAME + ./ci/tools/retry-gh-run-download \ + "${LATEST_PRIOR_RUN_ID}" \ + -p "${OLD_ARTIFACT_PATTERN}" \ + -R NVIDIA/cuda-python + OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}") + test -d "${OLD_ARTIFACT_DIR}" + ls -al "${OLD_ARTIFACT_DIR}" mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}" - mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ - rmdir $OLD_BASENAME + mv "${OLD_ARTIFACT_DIR}"/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ + rmdir "${OLD_ARTIFACT_DIR}" if ${{ env.TEST_PYTHON == 'true' }}; then - gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python + ./ci/tools/retry-gh-run-download \ + "${LATEST_PRIOR_RUN_ID}" \ + -p cuda-python-wheel \ + -R NVIDIA/cuda-python ls -al cuda-python-wheel mv cuda-python-wheel/*.whl . rmdir cuda-python-wheel diff --git a/ci/tools/retry-gh-run-download b/ci/tools/retry-gh-run-download new file mode 100755 index 00000000000..04883896c46 --- /dev/null +++ b/ci/tools/retry-gh-run-download @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +readonly MAX_ATTEMPTS=5 +delay=10 + +for ((attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)); do + if gh run download "$@"; then + exit 0 + fi + + if (( attempt == MAX_ATTEMPTS )); then + echo "Artifact download failed after ${MAX_ATTEMPTS} attempts." >&2 + exit 1 + fi + + echo "Artifact download failed (attempt ${attempt}/${MAX_ATTEMPTS}); retrying in ${delay}s." >&2 + sleep "${delay}" + delay=$((delay * 2)) +done From 1ccaaef7e94507f3e635ed5e12c54c9cb763d6d3 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Mon, 17 Aug 2026 15:43:19 -0400 Subject: [PATCH 07/13] ci: defer artifact download retries --- .github/workflows/build-wheel.yml | 2 +- .github/workflows/test-wheel-linux.yml | 4 ++-- .github/workflows/test-wheel-windows.yml | 4 ++-- ci/tools/retry-gh-run-download | 25 ------------------------ 4 files changed, 5 insertions(+), 30 deletions(-) delete mode 100755 ci/tools/retry-gh-run-download diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index a71f71c3e5f..72954deb2af 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -571,7 +571,7 @@ jobs: LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI") PREV_BINDINGS_DIR="cuda_bindings/dist-prev" - ./ci/tools/retry-gh-run-download \ + gh run download \ "${LATEST_PRIOR_RUN_ID}" \ -p "${OLD_ARTIFACT_PATTERN}" \ -R NVIDIA/cuda-python diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index aced9e93ea5..5e7dc32beef 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -218,7 +218,7 @@ jobs: OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]" LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI") - ./ci/tools/retry-gh-run-download \ + gh run download \ "${LATEST_PRIOR_RUN_ID}" \ -p "${OLD_ARTIFACT_PATTERN}" \ -R NVIDIA/cuda-python @@ -230,7 +230,7 @@ jobs: rmdir "${OLD_ARTIFACT_DIR}" if ${{ env.TEST_PYTHON == 'true' }}; then - ./ci/tools/retry-gh-run-download \ + gh run download \ "${LATEST_PRIOR_RUN_ID}" \ -p cuda-python-wheel \ -R NVIDIA/cuda-python diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index b6e163117db..6a58e4983b3 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -198,7 +198,7 @@ jobs: OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]" LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI") - ./ci/tools/retry-gh-run-download \ + gh run download \ "${LATEST_PRIOR_RUN_ID}" \ -p "${OLD_ARTIFACT_PATTERN}" \ -R NVIDIA/cuda-python @@ -210,7 +210,7 @@ jobs: rmdir "${OLD_ARTIFACT_DIR}" if ${{ env.TEST_PYTHON == 'true' }}; then - ./ci/tools/retry-gh-run-download \ + gh run download \ "${LATEST_PRIOR_RUN_ID}" \ -p cuda-python-wheel \ -R NVIDIA/cuda-python diff --git a/ci/tools/retry-gh-run-download b/ci/tools/retry-gh-run-download deleted file mode 100755 index 04883896c46..00000000000 --- a/ci/tools/retry-gh-run-download +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -readonly MAX_ATTEMPTS=5 -delay=10 - -for ((attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)); do - if gh run download "$@"; then - exit 0 - fi - - if (( attempt == MAX_ATTEMPTS )); then - echo "Artifact download failed after ${MAX_ATTEMPTS} attempts." >&2 - exit 1 - fi - - echo "Artifact download failed (attempt ${attempt}/${MAX_ATTEMPTS}); retrying in ${delay}s." >&2 - sleep "${delay}" - delay=$((delay * 2)) -done From ca29abd4d3f7b7de797e97af97127a38029e1d05 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Mon, 17 Aug 2026 16:18:16 -0400 Subject: [PATCH 08/13] ci: simplify workplan path classification --- ci/tools/compute_ci_plan.py | 164 +++++++++++-------------- ci/tools/tests/test_compute_ci_plan.py | 64 ++++++++-- 2 files changed, 127 insertions(+), 101 deletions(-) diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py index d459a227071..0bf4328ffca 100644 --- a/ci/tools/compute_ci_plan.py +++ b/ci/tools/compute_ci_plan.py @@ -11,8 +11,9 @@ import argparse import json import subprocess -from pathlib import PurePosixPath +from pathlib import Path, PurePosixPath +REPO_ROOT = Path(__file__).resolve().parents[2] MODULES = ("pathfinder", "bindings", "core", "python") PLATFORMS = ("linux", "windows") PACKAGE_MODULES = { @@ -32,100 +33,31 @@ "python": ({"bindings", "python"}, {"python"}), } -IGNORED_BASENAMES = {"AGENTS.md", "CLAUDE.md"} +IGNORED_BASENAMES = {"AGENTS.md", "CLAUDE.md", "pixi.lock", "pixi.toml"} +IGNORED_SUFFIXES = {".md", ".svg"} IGNORED_PATHS = { ".coveragerc", ".gitignore", ".pre-commit-config.yaml", ".spdx-ignore", - "CONTRIBUTING.md", "LICENSE", - "SECURITY.md", "context7.json", "greptile.json", - "pixi.lock", - "pixi.toml", - "pytest.ini", "ruff.toml", - "benchmarks/cuda_bindings/pixi.lock", - "benchmarks/cuda_bindings/pixi.toml", - "ci/.ci-pipeline-regen.md", - "ci/ci-pipeline.svg", - "ci/cleanup-pr-previews", - "ci/tools/check_mempool_hygiene.py", - "ci/tools/check_pixi_cuda_version.py", - "ci/tools/check_release_notes.py", - "ci/tools/download-wheels", - "ci/tools/run_pytest_with_stack.py", - "ci/tools/validate-release-wheels", - "cuda_bindings/pixi.lock", - "cuda_bindings/pixi.toml", - "cuda_core/pixi.lock", - "cuda_core/pixi.toml", - "cuda_pathfinder/pixi.lock", - "cuda_pathfinder/pixi.toml", } -IGNORED_PREFIXES = ( - ".agents/", - "benchmarks/cuda_core/", - "ci/tools/tests/", - "cuda_python_test_helpers/", - "toolshed/", -) +IGNORED_PREFIXES = (".agents/", "toolshed/") +# Only infrastructure exclusive to one OS belongs here; other CI paths force a full run. TEST_INFRA_PLATFORMS = { ".github/workflows/test-wheel-linux.yml": {"linux"}, ".github/workflows/test-wheel-windows.yml": {"windows"}, - "ci/test-matrix.yml": set(PLATFORMS), "ci/tools/configure_driver_mode.ps1": {"windows"}, "ci/tools/guess_latest.sh": {"linux"}, "ci/tools/install_gpu_driver.ps1": {"windows"}, "ci/tools/install_gpu_driver.sh": {"linux"}, - "ci/tools/run-tests": set(PLATFORMS), "ci/tools/setup-sanitizer": {"linux"}, } -INDEPENDENT_GITHUB_PATHS = { - ".github/PULL_REQUEST_TEMPLATE.md", - ".github/RELEASE-core.md", - ".github/actionlint.yaml", - ".github/copy-pr-bot.yaml", - ".github/dependabot.yml", - ".github/labeler.yml", -} -INDEPENDENT_WORKFLOWS = { - "backport.yml", - "bandit.yml", - "build-docs.yml", - "ci-nightly.yml", - "ci-pixi-source-test.yml", - "cleanup-pr-previews.yml", - "coverage.yml", - "pr-auto-label.yml", - "pr-metadata-check.yml", - "release-cuda-pathfinder.yml", - "release-upload.yml", - "release.yml", - "security-suite.yml", - "triagelabel.yml", -} -INDEPENDENT_ACTIONS = {"doc_preview", "get_pr_number"} - - -def _is_independent(path: str) -> bool: - if path.startswith(IGNORED_PREFIXES): - return True - - if path in INDEPENDENT_GITHUB_PATHS or path.startswith(".github/ISSUE_TEMPLATE/"): - return True - - parts = PurePosixPath(path).parts - if len(parts) >= 3 and parts[:2] == (".github", "workflows"): - return parts[2] in INDEPENDENT_WORKFLOWS - if len(parts) >= 3 and parts[:2] == (".github", "actions"): - return parts[2] in INDEPENDENT_ACTIONS - return False - def compute_workplan( paths: list[str], @@ -133,8 +65,10 @@ def compute_workplan( merge_base: str, baseline_run_id: str, baseline_sha: str, + linked_paths: set[str] | None = None, ) -> dict[str, object]: """Return the final CI decisions for the supplied changed paths.""" + linked_paths = linked_paths or set() source_changes: set[str] = set() test_changes: set[str] = set() test_platforms: set[str] = set() @@ -143,12 +77,20 @@ def compute_workplan( if not force_all: for path in paths: path_parts = PurePosixPath(path).parts - if not path_parts or path in IGNORED_PATHS or path_parts[-1] in IGNORED_BASENAMES: + if not path_parts: continue - if path == "README.md": - # cuda_python/README.md is a tracked symlink to this sdist input. - source_changes.add("python") + if platforms := TEST_INFRA_PLATFORMS.get(path): + test_platforms.update(platforms) + continue + + if path_parts[0] == "ci" or ( + len(path_parts) >= 2 and path_parts[:2] in {(".github", "actions"), (".github", "workflows")} + ): + force_all = True + break + + if path_parts[0] == ".github" or path_parts[-1] in IGNORED_BASENAMES: continue module = PACKAGE_MODULES.get(path_parts[0]) @@ -156,20 +98,33 @@ def compute_workplan( relative = path_parts[1:] if relative[0] == "docs": continue - if relative[0] in {"tests", "examples"} or (module == "core" and relative == ("pytest.ini",)): + if ( + any(part in {"test", "tests"} for part in relative[:-1]) + or relative[0] == "examples" + or (module == "core" and relative == ("pytest.ini",)) + ): test_changes.add(module) + elif PurePosixPath(path).suffix in IGNORED_SUFFIXES and path not in linked_paths: + continue else: source_changes.add(module) continue - if path.startswith("cuda_python_test_helpers/cuda_python_test_helpers/"): - test_changes.update(("bindings", "core")) - elif path.startswith("benchmarks/cuda_bindings/"): - test_changes.add("bindings") - elif platforms := TEST_INFRA_PLATFORMS.get(path): - test_platforms.update(platforms) - elif not _is_independent(path): + is_test_path = any(part in {"test", "tests"} for part in path_parts[:-1]) + if is_test_path: + test_changes.update(MODULES) + elif ( + path in IGNORED_PATHS + or path_parts[-1] in IGNORED_BASENAMES + or PurePosixPath(path).suffix in IGNORED_SUFFIXES + or path.startswith(IGNORED_PREFIXES) + ): + continue + elif path_parts[0] in {"benchmarks", "cuda_python_test_helpers"}: + test_changes.update(MODULES) + else: force_all = True + break if force_all: builds = set(MODULES) @@ -208,13 +163,43 @@ def compute_workplan( } -def _changed_paths(merge_base: str, head: str) -> list[str]: +def _changed_paths(merge_base: str, head: str) -> tuple[list[str], set[str]]: result = subprocess.run( # noqa: S603 - argv is passed directly to git without a shell. ["git", "diff", "--no-renames", "--name-only", "-z", f"{merge_base}...{head}"], # noqa: S607 check=True, + cwd=REPO_ROOT, + stdout=subprocess.PIPE, + ) + paths = [path.decode("utf-8", errors="surrogateescape") for path in result.stdout.split(b"\0") if path] + head_symlinks = _tracked_symlink_paths(head) + # Base links preserve the packaging impact of deleted or replaced symlinks. + linked_paths = set(head_symlinks) | set(_tracked_symlink_paths(merge_base)) + return _expand_linked_paths(paths, head_symlinks, root=REPO_ROOT), linked_paths + + +def _tracked_symlink_paths(ref: str) -> list[str]: + result = subprocess.run( # noqa: S603 - the Git ref is passed as an argv element. + ["git", "ls-tree", "--full-tree", "-r", "-z", ref], # noqa: S607 + check=True, + cwd=REPO_ROOT, stdout=subprocess.PIPE, ) - return [path.decode("utf-8", errors="surrogateescape") for path in result.stdout.split(b"\0") if path] + return [ + entry.partition(b"\t")[2].decode("utf-8", errors="surrogateescape") + for entry in result.stdout.split(b"\0") + if entry.startswith(b"120000 ") + ] + + +def _expand_linked_paths(paths: list[str], symlink_paths: list[str], *, root: Path) -> list[str]: + """Include tracked symlinks whose resolved targets changed.""" + resolved_paths = {(root / path).resolve(strict=False) for path in paths} + expanded = list(paths) + selected = set(paths) + expanded.extend( + path for path in symlink_paths if path not in selected and (root / path).resolve(strict=False) in resolved_paths + ) + return expanded def main() -> None: @@ -231,12 +216,13 @@ def main() -> None: parser.error("baseline SHA must match the merge base") reusable_baseline = bool(args.merge_base and args.baseline_run_id) - paths = _changed_paths(args.merge_base, args.head) if reusable_baseline else [] + paths, linked_paths = _changed_paths(args.merge_base, args.head) if reusable_baseline else ([], set()) plan = compute_workplan( paths, merge_base=args.merge_base, baseline_run_id=args.baseline_run_id, baseline_sha=args.baseline_sha, + linked_paths=linked_paths, ) print(json.dumps(plan, separators=(",", ":"), sort_keys=True)) diff --git a/ci/tools/tests/test_compute_ci_plan.py b/ci/tools/tests/test_compute_ci_plan.py index 303b4c8ac89..c0687cd2cfe 100644 --- a/ci/tools/tests/test_compute_ci_plan.py +++ b/ci/tools/tests/test_compute_ci_plan.py @@ -4,20 +4,27 @@ from __future__ import annotations +import tempfile import unittest +from pathlib import Path -from ci.tools.compute_ci_plan import compute_workplan +from ci.tools.compute_ci_plan import _expand_linked_paths, compute_workplan ALL_MODULES = {"pathfinder", "bindings", "core", "python"} ALL_PLATFORMS = {"linux", "windows"} -def plan_for(*paths: str, baseline: bool = True) -> dict[str, object]: +def plan_for( + *paths: str, + baseline: bool = True, + linked_paths: set[str] | None = None, +) -> dict[str, object]: return compute_workplan( list(paths), merge_base="base", baseline_run_id="123" if baseline else "", baseline_sha="base" if baseline else "", + linked_paths=linked_paths, ) @@ -46,21 +53,23 @@ def test_path_impacts(self) -> None: False, ), "cuda_core/cuda/core/_device.py": ({"core"}, {"core", "python"}, True), + "cuda_core/cuda/core/examples/demo.py": ({"core"}, {"core", "python"}, True), "cuda_python/pyproject.toml": ({"bindings", "python"}, {"python"}, False), - "README.md": ({"bindings", "python"}, {"python"}, False), "cuda_pathfinder/tests/test_loader.py": (set(), {"pathfinder"}, False), "cuda_bindings/examples/0_Introduction/vectorAddDrv.py": (set(), {"bindings"}, False), + "cuda_bindings/tests/README.md": (set(), {"bindings"}, False), "cuda_core/pytest.ini": (set(), {"core"}, False), - "cuda_core/tests/fixtures/pixi.toml": (set(), {"core"}, False), "cuda_python/tests/test_import.py": (set(), {"python"}, False), - "cuda_python/pixi.toml": ({"bindings", "python"}, {"python"}, False), "cuda_python_test_helpers/cuda_python_test_helpers/cuda_utils.py": ( set(), - {"bindings", "core"}, + ALL_MODULES, False, ), - "ci/tools/run-tests": (set(), ALL_MODULES, False), + "benchmarks/cuda_bindings/run_pyperf.py": (set(), ALL_MODULES, False), + "benchmarks/cuda_core/runner.py": (set(), ALL_MODULES, False), + "ci/tools/run-tests": (ALL_MODULES, ALL_MODULES, True), "ci/versions.yml": (ALL_MODULES, ALL_MODULES, True), + "pytest.ini": (ALL_MODULES, ALL_MODULES, True), } for path, (builds, tests, core_api) in cases.items(): @@ -76,12 +85,10 @@ def test_test_infrastructure_platforms(self) -> None: cases = { ".github/workflows/test-wheel-linux.yml": {"linux"}, ".github/workflows/test-wheel-windows.yml": {"windows"}, - "ci/test-matrix.yml": ALL_PLATFORMS, "ci/tools/configure_driver_mode.ps1": {"windows"}, "ci/tools/guess_latest.sh": {"linux"}, "ci/tools/install_gpu_driver.ps1": {"windows"}, "ci/tools/install_gpu_driver.sh": {"linux"}, - "ci/tools/run-tests": ALL_PLATFORMS, "ci/tools/setup-sanitizer": {"linux"}, } @@ -104,10 +111,17 @@ def test_ignored_paths_select_no_work(self) -> None: for path in ( "cuda_core/docs/index.rst", "cuda_core/pixi.toml", + "cuda_core/tests/fixtures/pixi.toml", "benchmarks/cuda_bindings/pixi.toml", "benchmarks/cuda_bindings/AGENTS.md", - ".github/workflows/ci-pixi-source-test.yml", - "benchmarks/cuda_core/benchmark.py", + "cuda_core/cuda/core/_cpp/DESIGN.md", + "cuda_bindings/README.md", + "cuda_core/README.md", + "new-area/pixi.toml", + "notes.md", + "diagram.svg", + ".github/labeler.yml", + ".github/ISSUE_TEMPLATE/bug.yml", ): with self.subTest(path=path): plan = plan_for(path) @@ -118,8 +132,10 @@ def test_ignored_paths_select_no_work(self) -> None: def test_unknown_path_and_missing_baseline_force_all(self) -> None: for plan in ( plan_for("new-top-level-file"), - plan_for("new-area/pixi.toml"), + plan_for("new-area/config.toml"), plan_for(".github/workflows/new-main-ci-workflow.yml"), + plan_for(".github/actions/doc_preview/action.yml"), + plan_for("ci/ci-pipeline.svg"), plan_for("cuda_core/docs/index.rst", baseline=False), compute_workplan([], merge_base="base", baseline_run_id="123", baseline_sha=""), ): @@ -136,6 +152,30 @@ def test_mixed_changes_are_combined(self) -> None: assert selected_platforms(plan) == ALL_PLATFORMS assert plan["jobs"]["sdist_tests"] + def test_changed_symlink_targets_include_their_consumers(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "cuda_python").mkdir() + (root / "cuda_core").mkdir() + (root / "README.md").write_text("readme", encoding="utf-8") + (root / "cuda_python" / "README.md").symlink_to("../README.md") + (root / "cuda_core" / "README.md").symlink_to("../README.md") + + paths = _expand_linked_paths( + ["README.md"], + ["cuda_python/README.md"], + root=root, + ) + + assert paths == ["README.md", "cuda_python/README.md"] + plan = plan_for(*paths, linked_paths={"cuda_python/README.md"}) + assert selected(plan, "needs_build") == {"bindings", "python"} + assert selected(plan, "needs_test") == {"python"} + + removed_link = plan_for("cuda_python/README.md", linked_paths={"cuda_python/README.md"}) + assert selected(removed_link, "needs_build") == {"bindings", "python"} + assert selected(removed_link, "needs_test") == {"python"} + if __name__ == "__main__": unittest.main() From 44c6a3a75904ceb289d3c7e3cdd2441b18f35e0a Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Wed, 19 Aug 2026 11:25:09 -0400 Subject: [PATCH 09/13] ci: simplify workplan plumbing --- .github/workflows/ci.yml | 101 ++++++++++--------------- ci/tools/compute_ci_plan.py | 66 ++++++---------- ci/tools/tests/test_compute_ci_plan.py | 4 +- 3 files changed, 67 insertions(+), 104 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3de756b1ce9..a5cc29c8fa4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,9 +48,13 @@ jobs: should-skip: runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read outputs: skip: ${{ steps.get-should-skip.outputs.skip }} doc-only: ${{ steps.get-should-skip.outputs.doc_only }} + base-ref: ${{ steps.get-should-skip.outputs.base_ref }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -62,15 +66,19 @@ jobs: set -euxo pipefail if ${{ startsWith(github.ref_name, 'pull-request/') }}; then pr_number="$(grep -Po '(\d+)$' <<< '${{ github.ref_name }}')" - pr_title="$(gh pr view "${pr_number}" --json title --jq '.title')" + pr="$(gh pr view "${pr_number}" --json baseRefName,title)" + pr_title="$(jq -r '.title' <<< "${pr}")" + base_ref="$(jq -r '.baseRefName' <<< "${pr}")" skip="$(echo "${pr_title}" | grep -q '\[no-ci\]' && echo true || echo false)" doc_only="$(echo "${pr_title}" | grep -q '\[doc-only\]' && echo true || echo false)" else skip=false doc_only=false + base_ref="" fi echo "skip=${skip}" >> "$GITHUB_OUTPUT" echo "doc_only=${doc_only}" >> "$GITHUB_OUTPUT" + echo "base_ref=${base_ref}" >> "$GITHUB_OUTPUT" # Detect which packages were touched by the PR so downstream build and test # jobs can avoid rebuilding/retesting packages unaffected by the change. @@ -93,10 +101,10 @@ jobs: # paths" baseline for those events. detect-changes: runs-on: ubuntu-latest + needs: should-skip permissions: actions: read contents: read - pull-requests: read outputs: workplan: ${{ steps.workplan.outputs.workplan }} steps: @@ -108,32 +116,15 @@ jobs: fetch-depth: 0 filter: blob:none - # copy-pr-bot pushes every PR (whether it targets main or a backport - # branch such as 12.9.x) to pull-request/, so the base branch - # cannot be inferred from github.ref_name. Look it up via the - # upstream PR metadata so change detection is rooted at the right place. - - name: Resolve PR base branch - id: pr-info - if: ${{ startsWith(github.ref_name, 'pull-request/') }} - uses: nv-gha-runners/get-pr-info@main - - name: Resolve PR merge base id: merge-base if: ${{ startsWith(github.ref_name, 'pull-request/') }} env: - # GitHub Actions evaluates step-level `env:` expressions eagerly — - # the step's `if:` gate does NOT short-circuit them. On non-PR - # events (push/tag/schedule), `pr-info` is skipped and its outputs - # are empty strings, so `fromJSON('')` would raise a template error - # and fail the step despite `if:` being false. Guard the - # `fromJSON` call with a short-circuit so the expression resolves - # to an empty string on non-PR events; the step is still gated - # off by `if:`, so `BASE_REF` is never consumed there. - BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} + BASE_REF: ${{ needs.should-skip.outputs.base-ref }} run: | set -euo pipefail if [[ -z "${BASE_REF}" ]]; then - echo "Could not resolve PR base branch from get-pr-info output" >&2 + echo "Could not resolve PR base branch" >&2 exit 1 fi @@ -144,14 +135,13 @@ jobs: id: baseline if: ${{ startsWith(github.ref_name, 'pull-request/') }} env: - BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} + BASE_REF: ${{ needs.should-skip.outputs.base-ref }} MERGE_BASE: ${{ steps.merge-base.outputs.sha }} GH_TOKEN: ${{ github.token }} run: | set -uo pipefail unavailable() { - echo "available=false" >> "$GITHUB_OUTPUT" echo "No complete reusable artifact set was found; this run will build and test everything." >> "$GITHUB_STEP_SUMMARY" exit 0 } @@ -180,8 +170,8 @@ jobs: # PR diff base. Using the latest base-branch run is unsafe for a PR # that was opened before newer changes landed on that branch. run_id=$(jq -r '.[0].databaseId // empty' <<< "$runs") - baseline_sha=$(jq -r '.[0].headSha // empty' <<< "$runs") - if [[ -z "${run_id}" || "${baseline_sha}" != "${merge_base}" ]]; then + run_sha=$(jq -r '.[0].headSha // empty' <<< "$runs") + if [[ -z "${run_id}" || "${run_sha}" != "${merge_base}" ]]; then unavailable fi @@ -211,8 +201,8 @@ jobs: while IFS= read -r python_version; do python=${python_version//./} for platform in linux-64 linux-aarch64 win-64; do - binding="cuda-bindings-python${python}-cuda${cuda_version}-${platform}-${baseline_sha}" - core="cuda-core-python${python}-${platform}-${baseline_sha}" + binding="cuda-bindings-python${python}-cuda${cuda_version}-${platform}-${merge_base}" + core="cuda-core-python${python}-${platform}-${merge_base}" has_artifact "$binding" || missing+=("$binding") has_artifact "$core" || missing+=("$core") done @@ -223,14 +213,10 @@ jobs: unavailable fi - { - echo "available=true" - echo "run_id=${run_id}" - echo "sha=${baseline_sha}" - } >> "$GITHUB_OUTPUT" + echo "run_id=${run_id}" >> "$GITHUB_OUTPUT" { echo - echo "Reusable artifacts: run \`${run_id}\` at \`${baseline_sha}\` on \`${BASE_REF}\`." + echo "Reusable artifacts: run \`${run_id}\` at \`${merge_base}\` on \`${BASE_REF}\`." } >> "$GITHUB_STEP_SUMMARY" - name: Test CI workplan planner @@ -240,20 +226,12 @@ jobs: id: workplan env: MERGE_BASE: ${{ steps.merge-base.outputs.sha }} - BASELINE_AVAILABLE: ${{ steps.baseline.outputs.available }} BASELINE_RUN_ID: ${{ steps.baseline.outputs.run_id }} - BASELINE_SHA: ${{ steps.baseline.outputs.sha }} run: | set -euo pipefail - args=(--head "$GITHUB_SHA") - if [[ -n "$MERGE_BASE" ]]; then - args+=(--merge-base "$MERGE_BASE") - fi - if [[ "$BASELINE_AVAILABLE" == "true" ]]; then - args+=(--baseline-run-id "$BASELINE_RUN_ID" --baseline-sha "$BASELINE_SHA") - fi - - workplan=$(python3 ci/tools/compute_ci_plan.py "${args[@]}") + workplan=$(python3 ci/tools/compute_ci_plan.py \ + --merge-base "$MERGE_BASE" \ + --baseline-run-id "$BASELINE_RUN_ID") echo "workplan=$workplan" >> "$GITHUB_OUTPUT" { echo @@ -640,6 +618,8 @@ jobs: - precommit-windows steps: - name: Exit + env: + NEEDS_JSON: ${{ toJSON(needs) }} run: | # GitHub treats `result == 'skipped'` as success for required # status checks (see CCCL gate comment + cccl#605). The previous @@ -663,7 +643,8 @@ jobs: is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { - name=$1; expected=$2; result=$3 + local name=$1 expected=$2 result + result=$(jq -r --arg name "$name" '.[$name].result // "missing"' <<< "$NEEDS_JSON") echo "Checking $name: result='$result' (expected '$expected')" if [[ "$result" != "$expected" ]]; then echo "::error::$name did not match expected result" @@ -673,12 +654,12 @@ jobs: # Control jobs, the universal linux build, docs, and Windows # pre-commit checks always run. - check_result "ci-vars" "success" "${{ needs.ci-vars.result }}" - check_result "should-skip" "success" "${{ needs.should-skip.result }}" - check_result "detect-changes" "success" "${{ needs.detect-changes.result }}" - check_result "build-linux-64" "success" "${{ needs.build-linux-64.result }}" - check_result "doc" "success" "${{ needs.doc.result }}" - check_result "precommit-windows" "success" "${{ needs.precommit-windows.result }}" + check_result "ci-vars" "success" + check_result "should-skip" "success" + check_result "detect-changes" "success" + check_result "build-linux-64" "success" + check_result "doc" "success" + check_result "precommit-windows" "success" # Optional platform builds and wheel tests share the platform plan. linux_expected="skipped" @@ -689,29 +670,29 @@ jobs: if [[ "$doc_only" != "true" && "$windows_selected" == "true" ]]; then windows_expected="success" fi - check_result "build-linux-aarch64" "$linux_expected" "${{ needs.build-linux-aarch64.result }}" - check_result "build-windows" "$windows_expected" "${{ needs.build-windows.result }}" + check_result "build-linux-aarch64" "$linux_expected" + check_result "build-windows" "$windows_expected" # Sdist tests follow build selection; wheel tests follow the platform plan. expected="skipped" if [[ "$doc_only" != "true" && "$build_selected" == "true" ]]; then expected="success" fi - check_result "test-sdist-linux" "$expected" "${{ needs.test-sdist-linux.result }}" - check_result "test-sdist-windows" "$expected" "${{ needs.test-sdist-windows.result }}" + check_result "test-sdist-linux" "$expected" + check_result "test-sdist-windows" "$expected" - check_result "test-linux-64" "$linux_expected" "${{ needs.test-linux-64.result }}" - check_result "test-linux-aarch64" "$linux_expected" "${{ needs.test-linux-aarch64.result }}" - check_result "test-windows" "$windows_expected" "${{ needs.test-windows.result }}" + check_result "test-linux-64" "$linux_expected" + check_result "test-linux-aarch64" "$linux_expected" + check_result "test-windows" "$windows_expected" # API compatibility checks run for cuda_core source changes and for # conservative full runs when reusable base artifacts are unavailable. expected="skipped" if [[ "$run_core_api_check" == "true" ]]; then expected="success"; fi - check_result "api-check-core-vs-release" "$expected" "${{ needs.api-check-core-vs-release.result }}" + check_result "api-check-core-vs-release" "$expected" expected="skipped" if [[ "$is_pr" == "true" && "$run_core_api_check" == "true" ]]; then expected="success"; fi - check_result "api-check-core-vs-base" "$expected" "${{ needs.api-check-core-vs-base.result }}" + check_result "api-check-core-vs-base" "$expected" [[ "$status" == "success" ]] diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py index 0bf4328ffca..9f94c67173d 100644 --- a/ci/tools/compute_ci_plan.py +++ b/ci/tools/compute_ci_plan.py @@ -16,12 +16,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] MODULES = ("pathfinder", "bindings", "core", "python") PLATFORMS = ("linux", "windows") -PACKAGE_MODULES = { - "cuda_pathfinder": "pathfinder", - "cuda_bindings": "bindings", - "cuda_core": "core", - "cuda_python": "python", -} +PACKAGE_MODULES = {f"cuda_{module}": module for module in MODULES} # Source changes have different build and test consumers. In particular, # cuda-python source needs a same-version bindings wheel, while a core-only @@ -49,13 +44,13 @@ # Only infrastructure exclusive to one OS belongs here; other CI paths force a full run. TEST_INFRA_PLATFORMS = { - ".github/workflows/test-wheel-linux.yml": {"linux"}, - ".github/workflows/test-wheel-windows.yml": {"windows"}, - "ci/tools/configure_driver_mode.ps1": {"windows"}, - "ci/tools/guess_latest.sh": {"linux"}, - "ci/tools/install_gpu_driver.ps1": {"windows"}, - "ci/tools/install_gpu_driver.sh": {"linux"}, - "ci/tools/setup-sanitizer": {"linux"}, + ".github/workflows/test-wheel-linux.yml": "linux", + ".github/workflows/test-wheel-windows.yml": "windows", + "ci/tools/configure_driver_mode.ps1": "windows", + "ci/tools/guess_latest.sh": "linux", + "ci/tools/install_gpu_driver.ps1": "windows", + "ci/tools/install_gpu_driver.sh": "linux", + "ci/tools/setup-sanitizer": "linux", } @@ -64,7 +59,6 @@ def compute_workplan( *, merge_base: str, baseline_run_id: str, - baseline_sha: str, linked_paths: set[str] | None = None, ) -> dict[str, object]: """Return the final CI decisions for the supplied changed paths.""" @@ -72,7 +66,7 @@ def compute_workplan( source_changes: set[str] = set() test_changes: set[str] = set() test_platforms: set[str] = set() - force_all = not merge_base or not baseline_run_id or not baseline_sha + force_all = not merge_base or not baseline_run_id if not force_all: for path in paths: @@ -80,8 +74,8 @@ def compute_workplan( if not path_parts: continue - if platforms := TEST_INFRA_PLATFORMS.get(path): - test_platforms.update(platforms) + if platform := TEST_INFRA_PLATFORMS.get(path): + test_platforms.add(platform) continue if path_parts[0] == "ci" or ( @@ -115,7 +109,6 @@ def compute_workplan( test_changes.update(MODULES) elif ( path in IGNORED_PATHS - or path_parts[-1] in IGNORED_BASENAMES or PurePosixPath(path).suffix in IGNORED_SUFFIXES or path.startswith(IGNORED_PREFIXES) ): @@ -158,35 +151,32 @@ def compute_workplan( "merge_base": merge_base, "baseline": { "run_id": baseline_run_id if not force_all else "", - "sha": baseline_sha if not force_all else "", + "sha": merge_base if not force_all else "", }, } -def _changed_paths(merge_base: str, head: str) -> tuple[list[str], set[str]]: - result = subprocess.run( # noqa: S603 - argv is passed directly to git without a shell. - ["git", "diff", "--no-renames", "--name-only", "-z", f"{merge_base}...{head}"], # noqa: S607 - check=True, +def _git_output(*args: str) -> bytes: + return subprocess.check_output( # noqa: S603 - argv is passed directly without a shell. + ["git", *args], # noqa: S607 cwd=REPO_ROOT, - stdout=subprocess.PIPE, ) - paths = [path.decode("utf-8", errors="surrogateescape") for path in result.stdout.split(b"\0") if path] - head_symlinks = _tracked_symlink_paths(head) + + +def _changed_paths(merge_base: str) -> tuple[list[str], set[str]]: + output = _git_output("diff", "--no-renames", "--name-only", "-z", merge_base, "HEAD") + paths = [path.decode("utf-8", errors="surrogateescape") for path in output.split(b"\0") if path] + head_symlinks = _tracked_symlink_paths("HEAD") # Base links preserve the packaging impact of deleted or replaced symlinks. linked_paths = set(head_symlinks) | set(_tracked_symlink_paths(merge_base)) return _expand_linked_paths(paths, head_symlinks, root=REPO_ROOT), linked_paths def _tracked_symlink_paths(ref: str) -> list[str]: - result = subprocess.run( # noqa: S603 - the Git ref is passed as an argv element. - ["git", "ls-tree", "--full-tree", "-r", "-z", ref], # noqa: S607 - check=True, - cwd=REPO_ROOT, - stdout=subprocess.PIPE, - ) + output = _git_output("ls-tree", "--full-tree", "-r", "-z", ref) return [ entry.partition(b"\t")[2].decode("utf-8", errors="surrogateescape") - for entry in result.stdout.split(b"\0") + for entry in output.split(b"\0") if entry.startswith(b"120000 ") ] @@ -205,23 +195,15 @@ def _expand_linked_paths(paths: list[str], symlink_paths: list[str], *, root: Pa def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--merge-base", default="") - parser.add_argument("--head", default="HEAD") parser.add_argument("--baseline-run-id", default="") - parser.add_argument("--baseline-sha", default="") args = parser.parse_args() - if bool(args.baseline_run_id) != bool(args.baseline_sha): - parser.error("baseline run ID and SHA must be supplied together") - if args.baseline_sha and args.baseline_sha != args.merge_base: - parser.error("baseline SHA must match the merge base") - reusable_baseline = bool(args.merge_base and args.baseline_run_id) - paths, linked_paths = _changed_paths(args.merge_base, args.head) if reusable_baseline else ([], set()) + paths, linked_paths = _changed_paths(args.merge_base) if reusable_baseline else ([], set()) plan = compute_workplan( paths, merge_base=args.merge_base, baseline_run_id=args.baseline_run_id, - baseline_sha=args.baseline_sha, linked_paths=linked_paths, ) print(json.dumps(plan, separators=(",", ":"), sort_keys=True)) diff --git a/ci/tools/tests/test_compute_ci_plan.py b/ci/tools/tests/test_compute_ci_plan.py index c0687cd2cfe..79a83394dfa 100644 --- a/ci/tools/tests/test_compute_ci_plan.py +++ b/ci/tools/tests/test_compute_ci_plan.py @@ -23,7 +23,6 @@ def plan_for( list(paths), merge_base="base", baseline_run_id="123" if baseline else "", - baseline_sha="base" if baseline else "", linked_paths=linked_paths, ) @@ -137,7 +136,7 @@ def test_unknown_path_and_missing_baseline_force_all(self) -> None: plan_for(".github/actions/doc_preview/action.yml"), plan_for("ci/ci-pipeline.svg"), plan_for("cuda_core/docs/index.rst", baseline=False), - compute_workplan([], merge_base="base", baseline_run_id="123", baseline_sha=""), + compute_workplan([], merge_base="", baseline_run_id="123"), ): assert selected(plan, "needs_build") == ALL_MODULES assert selected(plan, "needs_test") == ALL_MODULES @@ -151,6 +150,7 @@ def test_mixed_changes_are_combined(self) -> None: assert selected(plan, "needs_test") == {"core", "python"} assert selected_platforms(plan) == ALL_PLATFORMS assert plan["jobs"]["sdist_tests"] + assert plan["baseline"] == {"run_id": "123", "sha": "base"} def test_changed_symlink_targets_include_their_consumers(self) -> None: with tempfile.TemporaryDirectory() as directory: From 727ef5994e41212e09a3adaff1db8fbdf126f7cc Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Wed, 19 Aug 2026 17:29:02 -0400 Subject: [PATCH 10/13] Derive reusable artifact platforms from test matrix --- .github/workflows/ci.yml | 9 ++++++--- ci/test-matrix.yml | 6 ++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5cc29c8fa4..2a1068fb7b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -195,17 +195,20 @@ jobs: if ! python_versions=$(yq -r '.jobs.build.strategy.matrix."python-version"[]' .github/workflows/build-wheel.yml); then unavailable fi - if [[ -z "${python_versions}" ]]; then + if ! platforms=$(yq -r '.platforms[]' ci/test-matrix.yml); then + unavailable + fi + if [[ -z "${python_versions}" || -z "${platforms}" ]]; then unavailable fi while IFS= read -r python_version; do python=${python_version//./} - for platform in linux-64 linux-aarch64 win-64; do + while IFS= read -r platform; do binding="cuda-bindings-python${python}-cuda${cuda_version}-${platform}-${merge_base}" core="cuda-core-python${python}-${platform}-${merge_base}" has_artifact "$binding" || missing+=("$binding") has_artifact "$core" || missing+=("$core") - done + done <<< "${platforms}" done <<< "${python_versions}" if (( ${#missing[@]} != 0 )); then diff --git a/ci/test-matrix.yml b/ci/test-matrix.yml index 563774494e9..66f3196ab68 100644 --- a/ci/test-matrix.yml +++ b/ci/test-matrix.yml @@ -32,6 +32,12 @@ # ENV: { CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM: '1' } # ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.12.1', TORCH_CUDA: 'cu126' } +# Host platforms that produce wheel artifacts in the main CI workflow. +platforms: + - linux-64 + - linux-aarch64 + - win-64 + linux: pull-request: # linux-64 From 30da89194ceee804150dfa1b735cccbe1fef0063 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Thu, 20 Aug 2026 01:12:01 -0400 Subject: [PATCH 11/13] ci: route selective CI through Moon --- .github/workflows/build-docs.yml | 149 ++++-- .github/workflows/build-wheel.yml | 291 ++++++----- .github/workflows/ci.yml | 341 +++++++++---- .github/workflows/test-sdist-linux.yml | 93 ++-- .github/workflows/test-sdist-windows.yml | 84 ++-- .github/workflows/test-wheel-linux.yml | 121 ++--- .github/workflows/test-wheel-windows.yml | 123 +++-- .gitignore | 1 + .moon/workspace.yml | 15 + ci/test-matrix.yml | 6 - ci/tools/compute_ci_plan.py | 213 -------- ci/tools/tests/test_compute_ci_plan.py | 181 ------- cuda_bindings/moon.yml | 215 ++++++++ cuda_core/moon.yml | 310 ++++++++++++ cuda_pathfinder/moon.yml | 109 ++++ cuda_python/moon.yml | 194 ++++++++ moon.yml | 111 +++++ tests/test_moon_ci.py | 600 +++++++++++++++++++++++ toolshed/check_spdx.py | 2 + 19 files changed, 2311 insertions(+), 848 deletions(-) create mode 100644 .moon/workspace.yml delete mode 100644 ci/tools/compute_ci_plan.py delete mode 100644 ci/tools/tests/test_compute_ci_plan.py create mode 100644 cuda_bindings/moon.yml create mode 100644 cuda_core/moon.yml create mode 100644 cuda_pathfinder/moon.yml create mode 100644 cuda_python/moon.yml create mode 100644 moon.yml create mode 100644 tests/test_moon_ci.py diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 7bb70809556..8f3c58ed67a 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -18,6 +18,11 @@ on: # - cuda-python # - cuda-pathfinder # - all + targets: + description: "JSON array of exact Moon docs targets. Empty derives targets from component." + required: false + default: "" + type: string git-tag: description: "Target git tag to build docs for" required: false @@ -60,6 +65,39 @@ jobs: fetch-depth: 1 ref: ${{ inputs.git-tag }} + - name: Resolve Moon docs targets + env: + REQUESTED_TARGETS: ${{ inputs.targets }} + COMPONENT: ${{ inputs.component }} + run: | + if [[ -n "$REQUESTED_TARGETS" ]]; then + targets="$REQUESTED_TARGETS" + else + case "$COMPONENT" in + all) targets='["root:docs"]' ;; + cuda-pathfinder) targets='["pathfinder:docs"]' ;; + cuda-bindings) targets='["bindings:docs"]' ;; + cuda-core) targets='["core:docs"]' ;; + cuda-python) targets='["metapackage:docs"]' ;; + *) echo "error: unsupported docs component: $COMPONENT" >&2; exit 1 ;; + esac + fi + jq -e ' + type == "array" and length > 0 and + all(.[]; + . == "root:docs" or + . == "pathfinder:docs" or + . == "bindings:docs" or + . == "core:docs" or + . == "metapackage:docs") + ' <<< "$targets" >/dev/null + echo "DOCS_TARGETS=$(jq -c . <<< "$targets")" >> "$GITHUB_ENV" + if [[ -f .moon/workspace.yml && -f moon.yml ]]; then + echo "DOCS_USE_MOON=true" >> "$GITHUB_ENV" + else + echo "DOCS_USE_MOON=false" >> "$GITHUB_ENV" + fi + - name: Read build CTK version run: | if [[ -f ci/versions.yml ]]; then @@ -94,6 +132,12 @@ jobs: conda config --show-sources conda config --show + - name: Install Moon + if: ${{ env.DOCS_USE_MOON == 'true' }} + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + # WAR: Building the doc currently requires CTK installed (NVIDIA/cuda-python#326,327) - name: Set up mini CTK uses: ./.github/actions/fetch_ctk @@ -132,7 +176,7 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel - path: . + path: ./cuda_python/dist run-id: ${{ inputs.run-id }} github-token: ${{ github.token }} @@ -145,7 +189,7 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel - path: ./cuda_pathfinder + path: ./cuda_pathfinder/dist run-id: ${{ inputs.run-id }} github-token: ${{ github.token }} @@ -200,7 +244,7 @@ jobs: - name: Install all packages run: | - pushd cuda_pathfinder + pushd cuda_pathfinder/dist pip install *.whl popd @@ -214,7 +258,7 @@ jobs: # Subpackages are already installed from CI artifacts above. # --no-deps avoids re-resolving cuda-core from PyPI during tag releases. - pip install --no-deps cuda_python*.whl + pip install --no-deps cuda_python/dist/*.whl # This step sets the PR_NUMBER/BUILD_LATEST/BUILD_PREVIEW env vars. - name: Get PR number @@ -227,42 +271,79 @@ jobs: # create an empty folder for removal use mkdir -p artifacts/empty_docs - - name: Build all docs - if: ${{ inputs.component == 'all' }} + - name: Build selected docs + if: ${{ env.DOCS_USE_MOON == 'true' }} + env: + DOCS_BUILD_ARGS: ${{ !inputs.is-release && 'latest-only' || '' }} run: | - pushd cuda_python/docs/ - if [[ "${{ inputs.is-release }}" == "false" ]]; then - ./build_all_docs.sh latest-only - else - ./build_all_docs.sh - # At release time, we don't want to update the latest docs - rm -rf build/html/latest - fi - ls -l build - popd - mv cuda_python/docs/build/html/* artifacts/docs/ + mapfile -t targets < <(jq -r '.[]' <<< "$DOCS_TARGETS") + moon run "${targets[@]}" --upstream deep --downstream none - - name: Build component docs - if: ${{ inputs.component != 'all' }} + # Release workflows may check out tags created before Moon was added. + - name: Build selected docs from a legacy tag + if: ${{ env.DOCS_USE_MOON != 'true' }} run: | - COMPONENT=$(echo "${{ inputs.component }}" | tr '-' '_') - pushd ${COMPONENT}/docs/ - if [[ "${{ inputs.is-release }}" == "false" ]]; then - ./build_docs.sh latest-only + if [[ "${{ inputs.component }}" == "all" ]]; then + pushd cuda_python/docs + if [[ "${{ inputs.is-release }}" == "false" ]]; then + ./build_all_docs.sh latest-only + else + ./build_all_docs.sh + rm -rf build/html/latest + fi + popd else - ./build_docs.sh - # At release time, we don't want to update the latest docs - rm -rf build/html/latest + component="${{ inputs.component }}" + component=${component//-/_} + pushd "$component/docs" + if [[ "${{ inputs.is-release }}" == "false" ]]; then + ./build_docs.sh latest-only + else + ./build_docs.sh + rm -rf build/html/latest + fi + popd fi - ls -l build - popd - if [[ "${{ inputs.component }}" != "cuda-python" ]]; then - TARGET="${{ inputs.component }}" - mkdir -p artifacts/docs/${TARGET} - else - TARGET="" + + - name: Assemble selected docs + run: | + if jq -e 'index("root:docs") != null' <<< "$DOCS_TARGETS" >/dev/null; then + if [[ "${{ inputs.is-release }}" == "true" ]]; then + rm -rf cuda_python/docs/build/html/latest + fi + ls -l cuda_python/docs/build + mv cuda_python/docs/build/html/* artifacts/docs/ + exit 0 fi - mv ${COMPONENT}/docs/build/html/* artifacts/docs/${TARGET} + + while IFS= read -r target; do + case "$target" in + pathfinder:docs) + component=cuda_pathfinder + destination=cuda-pathfinder + ;; + bindings:docs) + component=cuda_bindings + destination=cuda-bindings + ;; + core:docs) + component=cuda_core + destination=cuda-core + ;; + metapackage:docs) + component=cuda_python + destination= + ;; + esac + if [[ "${{ inputs.is-release }}" == "true" ]]; then + rm -rf "$component/docs/build/html/latest" + fi + ls -l "$component/docs/build" + if [[ -n "$destination" ]]; then + mkdir -p "artifacts/docs/$destination" + fi + mv "$component"/docs/build/html/* "artifacts/docs/$destination" + done < <(jq -r '.[]' <<< "$DOCS_TARGETS") - name: Write rendered docs file list if: ${{ !inputs.is-release && github.ref_name != 'main' && !startsWith(github.ref_name, 'release/') }} diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 7e233244843..2285f8e985d 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -31,12 +31,16 @@ permissions: jobs: build: env: - BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} - BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} - BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} - BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} - TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} - TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} + WHEEL_FOUNDATION_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-foundation']) || '["pathfinder:wheel"]' }} + WHEEL_BINDINGS_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-bindings']) || '["bindings:wheel"]' }} + WHEEL_CONSUMER_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-consumers']) || '["core:wheel","metapackage:wheel"]' }} + WHEEL_MULTI_CTK_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-multi-ctk']) || '["core:wheel"]' }} + WHEEL_FINALIZE_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-finalize']) || '["core:wheel-merge"]' }} + TEST_ASSETS_CURRENT_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-test-assets-current']) || '["bindings:ci-test-assets","core:ci-test-assets"]' }} + TEST_ASSETS_PREVIOUS_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-test-assets-previous']) || '["core:ci-test-binaries"]' }} + TEST_LINUX_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-test-linux']) || '["pathfinder:ci-test-linux"]' }} + TEST_WINDOWS_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-test-windows']) || '["pathfinder:ci-test-windows"]' }} + HOST_PLATFORM: ${{ inputs.host-platform }} BASELINE_RUN_ID: ${{ inputs.workplan != '' && fromJSON(inputs.workplan).baseline.run_id || '' }} BASELINE_SHA: ${{ inputs.workplan != '' && fromJSON(inputs.workplan).baseline.sha || '' }} strategy: @@ -64,6 +68,63 @@ jobs: fetch-depth: 0 filter: blob:none + - name: Install target resolver dependencies + if: ${{ startsWith(inputs.host-platform, 'linux') }} + uses: ./.github/actions/install_unix_deps + with: + dependencies: "jq" + dependent_exes: "jq" + + - name: Resolve Moon phase targets + run: | + for value in \ + "$WHEEL_FOUNDATION_TARGETS" \ + "$WHEEL_BINDINGS_TARGETS" \ + "$WHEEL_CONSUMER_TARGETS" \ + "$WHEEL_MULTI_CTK_TARGETS" \ + "$WHEEL_FINALIZE_TARGETS" \ + "$TEST_ASSETS_CURRENT_TARGETS" \ + "$TEST_ASSETS_PREVIOUS_TARGETS" \ + "$TEST_LINUX_TARGETS" \ + "$TEST_WINDOWS_TARGETS"; do + jq -e 'type == "array" and all(.[]; type == "string")' <<< "$value" >/dev/null + done + + has_target() { + jq -e --arg target "$2" 'index($target) != null' <<< "$1" >/dev/null + } + if [[ "$HOST_PLATFORM" == "win-64" ]]; then + platform_test_targets="$TEST_WINDOWS_TARGETS" + else + platform_test_targets="$TEST_LINUX_TARGETS" + fi + run_test_assets=$(jq -r 'length > 0' <<< "$platform_test_targets") + { + echo "BUILD_PATHFINDER=$(has_target "$WHEEL_FOUNDATION_TARGETS" pathfinder:wheel && echo true || echo false)" + echo "BUILD_BINDINGS=$(has_target "$WHEEL_BINDINGS_TARGETS" bindings:wheel && echo true || echo false)" + echo "BUILD_CORE_CURRENT=$(has_target "$WHEEL_CONSUMER_TARGETS" core:wheel && echo true || echo false)" + echo "BUILD_CORE_PREVIOUS=$(has_target "$WHEEL_MULTI_CTK_TARGETS" core:wheel && echo true || echo false)" + echo "FINALIZE_CORE=$(has_target "$WHEEL_FINALIZE_TARGETS" core:wheel-merge && echo true || echo false)" + if has_target "$WHEEL_CONSUMER_TARGETS" core:wheel || \ + has_target "$WHEEL_MULTI_CTK_TARGETS" core:wheel || \ + has_target "$WHEEL_FINALIZE_TARGETS" core:wheel-merge; then + echo "BUILD_CORE=true" + else + echo "BUILD_CORE=false" + fi + echo "BUILD_PYTHON=$(has_target "$WHEEL_CONSUMER_TARGETS" metapackage:wheel && echo true || echo false)" + echo "TEST_BINDINGS=$(if [[ "$run_test_assets" == "true" ]]; then has_target "$TEST_ASSETS_CURRENT_TARGETS" bindings:ci-test-assets && echo true || echo false; else echo false; fi)" + echo "TEST_CORE_CURRENT=$(if [[ "$run_test_assets" == "true" ]]; then has_target "$TEST_ASSETS_CURRENT_TARGETS" core:ci-test-assets && echo true || echo false; else echo false; fi)" + echo "TEST_CORE_PREVIOUS=$(if [[ "$run_test_assets" == "true" ]]; then has_target "$TEST_ASSETS_PREVIOUS_TARGETS" core:ci-test-binaries && echo true || echo false; else echo false; fi)" + if [[ "$run_test_assets" == "true" ]] && \ + (has_target "$TEST_ASSETS_CURRENT_TARGETS" core:ci-test-assets || \ + has_target "$TEST_ASSETS_PREVIOUS_TARGETS" core:ci-test-binaries); then + echo "TEST_CORE=true" + else + echo "TEST_CORE=false" + fi + } >> "$GITHUB_ENV" + - name: Install latest rapidsai/sccache if: ${{ startsWith(inputs.host-platform, 'linux') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} run: | @@ -99,6 +160,12 @@ jobs: # see https://github.com/actions/setup-python/issues/871 python-version: "3.12" + - name: Install Moon and build tools + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + python -m pip install cibuildwheel twine wheel + - name: Set up MSVC if: ${{ startsWith(inputs.host-platform, 'win') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') }} uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 @@ -138,24 +205,19 @@ jobs: run: | env - - name: Install twine - run: | - pip install twine - # To keep the build workflow simple, all matrix jobs will build a wheel for later use within this workflow. - name: Build and check cuda.pathfinder wheel if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | - pushd cuda_pathfinder - pip wheel -v --no-deps . - popd + mapfile -t targets < <(jq -r '.[]' <<< "$WHEEL_FOUNDATION_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Download reusable cuda.pathfinder wheel if: ${{ env.BUILD_PATHFINDER != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel - path: cuda_pathfinder + path: cuda_pathfinder/dist github-token: ${{ github.token }} run-id: ${{ env.BASELINE_RUN_ID }} @@ -166,20 +228,20 @@ jobs: else export CHOWN="sudo chown" fi - $CHOWN -R $(whoami) cuda_pathfinder/*.whl - ls -lahR cuda_pathfinder + $CHOWN -R $(whoami) cuda_pathfinder/dist/*.whl + ls -lahR cuda_pathfinder/dist # We only need/want a single pure python wheel, pick linux-64 index 0. # This is what we will use for testing & releasing. - name: Check cuda.pathfinder wheel if: ${{ env.BUILD_PATHFINDER == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} run: | - twine check --strict cuda_pathfinder/*.whl + twine check --strict cuda_pathfinder/dist/*.whl - name: Constrain builds to the local cuda.pathfinder wheel if: ${{ env.BUILD_BINDINGS == 'true' }} run: | - pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) + pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test -f "${pathfinder_wheels[0]}" mkdir -p wheel-constraints @@ -195,7 +257,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: cuda-pathfinder-wheel - path: cuda_pathfinder/*.whl + path: cuda_pathfinder/dist/*.whl if-no-files-found: error - name: Set up mini CTK @@ -208,10 +270,9 @@ jobs: - name: Build cuda.bindings wheel if: ${{ env.BUILD_BINDINGS == 'true' }} - uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 - with: - package-dir: ./cuda_bindings/ - output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} + run: | + mapfile -t targets < <(jq -r '.[]' <<< "$WHEEL_BINDINGS_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none env: CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"' @@ -285,9 +346,9 @@ jobs: twine check --strict ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl - name: Constrain cuda.core to the local cuda.bindings wheel - if: ${{ env.BUILD_CORE == 'true' }} + if: ${{ env.BUILD_CORE_CURRENT == 'true' }} run: | - pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) + pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) bindings_wheels=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-"${BUILD_CUDA_MAJOR}".*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test "${#bindings_wheels[@]}" -eq 1 @@ -313,13 +374,19 @@ jobs: path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl if-no-files-found: error - - name: Build cuda.core wheel - if: ${{ env.BUILD_CORE == 'true' }} - uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 - with: - package-dir: ./cuda_core/ - output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + - name: Build current-context consumer wheels + if: ${{ env.BUILD_CORE_CURRENT == 'true' || (env.BUILD_PYTHON == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64') }} + run: | + targets="$WHEEL_CONSUMER_TARGETS" + if [[ "${{ inputs.host-platform }}" != "linux-64" || "${{ strategy.job-index }}" != "0" ]]; then + targets=$(jq -c 'map(select(. != "metapackage:wheel"))' <<< "$targets") + fi + mapfile -t target_args < <(jq -r '.[]' <<< "$targets") + if (( ${#target_args[@]} != 0 )); then + moon run "${target_args[@]}" --upstream none --downstream none + fi env: + CUDA_CORE_BUILD_MAJOR: ${{ env.BUILD_CUDA_MAJOR }} CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"' CIBW_BEFORE_BUILD_WINDOWS: 'python -m pip install --upgrade "pip>=25.3" delvewheel' @@ -362,15 +429,15 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.core) - if: ${{ env.BUILD_CORE == 'true' && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_CORE_CURRENT == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_core.json label: "cuda.core" build-step: "Build cuda.core wheel" - - name: List the cuda.core artifacts directory and rename - if: ${{ env.BUILD_CORE == 'true' }} + - name: List the cuda.core artifacts directory + if: ${{ env.BUILD_CORE_CURRENT == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == win* ]]; then export CHOWN=chown @@ -378,18 +445,6 @@ jobs: export CHOWN="sudo chown" fi $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - - # Rename wheel to include CUDA version suffix - mkdir -p "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}" - for wheel in ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl; do - if [[ -f "${wheel}" ]]; then - base_name=$(basename "${wheel}" .whl) - new_name="${base_name}.cu${BUILD_CUDA_MAJOR}.whl" - mv "${wheel}" "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}/${new_name}" - echo "Renamed wheel to: ${new_name}" - fi - done - ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - name: Download reusable cuda.core wheel @@ -401,21 +456,12 @@ jobs: github-token: ${{ github.token }} run-id: ${{ env.BASELINE_RUN_ID }} - # We only need/want a single pure python wheel, pick linux-64 index 0. - - name: Build and check cuda-python wheel - if: ${{ env.BUILD_PYTHON == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} - run: | - pushd cuda_python - pip wheel -v --no-deps . - twine check --strict *.whl - popd - - name: Download reusable cuda-python wheel if: ${{ env.BUILD_PYTHON != 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel - path: cuda_python + path: cuda_python/dist github-token: ${{ github.token }} run-id: ${{ env.BASELINE_RUN_ID }} @@ -427,38 +473,42 @@ jobs: else export CHOWN="sudo chown" fi - $CHOWN -R $(whoami) cuda_python/*.whl - ls -lahR cuda_python + $CHOWN -R $(whoami) cuda_python/dist/*.whl + ls -lahR cuda_python/dist - name: Upload cuda-python build artifacts if: ${{ strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: cuda-python-wheel - path: cuda_python/*.whl + path: cuda_python/dist/*.whl if-no-files-found: error - name: Set up Python id: setup-python2 - if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} # TODO: remove allow-prereleases once 3.15 is officially supported allow-prereleases: ${{ startsWith(matrix.python-version, '3.15') }} + - name: Reinstall build tools for the selected Python + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} + run: python -m pip install cibuildwheel twine wheel + - name: Enable Scientific Python Nightly Wheels for Python 3.15 - if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && startsWith(matrix.python-version, '3.15') }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true') && startsWith(matrix.python-version, '3.15') }} run: | echo "PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" >> "$GITHUB_ENV" echo "PIP_ONLY_BINARY=numpy" >> "$GITHUB_ENV" - name: verify free-threaded build - if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && endsWith(matrix.python-version, 't') }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true') && endsWith(matrix.python-version, 't') }} run: python -c 'import sys; assert not sys._is_gil_enabled()' - name: Set up Python include paths - if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == linux* ]]; then echo "CPLUS_INCLUDE_PATH=${Python3_ROOT_DIR}/include/python${{ matrix.python-version }}" >> $GITHUB_ENV @@ -469,24 +519,41 @@ jobs: echo "PY_EXT_SUFFIX=$(python -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")" >> $GITHUB_ENV - name: Install cuda.pathfinder (required for next step) - if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} run: | - pip install cuda_pathfinder/*.whl + pip install cuda_pathfinder/dist/*.whl - name: Hide GNU link.exe so Meson finds MSVC link.exe - if: ${{ startsWith(inputs.host-platform, 'win') && (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') }} + if: ${{ startsWith(inputs.host-platform, 'win') && (env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true') }} run: | if [ -f "/c/Program Files/Git/usr/bin/link.exe" ]; then mv "/c/Program Files/Git/usr/bin/link.exe" "/c/Program Files/Git/usr/bin/link.exe.bak" fi - - name: Build cuda.bindings Cython tests - if: ${{ env.TEST_BINDINGS == 'true' }} + - name: Install wheels for current-context native test assets + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} run: | - pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl --group ./cuda_bindings/pyproject.toml:test - pushd ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }} - bash build_tests.sh - popd + if [[ "$TEST_BINDINGS" == "true" ]]; then + pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl --group ./cuda_bindings/pyproject.toml:test + fi + if [[ "$TEST_CORE_CURRENT" == "true" ]]; then + pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl + if [[ "$BUILD_CORE_CURRENT" == "true" ]]; then + core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}" -maxdepth 1 -type f -name '*.whl' -print -quit) + else + core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" -maxdepth 1 -type f -name '*.whl' -print -quit) + fi + test -n "$core_wheel" + pip install "$core_wheel" --group ./cuda_core/pyproject.toml:test + fi + + - name: Build current-context native test assets + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} + env: + CUDA_CORE_BUILD_MAJOR: ${{ env.BUILD_CUDA_MAJOR }} + run: | + mapfile -t targets < <(jq -r '.[]' <<< "$TEST_ASSETS_CURRENT_TARGETS") + moon run "${targets[@]}" --upstream direct --downstream none - name: Upload cuda.bindings Cython tests if: ${{ env.TEST_BINDINGS == 'true' }} @@ -496,26 +563,8 @@ jobs: path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }}/test_*${{ env.PY_EXT_SUFFIX }} if-no-files-found: error - - name: Build cuda.core Cython tests - if: ${{ env.TEST_CORE == 'true' }} - run: | - pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl - if ${{ env.BUILD_CORE == 'true' }}; then - core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}" -maxdepth 1 -type f -name '*.whl' -print -quit) - else - core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" -maxdepth 1 -type f -name '*.whl' -print -quit) - fi - if [[ -z "${core_wheel}" ]]; then - echo "No cuda.core wheel found" >&2 - exit 1 - fi - pip install "${core_wheel}" --group ./cuda_core/pyproject.toml:test - pushd ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }} - bash build_tests.sh - popd - - name: Upload cuda.core Cython tests - if: ${{ env.TEST_CORE == 'true' }} + if: ${{ env.TEST_CORE_CURRENT == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests @@ -524,7 +573,7 @@ jobs: # Note: This overwrites CUDA_PATH etc - name: Set up mini CTK - if: ${{ env.BUILD_CORE == 'true' || env.TEST_CORE == 'true' }} + if: ${{ env.BUILD_CORE_PREVIOUS == 'true' || env.TEST_CORE_PREVIOUS == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -532,14 +581,14 @@ jobs: cuda-version: ${{ inputs.prev-cuda-version }} cuda-path: "./cuda_toolkit_prev" - - name: Build cuda.core test binaries - if: ${{ env.TEST_CORE == 'true' }} + - name: Build previous-context native test assets + if: ${{ env.TEST_CORE_PREVIOUS == 'true' }} run: | - nvcc --version - python "${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/build_test_binaries.py" + mapfile -t targets < <(jq -r '.[]' <<< "$TEST_ASSETS_PREVIOUS_TARGETS") + moon run "${targets[@]}" --upstream direct --downstream none - name: Upload cuda.core test binaries - if: ${{ env.TEST_CORE == 'true' }} + if: ${{ env.TEST_CORE_PREVIOUS == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries @@ -550,7 +599,7 @@ jobs: if-no-files-found: error - name: Download cuda.bindings build artifacts from the prior branch - if: ${{ env.BUILD_CORE == 'true' }} + if: ${{ env.BUILD_CORE_PREVIOUS == 'true' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -586,9 +635,9 @@ jobs: rmdir "${OLD_ARTIFACT_DIR}" - name: Constrain previous cuda.core to the downloaded cuda.bindings wheel - if: ${{ env.BUILD_CORE == 'true' }} + if: ${{ env.BUILD_CORE_PREVIOUS == 'true' }} run: | - pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) + pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) bindings_wheels=(cuda_bindings/dist-prev/cuda_bindings-"${BUILD_PREV_CUDA_MAJOR}".*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test "${#bindings_wheels[@]}" -eq 1 @@ -607,13 +656,13 @@ jobs: printf 'cuda-bindings @ %s\n' "${bindings_uri}" } | tee wheel-constraints/cuda-core-prev.txt - - name: Build cuda.core wheel - if: ${{ env.BUILD_CORE == 'true' }} - uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 - with: - package-dir: ./cuda_core/ - output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + - name: Build previous-context cuda.core wheel + if: ${{ env.BUILD_CORE_PREVIOUS == 'true' }} + run: | + mapfile -t targets < <(jq -r '.[]' <<< "$WHEEL_MULTI_CTK_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none env: + CUDA_CORE_BUILD_MAJOR: ${{ env.BUILD_PREV_CUDA_MAJOR }} CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"' CIBW_BEFORE_BUILD_WINDOWS: 'python -m pip install --upgrade "pip>=25.3" delvewheel' @@ -656,15 +705,15 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.core prev) - if: ${{ env.BUILD_CORE == 'true' && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_CORE_PREVIOUS == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_core_prev.json label: "cuda.core (prev CTK)" build-step: "Build cuda.core wheel" - - name: List the cuda.core artifacts directory and rename - if: ${{ env.BUILD_CORE == 'true' }} + - name: List the previous-context cuda.core artifacts + if: ${{ env.BUILD_CORE_PREVIOUS == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == win* ]]; then export CHOWN=chown @@ -674,30 +723,14 @@ jobs: $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }} ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - # Rename wheel to include CUDA version suffix - mkdir -p "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_PREV_CUDA_MAJOR}" - for wheel in ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl; do - if [[ -f "${wheel}" ]]; then - base_name=$(basename "${wheel}" .whl) - new_name="${base_name}.cu${BUILD_PREV_CUDA_MAJOR}.whl" - mv "${wheel}" "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_PREV_CUDA_MAJOR}/${new_name}" - echo "Renamed wheel to: ${new_name}" - fi - done - - ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - - name: Merge cuda.core wheels - if: ${{ env.BUILD_CORE == 'true' }} + if: ${{ env.FINALIZE_CORE == 'true' }} run: | - pip install wheel - python ci/tools/merge_cuda_core_wheels.py \ - "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cu"${BUILD_CUDA_MAJOR}"/cuda_core*.whl \ - "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cu"${BUILD_PREV_CUDA_MAJOR}"/cuda_core*.whl \ - --output-dir "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" + mapfile -t targets < <(jq -r '.[]' <<< "$WHEEL_FINALIZE_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Check cuda.core wheel - if: ${{ env.BUILD_CORE == 'true' }} + if: ${{ env.FINALIZE_CORE == 'true' }} run: | twine check --strict ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a1068fb7b0..a65710331df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,25 +80,9 @@ jobs: echo "doc_only=${doc_only}" >> "$GITHUB_OUTPUT" echo "base_ref=${base_ref}" >> "$GITHUB_OUTPUT" - # Detect which packages were touched by the PR so downstream build and test - # jobs can avoid rebuilding/retesting packages unaffected by the change. - # See issue #299. - # - # Dependency graph (verified in pyproject.toml files): - # cuda_pathfinder -> (no internal deps) - # cuda_bindings -> cuda_pathfinder - # cuda_core -> cuda_pathfinder, cuda_bindings - # cuda_python -> cuda_pathfinder, cuda_bindings, cuda_core (meta package) - # - # A change to cuda_pathfinder (or shared infra) forces a rebuild of every - # downstream module. A change to cuda_bindings forces rebuild of cuda_core. - # A change to cuda_core alone skips rebuilding/retesting cuda_bindings and - # cuda_pathfinder, but still retests the downstream cuda-python metapackage. - # Shared build/orchestration changes run the full pipeline; test-only CI - # infrastructure runs every test suite without rebuilding package wheels. - # On push to main, tag refs, schedule, or workflow_dispatch events we - # unconditionally run everything because there is no meaningful "changed - # paths" baseline for those events. + # Moon owns file ownership, package impact, and the task graph. This job only + # establishes whether trusted artifacts may be reused and groups Moon's + # directly affected tasks by semantic CI phase for the heterogeneous runners. detect-changes: runs-on: ubuntu-latest needs: should-skip @@ -116,6 +100,11 @@ jobs: fetch-depth: 0 filter: blob:none + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + - name: Resolve PR merge base id: merge-base if: ${{ startsWith(github.ref_name, 'pull-request/') }} @@ -161,7 +150,7 @@ jobs: --event push \ --workflow ci.yml \ --status success \ - --limit 1 \ + --limit 100 \ --json databaseId,headSha); then unavailable fi @@ -169,21 +158,26 @@ jobs: # Reuse only artifacts produced from the exact commit used as the # PR diff base. Using the latest base-branch run is unsafe for a PR # that was opened before newer changes landed on that branch. + if [[ $(jq 'length' <<< "$runs") -ne 1 ]]; then + unavailable + fi run_id=$(jq -r '.[0].databaseId // empty' <<< "$runs") run_sha=$(jq -r '.[0].headSha // empty' <<< "$runs") if [[ -z "${run_id}" || "${run_sha}" != "${merge_base}" ]]; then unavailable fi - if ! artifact_names=$(gh api \ + if ! artifacts=$(gh api \ "repos/${{ github.repository }}/actions/runs/${run_id}/artifacts?per_page=100" \ --paginate \ - --jq '.artifacts[] | select(.expired == false) | .name'); then + --jq '.artifacts[] | {name, expired}'); then unavailable fi has_artifact() { - grep -Fxq "$1" <<< "$artifact_names" + jq -se --arg name "$1" \ + '[.[] | select(.name == $name)] | length == 1 and .[0].expired == false' \ + <<< "$artifacts" >/dev/null } missing=() @@ -195,20 +189,17 @@ jobs: if ! python_versions=$(yq -r '.jobs.build.strategy.matrix."python-version"[]' .github/workflows/build-wheel.yml); then unavailable fi - if ! platforms=$(yq -r '.platforms[]' ci/test-matrix.yml); then - unavailable - fi - if [[ -z "${python_versions}" || -z "${platforms}" ]]; then + if [[ -z "${python_versions}" ]]; then unavailable fi while IFS= read -r python_version; do python=${python_version//./} - while IFS= read -r platform; do + for platform in linux-64 linux-aarch64 win-64; do binding="cuda-bindings-python${python}-cuda${cuda_version}-${platform}-${merge_base}" core="cuda-core-python${python}-${platform}-${merge_base}" has_artifact "$binding" || missing+=("$binding") has_artifact "$core" || missing+=("$core") - done <<< "${platforms}" + done done <<< "${python_versions}" if (( ${#missing[@]} != 0 )); then @@ -222,19 +213,104 @@ jobs: echo "Reusable artifacts: run \`${run_id}\` at \`${merge_base}\` on \`${BASE_REF}\`." } >> "$GITHUB_STEP_SUMMARY" - - name: Test CI workplan planner - run: python3 -m unittest ci/tools/tests/test_compute_ci_plan.py - - - name: Compute CI workplan + - name: Compute Moon CI workplan id: workplan env: MERGE_BASE: ${{ steps.merge-base.outputs.sha }} BASELINE_RUN_ID: ${{ steps.baseline.outputs.run_id }} + DOC_ONLY: ${{ needs.should-skip.outputs.doc-only }} run: | set -euo pipefail - workplan=$(python3 ci/tools/compute_ci_plan.py \ - --merge-base "$MERGE_BASE" \ - --baseline-run-id "$BASELINE_RUN_ID") + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + export PATH="$HOME/.moon/bin:$PATH" + uvx --from pytest pytest -q tests/test_moon_ci.py + + semantic_tags='[ + "ci-wheel-foundation", + "ci-wheel-bindings", + "ci-wheel-consumers", + "ci-wheel-multi-ctk", + "ci-wheel-finalize", + "ci-sdist-foundation", + "ci-sdist-bindings", + "ci-sdist-consumers", + "ci-test-assets-current", + "ci-test-assets-previous", + "ci-test-linux", + "ci-test-windows", + "ci-docs", + "ci-api", + "ci-ignore", + "ci-force-all" + ]' + + visible_tasks() { + jq -c '[ + .tasks + | to_entries[] as $project + | $project.value + | to_entries[] + | select(.value.options.internal != true) + | { + target: "\($project.key):\(.key)", + tags: (.value.tags // []) + } + ]' + } + + force_all=false + selected='[]' + if [[ -z "${MERGE_BASE}" || -z "${BASELINE_RUN_ID}" ]]; then + force_all=true + else + git diff --no-renames --name-only -z "${MERGE_BASE}"...HEAD > changed-paths + while IFS= read -r -d '' path; do + [[ -n "${path}" ]] || continue + result=$(printf '%s\n' "${path}" | moon query tasks --affected stdin --upstream none --downstream none) + owned=$(visible_tasks <<< "${result}") + if jq -e 'length == 0 or any(.[]; .target == "root:ci-fallback")' <<< "${owned}" >/dev/null; then + force_all=true + break + fi + selected=$(jq -cn \ + --argjson current "${selected}" \ + --argjson next "${owned}" \ + '$current + $next | unique_by(.target)') + done < changed-paths + fi + + if [[ "${force_all}" == "true" ]]; then + selected=$(moon query tasks | visible_tasks) + baseline_run_id="" + baseline_sha="" + else + baseline_run_id="${BASELINE_RUN_ID}" + baseline_sha="${MERGE_BASE}" + fi + + # Preserve the established [doc-only] behavior: build the complete + # documentation site even when the changed paths do not own docs. + if [[ "${DOC_ONLY}" == "true" ]]; then + docs=$(moon query tasks | visible_tasks | jq -c '[.[] | select(.tags | index("ci-docs"))]') + selected=$(jq -cn \ + --argjson current "${selected}" \ + --argjson docs "${docs}" \ + '$current + $docs | unique_by(.target)') + fi + + targets=$(jq -cn \ + --argjson tags "${semantic_tags}" \ + --argjson selected "${selected}" \ + 'reduce $tags[] as $tag ({}; + .[$tag] = ([$selected[] + | select(.tags | index($tag)) + | .target] | unique | sort))') + workplan=$(jq -cn \ + --argjson targets "${targets}" \ + --arg merge_base "${MERGE_BASE}" \ + --arg baseline_run_id "${baseline_run_id}" \ + --arg baseline_sha "${baseline_sha}" \ + '{targets: $targets, merge_base: $merge_base, baseline: {run_id: $baseline_run_id, sha: $baseline_sha}}') echo "workplan=$workplan" >> "$GITHUB_OUTPUT" { echo @@ -248,7 +324,7 @@ jobs: name: API check (cuda_core vs. latest release) if: >- ${{ !fromJSON(needs.should-skip.outputs.skip) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks }} + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-api'][0] }} runs-on: ubuntu-latest needs: - should-skip @@ -286,20 +362,28 @@ jobs: git fetch --depth=1 --filter=blob:none origin \ "refs/tags/${{ steps.latest-tag.outputs.tag }}:refs/tags/${{ steps.latest-tag.outputs.tag }}" - - name: Check cuda_core public API - id: griffe - uses: ./.github/actions/griffe-api-check + - name: Install Moon 2.5.1 + shell: bash --noprofile --norc -euo pipefail {0} + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: - package-name: cuda.core - package-dir: cuda_core - merge-base: ${{ steps.latest-tag.outputs.tag }} + enable-cache: false + + - name: Check cuda_core public API + env: + CUDA_CORE_API_REF: ${{ steps.latest-tag.outputs.tag }} + run: moon run core:api-check --upstream none --downstream none api-check-core-vs-base: name: API check (cuda_core vs. merge base) if: >- ${{ startsWith(github.ref_name, 'pull-request/') && !fromJSON(needs.should-skip.outputs.skip) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks }} + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-api'][0] }} runs-on: ubuntu-latest needs: - should-skip @@ -319,13 +403,21 @@ jobs: git fetch --depth=1 --filter=blob:none origin \ "${{ fromJSON(needs.detect-changes.outputs.workplan).merge_base }}" - - name: Check cuda_core public API - id: griffe - uses: ./.github/actions/griffe-api-check + - name: Install Moon 2.5.1 + shell: bash --noprofile --norc -euo pipefail {0} + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: - package-name: cuda.core - package-dir: cuda_core - merge-base: ${{ fromJSON(needs.detect-changes.outputs.workplan).merge_base }} + enable-cache: false + + - name: Check cuda_core public API + env: + CUDA_CORE_API_REF: ${{ fromJSON(needs.detect-changes.outputs.workplan).merge_base }} + run: moon run core:api-check --upstream none --downstream none # NOTE: Build jobs are intentionally split by platform rather than using a single # matrix. This lets each test job consume its platform-specific artifacts as @@ -345,7 +437,21 @@ jobs: host-platform: - linux-64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) }} + if: >- + ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + (fromJSON(needs.should-skip.outputs.doc-only) || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-consumers'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-multi-ctk'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-finalize'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-consumers'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-windows'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-docs'][0]) }} permissions: actions: read contents: read @@ -372,7 +478,12 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }} + (fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-consumers'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-multi-ctk'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-finalize'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux'][0]) }} permissions: actions: read contents: read @@ -399,7 +510,15 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows }} + (fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-consumers'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-multi-ctk'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-finalize'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-consumers'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-windows'][0]) }} permissions: actions: read contents: read @@ -427,7 +546,9 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests }} + (fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-consumers'][0]) }} permissions: actions: read contents: read @@ -436,7 +557,7 @@ jobs: with: host-platform: linux-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - workplan: ${{ needs.detect-changes.outputs.workplan }} + targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets) }} # See test-sdist-linux for why sdist test jobs are split by platform. test-sdist-windows: @@ -450,7 +571,9 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests }} + (fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-consumers'][0]) }} permissions: actions: read contents: read @@ -459,7 +582,7 @@ jobs: with: host-platform: win-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - workplan: ${{ needs.detect-changes.outputs.workplan }} + targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets) }} # NOTE: Test jobs are split by platform for the same reason as build jobs (see # build-linux-64). Keep these job definitions textually identical except for: @@ -474,8 +597,9 @@ jobs: - linux-64 name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }} + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux'][0] }} permissions: actions: read contents: read # This is required for actions/checkout @@ -491,7 +615,7 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - workplan: ${{ needs.detect-changes.outputs.workplan }} + targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux']) }} # See test-linux-64 for why test jobs are split by platform. test-linux-aarch64: @@ -502,8 +626,9 @@ jobs: - linux-aarch64 name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }} + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux'][0] }} permissions: actions: read contents: read # This is required for actions/checkout @@ -520,7 +645,7 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - workplan: ${{ needs.detect-changes.outputs.workplan }} + targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux']) }} # See test-linux-64 for why test jobs are split by platform. test-windows: @@ -531,8 +656,9 @@ jobs: - win-64 name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows }} + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-windows'][0] }} permissions: actions: read contents: read # This is required for actions/checkout @@ -549,11 +675,14 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - workplan: ${{ needs.detect-changes.outputs.workplan }} + targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-windows']) }} doc: name: Docs - if: ${{ github.repository_owner == 'nvidia' }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + (fromJSON(needs.should-skip.outputs.doc-only) || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-docs'][0]) }} # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: id-token: write @@ -561,11 +690,14 @@ jobs: pull-requests: write needs: - ci-vars + - should-skip + - detect-changes - build-linux-64 secrets: inherit uses: ./.github/workflows/build-docs.yml with: is-release: ${{ github.ref_type == 'tag' }} + targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets['ci-docs']) }} precommit-windows: name: Pre-commit on Windows @@ -623,6 +755,7 @@ jobs: - name: Exit env: NEEDS_JSON: ${{ toJSON(needs) }} + WORKPLAN: ${{ needs.detect-changes.outputs.workplan }} run: | # GitHub treats `result == 'skipped'` as success for required # status checks (see CCCL gate comment + cccl#605). The previous @@ -639,10 +772,22 @@ jobs: fi doc_only="${{ needs.should-skip.outputs.doc-only }}" - linux_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux || false }}" - windows_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows || false }}" - build_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests || false }}" - run_core_api_check="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks || false }}" + wheel_selected=$(jq -r '([ + .targets["ci-wheel-foundation"][], + .targets["ci-wheel-bindings"][], + .targets["ci-wheel-consumers"][], + .targets["ci-wheel-multi-ctk"][], + .targets["ci-wheel-finalize"][] + ] | length > 0)' <<< "$WORKPLAN") + sdist_selected=$(jq -r '([ + .targets["ci-sdist-foundation"][], + .targets["ci-sdist-bindings"][], + .targets["ci-sdist-consumers"][] + ] | length > 0)' <<< "$WORKPLAN") + linux_selected=$(jq -r '.targets["ci-test-linux"] | length > 0' <<< "$WORKPLAN") + windows_selected=$(jq -r '.targets["ci-test-windows"] | length > 0' <<< "$WORKPLAN") + docs_selected=$(jq -r '.targets["ci-docs"] | length > 0' <<< "$WORKPLAN") + run_core_api_check=$(jq -r '.targets["ci-api"] | length > 0' <<< "$WORKPLAN") is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { @@ -655,38 +800,62 @@ jobs: fi } - # Control jobs, the universal linux build, docs, and Windows - # pre-commit checks always run. + # Control jobs and Windows pre-commit always run. check_result "ci-vars" "success" check_result "should-skip" "success" check_result "detect-changes" "success" - check_result "build-linux-64" "success" - check_result "doc" "success" check_result "precommit-windows" "success" - # Optional platform builds and wheel tests share the platform plan. - linux_expected="skipped" - if [[ "$doc_only" != "true" && "$linux_selected" == "true" ]]; then - linux_expected="success" + # Build jobs copy forward the complete trusted artifact set whenever + # downstream work needs it, even if no package wheel is rebuilt. + expected="skipped" + if [[ "$doc_only" == "true" || "$wheel_selected" == "true" || + "$sdist_selected" == "true" || "$linux_selected" == "true" || + "$windows_selected" == "true" || "$docs_selected" == "true" ]]; then + expected="success" fi - windows_expected="skipped" - if [[ "$doc_only" != "true" && "$windows_selected" == "true" ]]; then - windows_expected="success" + check_result "build-linux-64" "$expected" + + expected="skipped" + if [[ "$doc_only" != "true" && + ( "$wheel_selected" == "true" || "$linux_selected" == "true" ) ]]; then + expected="success" fi - check_result "build-linux-aarch64" "$linux_expected" - check_result "build-windows" "$windows_expected" + check_result "build-linux-aarch64" "$expected" - # Sdist tests follow build selection; wheel tests follow the platform plan. expected="skipped" - if [[ "$doc_only" != "true" && "$build_selected" == "true" ]]; then + if [[ "$doc_only" != "true" && + ( "$wheel_selected" == "true" || "$sdist_selected" == "true" || + "$windows_selected" == "true" ) ]]; then + expected="success" + fi + check_result "build-windows" "$expected" + + expected="skipped" + if [[ "$doc_only" != "true" && "$sdist_selected" == "true" ]]; then expected="success" fi check_result "test-sdist-linux" "$expected" check_result "test-sdist-windows" "$expected" - check_result "test-linux-64" "$linux_expected" - check_result "test-linux-aarch64" "$linux_expected" - check_result "test-windows" "$windows_expected" + expected="skipped" + if [[ "$doc_only" != "true" && "$linux_selected" == "true" ]]; then + expected="success" + fi + check_result "test-linux-64" "$expected" + check_result "test-linux-aarch64" "$expected" + + expected="skipped" + if [[ "$doc_only" != "true" && "$windows_selected" == "true" ]]; then + expected="success" + fi + check_result "test-windows" "$expected" + + expected="skipped" + if [[ "$doc_only" == "true" || "$docs_selected" == "true" ]]; then + expected="success" + fi + check_result "doc" "$expected" # API compatibility checks run for cuda_core source changes and for # conservative full runs when reusable base artifacts are unavailable. diff --git a/.github/workflows/test-sdist-linux.yml b/.github/workflows/test-sdist-linux.yml index ba7cdfc6ef1..00722002da8 100644 --- a/.github/workflows/test-sdist-linux.yml +++ b/.github/workflows/test-sdist-linux.yml @@ -11,8 +11,8 @@ on: cuda-version: required: true type: string - workplan: - description: JSON workplan. An empty value builds everything. + targets: + description: JSON object keyed by semantic Moon tags. An empty value builds everything. required: false default: "" type: string @@ -28,12 +28,8 @@ permissions: jobs: test-sdist: name: Test sdist builds - if: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).jobs.sdist_tests }} env: - BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} - BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} - BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} - BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} + SEMANTIC_TARGETS: ${{ inputs.targets != '' && inputs.targets || '{"ci-sdist-foundation":["pathfinder:sdist"],"ci-sdist-bindings":["bindings:sdist"],"ci-sdist-consumers":["core:sdist","metapackage:sdist"]}' }} timeout-minutes: 60 runs-on: linux-amd64-cpu8 steps: @@ -45,26 +41,59 @@ jobs: fetch-depth: 0 filter: blob:none + - name: Install target resolver dependencies + uses: ./.github/actions/install_unix_deps + with: + dependencies: "jq" + dependent_exes: "jq" + + - name: Resolve Moon sdist targets + run: | + jq -e ' + . as $root | + type == "object" and + all(["ci-sdist-foundation", "ci-sdist-bindings", "ci-sdist-consumers"][]; + . as $tag | + (($root[$tag] // []) | type == "array" and all(.[]; type == "string"))) + ' <<< "$SEMANTIC_TARGETS" >/dev/null + foundation=$(jq -c '.["ci-sdist-foundation"] // []' <<< "$SEMANTIC_TARGETS") + bindings=$(jq -c '.["ci-sdist-bindings"] // []' <<< "$SEMANTIC_TARGETS") + consumers=$(jq -c '.["ci-sdist-consumers"] // []' <<< "$SEMANTIC_TARGETS") + all_targets=$(jq -cn \ + --argjson foundation "$foundation" \ + --argjson bindings "$bindings" \ + --argjson consumers "$consumers" \ + '$foundation + $bindings + $consumers') + has_target() { + jq -e --arg target "$1" 'index($target) != null' <<< "$all_targets" >/dev/null + } + { + echo "SDIST_FOUNDATION_TARGETS=$foundation" + echo "SDIST_BINDINGS_TARGETS=$bindings" + echo "SDIST_CONSUMER_TARGETS=$consumers" + echo "BUILD_PATHFINDER=$(has_target pathfinder:sdist && echo true || echo false)" + echo "BUILD_BINDINGS=$(has_target bindings:sdist && echo true || echo false)" + echo "BUILD_CORE=$(has_target core:sdist && echo true || echo false)" + echo "BUILD_PYTHON=$(has_target metapackage:sdist && echo true || echo false)" + } >> "$GITHUB_ENV" + - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" - - name: Install build tools - run: python -m pip install "pip>=25.3" build + - name: Install Moon and build tools + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + python -m pip install "pip>=25.3" build # Pure Python packages -- no CTK needed. - - name: Build cuda.pathfinder sdist and wheel-from-sdist + - name: Build foundation sdists and wheels-from-sdists if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | - python -m build --sdist cuda_pathfinder/ - pip wheel --no-deps --wheel-dir cuda_pathfinder/dist cuda_pathfinder/dist/*.tar.gz - - - name: Build cuda-python sdist and wheel-from-sdist - if: ${{ env.BUILD_PYTHON == 'true' }} - run: | - python -m build --sdist cuda_python/ - pip wheel --no-deps --wheel-dir cuda_python/dist cuda_python/dist/*.tar.gz + mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_FOUNDATION_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Download cuda.pathfinder wheel if: ${{ env.BUILD_PATHFINDER != 'true' && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} @@ -118,7 +147,7 @@ jobs: # cuda_bindings/setup.py parses CUDA headers at import time, so CUDA_PATH # (set by fetch_ctk) must be available for both sdist and wheel builds. - - name: Build cuda.bindings sdist and wheel-from-sdist + - name: Build bindings sdists and wheels-from-sdists if: ${{ env.BUILD_BINDINGS == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) @@ -126,8 +155,8 @@ jobs: export CXX="sccache c++" export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-bindings.txt" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - python -m build --sdist cuda_bindings/ - pip wheel --no-deps --wheel-dir cuda_bindings/dist cuda_bindings/dist/*.tar.gz + mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_BINDINGS_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Download cuda.bindings wheel if: ${{ env.BUILD_BINDINGS != 'true' && env.BUILD_CORE == 'true' }} @@ -157,17 +186,19 @@ jobs: # cuda_core sdist delegates to setuptools (no CTK needed), but # wheel-from-sdist needs CTK and cuda-bindings (dynamic build dep via # get_requires_for_build_wheel in build_hooks.py). - - name: Build cuda.core sdist and wheel-from-sdist - if: ${{ env.BUILD_CORE == 'true' }} + - name: Build consumer sdists and wheels-from-sdists + if: ${{ env.BUILD_CORE == 'true' || env.BUILD_PYTHON == 'true' }} run: | - export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) - export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" - export CC="sccache cc" - export CXX="sccache c++" - export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-core.txt" - export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - python -m build --sdist cuda_core/ - pip wheel --no-deps --wheel-dir cuda_core/dist cuda_core/dist/*.tar.gz + if [[ "$BUILD_CORE" == "true" ]]; then + export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) + export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" + export CC="sccache cc" + export CXX="sccache c++" + export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-core.txt" + export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" + fi + mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_CONSUMER_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Show sccache stats if: ${{ always() && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} diff --git a/.github/workflows/test-sdist-windows.yml b/.github/workflows/test-sdist-windows.yml index a0594800eba..623ce85fec1 100644 --- a/.github/workflows/test-sdist-windows.yml +++ b/.github/workflows/test-sdist-windows.yml @@ -17,8 +17,8 @@ on: cuda-version: required: true type: string - workplan: - description: JSON workplan. An empty value builds everything. + targets: + description: JSON object keyed by semantic Moon tags. An empty value builds everything. required: false default: "" type: string @@ -34,12 +34,8 @@ permissions: jobs: test-sdist: name: Test sdist builds - if: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).jobs.sdist_tests }} env: - BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} - BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} - BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} - BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} + SEMANTIC_TARGETS: ${{ inputs.targets != '' && inputs.targets || '{"ci-sdist-foundation":["pathfinder:sdist"],"ci-sdist-bindings":["bindings:sdist"],"ci-sdist-consumers":["core:sdist","metapackage:sdist"]}' }} timeout-minutes: 60 runs-on: windows-2022 steps: @@ -51,6 +47,37 @@ jobs: fetch-depth: 0 filter: blob:none + - name: Resolve Moon sdist targets + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + jq -e ' + . as $root | + type == "object" and + all(["ci-sdist-foundation", "ci-sdist-bindings", "ci-sdist-consumers"][]; + . as $tag | + (($root[$tag] // []) | type == "array" and all(.[]; type == "string"))) + ' <<< "$SEMANTIC_TARGETS" >/dev/null + foundation=$(jq -c '.["ci-sdist-foundation"] // []' <<< "$SEMANTIC_TARGETS") + bindings=$(jq -c '.["ci-sdist-bindings"] // []' <<< "$SEMANTIC_TARGETS") + consumers=$(jq -c '.["ci-sdist-consumers"] // []' <<< "$SEMANTIC_TARGETS") + all_targets=$(jq -cn \ + --argjson foundation "$foundation" \ + --argjson bindings "$bindings" \ + --argjson consumers "$consumers" \ + '$foundation + $bindings + $consumers') + has_target() { + jq -e --arg target "$1" 'index($target) != null' <<< "$all_targets" >/dev/null + } + { + echo "SDIST_FOUNDATION_TARGETS=$foundation" + echo "SDIST_BINDINGS_TARGETS=$bindings" + echo "SDIST_CONSUMER_TARGETS=$consumers" + echo "BUILD_PATHFINDER=$(has_target pathfinder:sdist && echo true || echo false)" + echo "BUILD_BINDINGS=$(has_target bindings:sdist && echo true || echo false)" + echo "BUILD_CORE=$(has_target core:sdist && echo true || echo false)" + echo "BUILD_PYTHON=$(has_target metapackage:sdist && echo true || echo false)" + } >> "$GITHUB_ENV" + - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: @@ -60,21 +87,18 @@ jobs: if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 - - name: Install build tools - run: python -m pip install "pip>=25.3" build + - name: Install Moon and build tools + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + python -m pip install "pip>=25.3" build # Pure Python packages -- no CTK needed. - - name: Build cuda.pathfinder sdist and wheel-from-sdist + - name: Build foundation sdists and wheels-from-sdists if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | - python -m build --sdist cuda_pathfinder/ - pip wheel --no-deps --wheel-dir cuda_pathfinder/dist cuda_pathfinder/dist/*.tar.gz - - - name: Build cuda-python sdist and wheel-from-sdist - if: ${{ env.BUILD_PYTHON == 'true' }} - run: | - python -m build --sdist cuda_python/ - pip wheel --no-deps --wheel-dir cuda_python/dist cuda_python/dist/*.tar.gz + mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_FOUNDATION_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Download cuda.pathfinder wheel if: ${{ env.BUILD_PATHFINDER != 'true' && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} @@ -108,14 +132,14 @@ jobs: # (set by fetch_ctk) must be available for both sdist and wheel builds. # Constraint paths are passed as native Windows paths because the pip # subprocesses run outside Git Bash. - - name: Build cuda.bindings sdist and wheel-from-sdist + - name: Build bindings sdists and wheels-from-sdists if: ${{ env.BUILD_BINDINGS == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-bindings.txt")" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - python -m build --sdist cuda_bindings/ - pip wheel --no-deps --wheel-dir cuda_bindings/dist cuda_bindings/dist/*.tar.gz + mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_BINDINGS_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Download cuda.bindings wheel if: ${{ env.BUILD_BINDINGS != 'true' && env.BUILD_CORE == 'true' }} @@ -145,12 +169,14 @@ jobs: # cuda_core sdist delegates to setuptools (no CTK needed), but # wheel-from-sdist needs CTK and cuda-bindings (dynamic build dep via # get_requires_for_build_wheel in build_hooks.py). - - name: Build cuda.core sdist and wheel-from-sdist - if: ${{ env.BUILD_CORE == 'true' }} + - name: Build consumer sdists and wheels-from-sdists + if: ${{ env.BUILD_CORE == 'true' || env.BUILD_PYTHON == 'true' }} run: | - export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) - export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" - export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-core.txt")" - export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - python -m build --sdist cuda_core/ - pip wheel --no-deps --wheel-dir cuda_core/dist cuda_core/dist/*.tar.gz + if [[ "$BUILD_CORE" == "true" ]]; then + export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) + export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" + export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-core.txt")" + export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" + fi + mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_CONSUMER_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 814e4e756b0..dd99ae55384 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -22,8 +22,8 @@ on: nruns: type: number default: 1 - workplan: - description: JSON workplan. An empty value tests everything. + targets: + description: JSON array of exact Moon Linux test route targets. An empty value tests everything. type: string default: "" run-id: @@ -99,10 +99,7 @@ jobs: test: env: - TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }} - TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} - TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} - TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }} + MOON_TARGETS: ${{ inputs.targets != '' && inputs.targets || '["pathfinder:ci-test-linux","bindings:ci-test-linux","core:ci-test-linux","metapackage:ci-test-linux"]' }} name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }}${{ matrix.FLAVOR && format(', {0}', matrix.FLAVOR) || '' }}${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }} timeout-minutes: 60 needs: compute-matrix @@ -143,6 +140,19 @@ jobs: dependencies: "jq wget libgl1 libegl1 g++ util-linux" dependent_exes: "jq wget" + - name: Resolve Moon test targets + run: | + jq -e 'type == "array" and all(.[]; type == "string")' <<< "$MOON_TARGETS" >/dev/null + has_target() { + jq -e --arg target "$1" 'index($target) != null' <<< "$MOON_TARGETS" >/dev/null + } + { + echo "TEST_PATHFINDER=$(has_target pathfinder:ci-test-linux && echo true || echo false)" + echo "TEST_BINDINGS=$(has_target bindings:ci-test-linux && echo true || echo false)" + echo "TEST_CORE=$(has_target core:ci-test-linux && echo true || echo false)" + echo "TEST_PYTHON=$(has_target metapackage:ci-test-linux && echo true || echo false)" + } >> "$GITHUB_ENV" + - name: Install GPU driver if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} env: @@ -247,11 +257,24 @@ jobs: rmdir cuda-python-wheel fi + - name: Stage pure wheels for Moon + if: ${{ inputs.test-mode == 'standard' }} + run: | + mkdir -p cuda_pathfinder/dist cuda_python/dist + pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) + if (( ${#pathfinder_wheels[@]} != 0 )) && [[ -f "${pathfinder_wheels[0]}" ]]; then + cp "${pathfinder_wheels[@]}" cuda_pathfinder/dist/ + fi + python_wheels=(cuda_python-*.whl) + if (( ${#python_wheels[@]} != 0 )) && [[ -f "${python_wheels[0]}" ]]; then + cp "${python_wheels[@]}" cuda_python/dist/ + fi + - name: Display structure of downloaded cuda-python artifacts if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'published' }} run: | pwd - ls -lah cuda_python*.whl cuda_pathfinder/ + find cuda_pathfinder cuda_python -maxdepth 2 -type f -name '*.whl' -print - name: Display structure of downloaded cuda.bindings artifacts if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && @@ -276,7 +299,7 @@ jobs: ls -lahR $CUDA_BINDINGS_CYTHON_TESTS_DIR - name: Download cuda.core build artifacts - if: ${{ env.TEST_CORE == 'true' }} + if: ${{ env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} @@ -285,7 +308,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core build artifacts - if: ${{ env.TEST_CORE == 'true' }} + if: ${{ env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} run: | pwd ls -lahR $CUDA_CORE_ARTIFACTS_DIR @@ -330,6 +353,12 @@ jobs: # we use self-hosted runners on which setup-python behaves weirdly (Python include can't be found)... AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache" + - name: Install Moon + if: ${{ inputs.test-mode == 'standard' }} + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + - name: Enable Scientific Python Nightly Wheels for Python 3.15 if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && startsWith(matrix.PY_VER, '3.15') }} @@ -361,74 +390,24 @@ jobs: - name: Set up test repetition on nightly runs run: echo "PYTEST_ADDOPTS=\"--count=${{ inputs.nruns }}\"" >> "$GITHUB_ENV" - # ── Standard test steps (skipped for nightly modes) ── - - name: Run cuda.pathfinder tests with see_what_works - if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} - env: - CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works - CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works - CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: see_what_works - run: run-tests pathfinder - - - name: Run cuda.bindings tests - if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} + # ── Standard test route (skipped for nightly modes) ── + - name: Run selected installed-wheel tests + if: ${{ inputs.test-mode == 'standard' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} - run: run-tests bindings - - - name: Run cuda.bindings benchmarks (smoke test) - if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} run: | - pip install pyperf - pushd benchmarks/cuda_bindings - python run_pyperf.py --debug-single-value - popd - - - name: Run cuda.core tests - if: ${{ inputs.test-mode == 'standard' && env.TEST_CORE == 'true' }} - env: - CUDA_VER: ${{ matrix.CUDA_VER }} - LOCAL_CTK: ${{ matrix.LOCAL_CTK }} - run: run-tests core - - - name: Ensure cuda-python installable - if: ${{ inputs.test-mode == 'standard' && env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} - run: | - # Package suites install their own dependencies. A metapackage-only - # run has no preceding suite, so install the exact local internal - # wheels in one transaction while resolving released dependencies - # such as cuda-core from the package index. - if ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }}; then - dependency_args=(--no-deps) - else - dependency_args=( - ./cuda_pathfinder/cuda_pathfinder-*.whl - "${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-*.whl - ) + targets="$MOON_TARGETS" + if [[ "$SKIP_CUDA_BINDINGS_TEST" != "0" ]]; then + targets=$(jq -c 'map(select(. != "bindings:ci-test-linux"))' <<< "$targets") fi - python_requirements=(cuda_python*.whl) - if [[ "${{ matrix.LOCAL_CTK }}" != 1 ]]; then - python_requirements=("${python_requirements[@]/%/[all]}") + if [[ "$BINDINGS_SOURCE" != "main" ]]; then + targets=$(jq -c 'map(select(. != "metapackage:ci-test-linux"))' <<< "$targets") + fi + mapfile -t target_args < <(jq -r '.[]' <<< "$targets") + if (( ${#target_args[@]} != 0 )); then + moon run "${target_args[@]}" --upstream direct --downstream none fi - pip install --only-binary=:all: "${dependency_args[@]}" "${python_requirements[@]}" - - - name: Install cuda.pathfinder extra wheels for testing - if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} - run: | - set -euo pipefail - pushd cuda_pathfinder - pip install --only-binary=:all: -v ./*.whl --group "test-cu${TEST_CUDA_MAJOR}" - pip list - popd - - - name: Run cuda.pathfinder tests with all_must_work - if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} - env: - CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work - CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work - CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: all_must_work - run: run-tests pathfinder # ── Nightly: install wheels + optional dep together ── - name: Install cuda-python wheels + PyTorch diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 3da9c180dd2..1ee5600042d 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -22,8 +22,8 @@ on: nruns: type: number default: 1 - workplan: - description: JSON workplan. An empty value tests everything. + targets: + description: JSON array of exact Moon Windows test route targets. An empty value tests everything. type: string default: "" run-id: @@ -89,10 +89,7 @@ jobs: test: env: - TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }} - TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} - TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} - TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }} + MOON_TARGETS: ${{ inputs.targets != '' && inputs.targets || '["pathfinder:ci-test-windows","bindings:ci-test-windows","core:ci-test-windows","metapackage:ci-test-windows"]' }} name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }} (${{ matrix.DRIVER_MODE }})${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }} timeout-minutes: 60 # The build stage could fail but we want the CI to keep moving. @@ -108,6 +105,20 @@ jobs: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Resolve Moon test targets + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + jq -e 'type == "array" and all(.[]; type == "string")' <<< "$MOON_TARGETS" >/dev/null + has_target() { + jq -e --arg target "$1" 'index($target) != null' <<< "$MOON_TARGETS" >/dev/null + } + { + echo "TEST_PATHFINDER=$(has_target pathfinder:ci-test-windows && echo true || echo false)" + echo "TEST_BINDINGS=$(has_target bindings:ci-test-windows && echo true || echo false)" + echo "TEST_CORE=$(has_target core:ci-test-windows && echo true || echo false)" + echo "TEST_PYTHON=$(has_target metapackage:ci-test-windows && echo true || echo false)" + } >> "$GITHUB_ENV" + - name: Setup proxy cache uses: nv-gha-runners/setup-proxy-cache@main continue-on-error: true @@ -227,11 +238,25 @@ jobs: rmdir cuda-python-wheel fi + - name: Stage pure wheels for Moon + if: ${{ inputs.test-mode == 'standard' }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + mkdir -p cuda_pathfinder/dist cuda_python/dist + pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) + if (( ${#pathfinder_wheels[@]} != 0 )) && [[ -f "${pathfinder_wheels[0]}" ]]; then + cp "${pathfinder_wheels[@]}" cuda_pathfinder/dist/ + fi + python_wheels=(cuda_python-*.whl) + if (( ${#python_wheels[@]} != 0 )) && [[ -f "${python_wheels[0]}" ]]; then + cp "${python_wheels[@]}" cuda_python/dist/ + fi + - name: Display structure of downloaded cuda-python artifacts if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'published' }} run: | Get-Location - Get-ChildItem cuda_python*.whl | Select-Object Mode, LastWriteTime, Length, FullName + Get-ChildItem -Recurse cuda_pathfinder,cuda_python -Filter *.whl | Select-Object Mode, LastWriteTime, Length, FullName - name: Display structure of downloaded cuda.bindings artifacts if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && @@ -256,7 +281,7 @@ jobs: Get-ChildItem -Recurse -Force $env:CUDA_BINDINGS_CYTHON_TESTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.core build artifacts - if: ${{ env.TEST_CORE == 'true' }} + if: ${{ env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} @@ -265,7 +290,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core build artifacts - if: ${{ env.TEST_CORE == 'true' }} + if: ${{ env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_CORE_ARTIFACTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName @@ -307,6 +332,13 @@ jobs: # TODO: remove allow-prereleases once 3.15 is officially supported allow-prereleases: ${{ startsWith(matrix.PY_VER, '3.15') }} + - name: Install Moon + if: ${{ inputs.test-mode == 'standard' }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + - name: Enable Scientific Python Nightly Wheels for Python 3.15 if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && startsWith(matrix.PY_VER, '3.15') }} @@ -336,70 +368,25 @@ jobs: shell: bash --noprofile --norc -xeuo pipefail {0} run: echo "PYTEST_ADDOPTS=\"--count=${{ inputs.nruns }}\"" >> "$GITHUB_ENV" - # ── Standard test steps (skipped for nightly modes) ── - - name: Run cuda.pathfinder tests with see_what_works - if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} - env: - CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works - CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works - CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: see_what_works - shell: bash --noprofile --norc -xeuo pipefail {0} - run: run-tests pathfinder - - - name: Run cuda.bindings tests - if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} - env: - CUDA_VER: ${{ matrix.CUDA_VER }} - LOCAL_CTK: ${{ matrix.LOCAL_CTK }} - shell: bash --noprofile --norc -xeuo pipefail {0} - run: run-tests bindings - - - name: Run cuda.core tests - if: ${{ inputs.test-mode == 'standard' && env.TEST_CORE == 'true' }} + # ── Standard test route (skipped for nightly modes) ── + - name: Run selected installed-wheel tests + if: ${{ inputs.test-mode == 'standard' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} shell: bash --noprofile --norc -xeuo pipefail {0} - run: run-tests core - - - name: Ensure cuda-python installable - if: ${{ inputs.test-mode == 'standard' && env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} - run: | - # Package suites install their own dependencies. A metapackage-only - # run has no preceding suite, so install the exact local internal - # wheels in one transaction while resolving released dependencies - # such as cuda-core from the package index. - if ('${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }}' -eq 'true') { - $dependencyArgs = @('--no-deps') - } else { - $dependencyArgs = @( - (Get-Item ./cuda_pathfinder/cuda_pathfinder-*.whl).FullName - (Get-Item "$env:CUDA_BINDINGS_ARTIFACTS_DIR/cuda_bindings-*.whl").FullName - ) - } - $pythonRequirements = @((Get-Item ./cuda_python*.whl).FullName) - if ('${{ matrix.LOCAL_CTK }}' -ne '1') { - $pythonRequirements = @($pythonRequirements | ForEach-Object { "$($_)[all]" }) - } - pip install --only-binary=:all: @dependencyArgs @pythonRequirements - - - name: Install cuda.pathfinder extra wheels for testing - if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} - shell: bash --noprofile --norc -xeuo pipefail {0} run: | - pushd cuda_pathfinder - pip install --only-binary=:all: -v ./*.whl --group "test-cu${TEST_CUDA_MAJOR}" - pip list - popd - - - name: Run cuda.pathfinder tests with all_must_work - if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} - env: - CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work - CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work - CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: all_must_work - shell: bash --noprofile --norc -xeuo pipefail {0} - run: run-tests pathfinder + targets="$MOON_TARGETS" + if [[ "$SKIP_CUDA_BINDINGS_TEST" != "0" ]]; then + targets=$(jq -c 'map(select(. != "bindings:ci-test-windows"))' <<< "$targets") + fi + if [[ "$BINDINGS_SOURCE" != "main" ]]; then + targets=$(jq -c 'map(select(. != "metapackage:ci-test-windows"))' <<< "$targets") + fi + mapfile -t target_args < <(jq -r '.[]' <<< "$targets") + if (( ${#target_args[@]} != 0 )); then + moon run "${target_args[@]}" --upstream direct --downstream none + fi # ── Nightly: install wheels + optional dep together ── - name: Install Visual C++ Redistributable (required by PyTorch on Windows) diff --git a/.gitignore b/.gitignore index 6b6a7dfc0b5..4824d472ec6 100644 --- a/.gitignore +++ b/.gitignore @@ -182,3 +182,4 @@ cython_debug/ # Cursor .cursorrules .claude/settings.local.json +.moon/cache/ diff --git a/.moon/workspace.yml b/.moon/workspace.yml new file mode 100644 index 00000000000..02cc41d1447 --- /dev/null +++ b/.moon/workspace.yml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +projects: + root: '.' + pathfinder: 'cuda_pathfinder' + bindings: 'cuda_bindings' + core: 'cuda_core' + metapackage: 'cuda_python' + +vcs: + defaultBranch: 'main' + +versionConstraint: '=2.5.1' diff --git a/ci/test-matrix.yml b/ci/test-matrix.yml index 66f3196ab68..563774494e9 100644 --- a/ci/test-matrix.yml +++ b/ci/test-matrix.yml @@ -32,12 +32,6 @@ # ENV: { CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM: '1' } # ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.12.1', TORCH_CUDA: 'cu126' } -# Host platforms that produce wheel artifacts in the main CI workflow. -platforms: - - linux-64 - - linux-aarch64 - - win-64 - linux: pull-request: # linux-64 diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py deleted file mode 100644 index 9f94c67173d..00000000000 --- a/ci/tools/compute_ci_plan.py +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Compute the CI build and test workplan for a pull request.""" - -from __future__ import annotations - -import argparse -import json -import subprocess -from pathlib import Path, PurePosixPath - -REPO_ROOT = Path(__file__).resolve().parents[2] -MODULES = ("pathfinder", "bindings", "core", "python") -PLATFORMS = ("linux", "windows") -PACKAGE_MODULES = {f"cuda_{module}": module for module in MODULES} - -# Source changes have different build and test consumers. In particular, -# cuda-python source needs a same-version bindings wheel, while a core-only -# change can reuse the baseline cuda-python wheel. -SOURCE_IMPACT = { - "pathfinder": (set(MODULES), set(MODULES)), - "bindings": ({"bindings", "core", "python"}, {"bindings", "core", "python"}), - "core": ({"core"}, {"core", "python"}), - "python": ({"bindings", "python"}, {"python"}), -} - -IGNORED_BASENAMES = {"AGENTS.md", "CLAUDE.md", "pixi.lock", "pixi.toml"} -IGNORED_SUFFIXES = {".md", ".svg"} -IGNORED_PATHS = { - ".coveragerc", - ".gitignore", - ".pre-commit-config.yaml", - ".spdx-ignore", - "LICENSE", - "context7.json", - "greptile.json", - "ruff.toml", -} -IGNORED_PREFIXES = (".agents/", "toolshed/") - -# Only infrastructure exclusive to one OS belongs here; other CI paths force a full run. -TEST_INFRA_PLATFORMS = { - ".github/workflows/test-wheel-linux.yml": "linux", - ".github/workflows/test-wheel-windows.yml": "windows", - "ci/tools/configure_driver_mode.ps1": "windows", - "ci/tools/guess_latest.sh": "linux", - "ci/tools/install_gpu_driver.ps1": "windows", - "ci/tools/install_gpu_driver.sh": "linux", - "ci/tools/setup-sanitizer": "linux", -} - - -def compute_workplan( - paths: list[str], - *, - merge_base: str, - baseline_run_id: str, - linked_paths: set[str] | None = None, -) -> dict[str, object]: - """Return the final CI decisions for the supplied changed paths.""" - linked_paths = linked_paths or set() - source_changes: set[str] = set() - test_changes: set[str] = set() - test_platforms: set[str] = set() - force_all = not merge_base or not baseline_run_id - - if not force_all: - for path in paths: - path_parts = PurePosixPath(path).parts - if not path_parts: - continue - - if platform := TEST_INFRA_PLATFORMS.get(path): - test_platforms.add(platform) - continue - - if path_parts[0] == "ci" or ( - len(path_parts) >= 2 and path_parts[:2] in {(".github", "actions"), (".github", "workflows")} - ): - force_all = True - break - - if path_parts[0] == ".github" or path_parts[-1] in IGNORED_BASENAMES: - continue - - module = PACKAGE_MODULES.get(path_parts[0]) - if module is not None and len(path_parts) > 1: - relative = path_parts[1:] - if relative[0] == "docs": - continue - if ( - any(part in {"test", "tests"} for part in relative[:-1]) - or relative[0] == "examples" - or (module == "core" and relative == ("pytest.ini",)) - ): - test_changes.add(module) - elif PurePosixPath(path).suffix in IGNORED_SUFFIXES and path not in linked_paths: - continue - else: - source_changes.add(module) - continue - - is_test_path = any(part in {"test", "tests"} for part in path_parts[:-1]) - if is_test_path: - test_changes.update(MODULES) - elif ( - path in IGNORED_PATHS - or PurePosixPath(path).suffix in IGNORED_SUFFIXES - or path.startswith(IGNORED_PREFIXES) - ): - continue - elif path_parts[0] in {"benchmarks", "cuda_python_test_helpers"}: - test_changes.update(MODULES) - else: - force_all = True - break - - if force_all: - builds = set(MODULES) - tests = set(MODULES) - test_platforms = set(PLATFORMS) - else: - builds: set[str] = set() - tests = set(MODULES) if test_platforms else set(test_changes) - for module in source_changes: - build_impact, test_impact = SOURCE_IMPACT[module] - builds.update(build_impact) - tests.update(test_impact) - if source_changes or test_changes: - test_platforms.update(PLATFORMS) - - modules = { - module: { - "needs_build": module in builds, - "needs_test": module in tests, - } - for module in MODULES - } - return { - "modules": modules, - "jobs": { - # These gates cover both optional artifact builds and wheel tests. - "platforms": {platform: platform in test_platforms for platform in PLATFORMS}, - "sdist_tests": bool(builds), - "core_api_checks": force_all or "core" in source_changes, - }, - "merge_base": merge_base, - "baseline": { - "run_id": baseline_run_id if not force_all else "", - "sha": merge_base if not force_all else "", - }, - } - - -def _git_output(*args: str) -> bytes: - return subprocess.check_output( # noqa: S603 - argv is passed directly without a shell. - ["git", *args], # noqa: S607 - cwd=REPO_ROOT, - ) - - -def _changed_paths(merge_base: str) -> tuple[list[str], set[str]]: - output = _git_output("diff", "--no-renames", "--name-only", "-z", merge_base, "HEAD") - paths = [path.decode("utf-8", errors="surrogateescape") for path in output.split(b"\0") if path] - head_symlinks = _tracked_symlink_paths("HEAD") - # Base links preserve the packaging impact of deleted or replaced symlinks. - linked_paths = set(head_symlinks) | set(_tracked_symlink_paths(merge_base)) - return _expand_linked_paths(paths, head_symlinks, root=REPO_ROOT), linked_paths - - -def _tracked_symlink_paths(ref: str) -> list[str]: - output = _git_output("ls-tree", "--full-tree", "-r", "-z", ref) - return [ - entry.partition(b"\t")[2].decode("utf-8", errors="surrogateescape") - for entry in output.split(b"\0") - if entry.startswith(b"120000 ") - ] - - -def _expand_linked_paths(paths: list[str], symlink_paths: list[str], *, root: Path) -> list[str]: - """Include tracked symlinks whose resolved targets changed.""" - resolved_paths = {(root / path).resolve(strict=False) for path in paths} - expanded = list(paths) - selected = set(paths) - expanded.extend( - path for path in symlink_paths if path not in selected and (root / path).resolve(strict=False) in resolved_paths - ) - return expanded - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--merge-base", default="") - parser.add_argument("--baseline-run-id", default="") - args = parser.parse_args() - - reusable_baseline = bool(args.merge_base and args.baseline_run_id) - paths, linked_paths = _changed_paths(args.merge_base) if reusable_baseline else ([], set()) - plan = compute_workplan( - paths, - merge_base=args.merge_base, - baseline_run_id=args.baseline_run_id, - linked_paths=linked_paths, - ) - print(json.dumps(plan, separators=(",", ":"), sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/ci/tools/tests/test_compute_ci_plan.py b/ci/tools/tests/test_compute_ci_plan.py deleted file mode 100644 index 79a83394dfa..00000000000 --- a/ci/tools/tests/test_compute_ci_plan.py +++ /dev/null @@ -1,181 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import tempfile -import unittest -from pathlib import Path - -from ci.tools.compute_ci_plan import _expand_linked_paths, compute_workplan - -ALL_MODULES = {"pathfinder", "bindings", "core", "python"} -ALL_PLATFORMS = {"linux", "windows"} - - -def plan_for( - *paths: str, - baseline: bool = True, - linked_paths: set[str] | None = None, -) -> dict[str, object]: - return compute_workplan( - list(paths), - merge_base="base", - baseline_run_id="123" if baseline else "", - linked_paths=linked_paths, - ) - - -def selected(plan: dict[str, object], key: str) -> set[str]: - modules = plan["modules"] - assert isinstance(modules, dict) - return {name for name, decision in modules.items() if decision[key]} - - -def selected_platforms(plan: dict[str, object]) -> set[str]: - jobs = plan["jobs"] - assert isinstance(jobs, dict) - platforms = jobs["platforms"] - assert isinstance(platforms, dict) - assert set(platforms) == ALL_PLATFORMS - return {name for name, enabled in platforms.items() if enabled} - - -class ComputeWorkplanTest(unittest.TestCase): - def test_path_impacts(self) -> None: - cases = { - "cuda_pathfinder/cuda/pathfinder/_loader.py": (ALL_MODULES, ALL_MODULES, False), - "cuda_bindings/cuda/bindings/driver.pyx": ( - {"bindings", "core", "python"}, - {"bindings", "core", "python"}, - False, - ), - "cuda_core/cuda/core/_device.py": ({"core"}, {"core", "python"}, True), - "cuda_core/cuda/core/examples/demo.py": ({"core"}, {"core", "python"}, True), - "cuda_python/pyproject.toml": ({"bindings", "python"}, {"python"}, False), - "cuda_pathfinder/tests/test_loader.py": (set(), {"pathfinder"}, False), - "cuda_bindings/examples/0_Introduction/vectorAddDrv.py": (set(), {"bindings"}, False), - "cuda_bindings/tests/README.md": (set(), {"bindings"}, False), - "cuda_core/pytest.ini": (set(), {"core"}, False), - "cuda_python/tests/test_import.py": (set(), {"python"}, False), - "cuda_python_test_helpers/cuda_python_test_helpers/cuda_utils.py": ( - set(), - ALL_MODULES, - False, - ), - "benchmarks/cuda_bindings/run_pyperf.py": (set(), ALL_MODULES, False), - "benchmarks/cuda_core/runner.py": (set(), ALL_MODULES, False), - "ci/tools/run-tests": (ALL_MODULES, ALL_MODULES, True), - "ci/versions.yml": (ALL_MODULES, ALL_MODULES, True), - "pytest.ini": (ALL_MODULES, ALL_MODULES, True), - } - - for path, (builds, tests, core_api) in cases.items(): - with self.subTest(path=path): - plan = plan_for(path) - assert selected(plan, "needs_build") == builds - assert selected(plan, "needs_test") == tests - assert selected_platforms(plan) == ALL_PLATFORMS - assert plan["jobs"]["sdist_tests"] == bool(builds) - assert plan["jobs"]["core_api_checks"] == core_api - - def test_test_infrastructure_platforms(self) -> None: - cases = { - ".github/workflows/test-wheel-linux.yml": {"linux"}, - ".github/workflows/test-wheel-windows.yml": {"windows"}, - "ci/tools/configure_driver_mode.ps1": {"windows"}, - "ci/tools/guess_latest.sh": {"linux"}, - "ci/tools/install_gpu_driver.ps1": {"windows"}, - "ci/tools/install_gpu_driver.sh": {"linux"}, - "ci/tools/setup-sanitizer": {"linux"}, - } - - for path, platforms in cases.items(): - with self.subTest(path=path): - plan = plan_for(path) - assert not selected(plan, "needs_build") - assert selected(plan, "needs_test") == ALL_MODULES - assert selected_platforms(plan) == platforms - assert not plan["jobs"]["sdist_tests"] - assert not plan["jobs"]["core_api_checks"] - - mixed_plan = plan_for("ci/tools/install_gpu_driver.sh", "ci/tools/install_gpu_driver.ps1") - assert selected_platforms(mixed_plan) == ALL_PLATFORMS - - source_plan = plan_for("ci/tools/install_gpu_driver.sh", "cuda_python/pyproject.toml") - assert selected_platforms(source_plan) == ALL_PLATFORMS - - def test_ignored_paths_select_no_work(self) -> None: - for path in ( - "cuda_core/docs/index.rst", - "cuda_core/pixi.toml", - "cuda_core/tests/fixtures/pixi.toml", - "benchmarks/cuda_bindings/pixi.toml", - "benchmarks/cuda_bindings/AGENTS.md", - "cuda_core/cuda/core/_cpp/DESIGN.md", - "cuda_bindings/README.md", - "cuda_core/README.md", - "new-area/pixi.toml", - "notes.md", - "diagram.svg", - ".github/labeler.yml", - ".github/ISSUE_TEMPLATE/bug.yml", - ): - with self.subTest(path=path): - plan = plan_for(path) - assert not selected(plan, "needs_build") - assert not selected(plan, "needs_test") - assert not selected_platforms(plan) - - def test_unknown_path_and_missing_baseline_force_all(self) -> None: - for plan in ( - plan_for("new-top-level-file"), - plan_for("new-area/config.toml"), - plan_for(".github/workflows/new-main-ci-workflow.yml"), - plan_for(".github/actions/doc_preview/action.yml"), - plan_for("ci/ci-pipeline.svg"), - plan_for("cuda_core/docs/index.rst", baseline=False), - compute_workplan([], merge_base="", baseline_run_id="123"), - ): - assert selected(plan, "needs_build") == ALL_MODULES - assert selected(plan, "needs_test") == ALL_MODULES - assert selected_platforms(plan) == ALL_PLATFORMS - assert plan["jobs"]["core_api_checks"] - assert plan["baseline"] == {"run_id": "", "sha": ""} - - def test_mixed_changes_are_combined(self) -> None: - plan = plan_for("cuda_core/tests/test_device.py", "cuda_python/pyproject.toml") - assert selected(plan, "needs_build") == {"bindings", "python"} - assert selected(plan, "needs_test") == {"core", "python"} - assert selected_platforms(plan) == ALL_PLATFORMS - assert plan["jobs"]["sdist_tests"] - assert plan["baseline"] == {"run_id": "123", "sha": "base"} - - def test_changed_symlink_targets_include_their_consumers(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - (root / "cuda_python").mkdir() - (root / "cuda_core").mkdir() - (root / "README.md").write_text("readme", encoding="utf-8") - (root / "cuda_python" / "README.md").symlink_to("../README.md") - (root / "cuda_core" / "README.md").symlink_to("../README.md") - - paths = _expand_linked_paths( - ["README.md"], - ["cuda_python/README.md"], - root=root, - ) - - assert paths == ["README.md", "cuda_python/README.md"] - plan = plan_for(*paths, linked_paths={"cuda_python/README.md"}) - assert selected(plan, "needs_build") == {"bindings", "python"} - assert selected(plan, "needs_test") == {"python"} - - removed_link = plan_for("cuda_python/README.md", linked_paths={"cuda_python/README.md"}) - assert selected(removed_link, "needs_build") == {"bindings", "python"} - assert selected(removed_link, "needs_test") == {"python"} - - -if __name__ == "__main__": - unittest.main() diff --git a/cuda_bindings/moon.yml b/cuda_bindings/moon.yml new file mode 100644 index 00000000000..d38add85af1 --- /dev/null +++ b/cuda_bindings/moon.yml @@ -0,0 +1,215 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +$schema: https://moonrepo.dev/schemas/v2/project.json + +dependsOn: ['pathfinder'] + +fileGroups: + package: + - 'cuda/**/*' + - '!cuda/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!cuda/**/*.{md,svg}' + - 'build_hooks.py' + - 'DESCRIPTION.rst' + - 'LICENSE' + - 'MANIFEST.in' + - 'pyproject.toml' + - 'setup.py' + - '.git_archival.txt' + - '/.git_archival.txt' + - '/cuda_pathfinder/.git_archival.txt' + tests: + - 'tests/**/*' + - 'examples/**/*' + - '!{tests,examples}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + docs: + - 'docs/**/*' + - '!docs/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + benchmarks: + - '/benchmarks/cuda_bindings/benchmarks/**/*' + - '/benchmarks/cuda_bindings/runner/**/*' + - '/benchmarks/cuda_bindings/tests/**/*' + - '/benchmarks/cuda_bindings/compare.py' + - '/benchmarks/cuda_bindings/run_cpp.py' + - '/benchmarks/cuda_bindings/run_pyperf.py' + - '/benchmarks/cuda_bindings/pixi.lock' + - '/benchmarks/cuda_bindings/pixi.toml' + sharedTestInfra: + - '/cuda_python_test_helpers/**/*' + - '/benchmarks/**/*' + - '!/{benchmarks,cuda_python_test_helpers}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!/benchmarks/**/*.{md,svg}' + - '/ci/test-matrix.yml' + - '/ci/tools/download-wheels' + - '/ci/tools/run-tests' + linuxTestInfra: + - '/.github/workflows/test-wheel-linux.yml' + - '/ci/tools/guess_latest.sh' + - '/ci/tools/install_gpu_driver.sh' + - '/ci/tools/setup-sanitizer' + windowsTestInfra: + - '/.github/workflows/test-wheel-windows.yml' + - '/ci/tools/configure_driver_mode.ps1' + - '/ci/tools/install_gpu_driver.ps1' + +tasks: + test: + script: | + python -m pip install -e ../cuda_pathfinder -e . --group test + bash tests/cython/build_tests.sh + python -m pytest . --override-ini norecursedirs=examples + python -m pytest ../benchmarks/cuda_bindings/tests/ + inputs: ['@group(package)', '@group(tests)', '@group(benchmarks)', '/cuda_python_test_helpers/**/*'] + options: + cache: false + windowsShell: 'bash' + + test-installed: + script: | + set -euo pipefail + cd .. + temporary_wheels=() + for wheel in cuda_pathfinder/dist/*.whl; do + staged="cuda_pathfinder/$(basename "$wheel")" + if [[ ! -e "$staged" ]]; then + cp "$wheel" "$staged" + temporary_wheels+=("$staged") + fi + done + trap 'rm -f "${temporary_wheels[@]}"' EXIT + ci/tools/run-tests bindings + deps: ['wheel', 'build-cython-tests'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + options: + cache: false + internal: true + mutex: 'ci-python-gpu' + windowsShell: 'bash' + + build-cython-tests: + script: 'python -m pip install ../cuda_pathfinder/dist/*.whl dist/*.whl --group ./pyproject.toml:test && bash tests/cython/build_tests.sh' + deps: ['wheel'] + inputs: ['@group(package)', 'tests/cython/**/*', '/cuda_python_test_helpers/**/*'] + options: + cache: false + internal: true + mutex: 'ci-python-build' + windowsShell: 'bash' + + benchmark-smoke: + script: 'python -m pip install ../cuda_pathfinder/dist/*.whl dist/*.whl pyperf --group ./pyproject.toml:test && cd ../benchmarks/cuda_bindings && python run_pyperf.py --debug-single-value' + deps: ['wheel'] + inputs: ['@group(package)', '@group(benchmarks)'] + options: + cache: false + internal: true + mutex: 'ci-python-gpu' + windowsShell: 'bash' + + wheel: + script: 'cd .. && mkdir -p cuda_bindings/dist && python -m cibuildwheel cuda_bindings --output-dir cuda_bindings/dist' + deps: ['pathfinder:wheel'] + inputs: + - '@group(package)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_python/DESCRIPTION.rst' + - '/cuda_python/LICENSE' + - '/cuda_python/pyproject.toml' + - '/cuda_python/setup.py' + - '/cuda_python/README.md' + - '/README.md' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-wheel-bindings'] + + sdist: + script: 'mkdir -p dist && export PIP_FIND_LINKS=../cuda_pathfinder/dist PIP_PRE=1 && python -m build --sdist --outdir dist . && python -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz' + deps: ['pathfinder:sdist'] + inputs: + - '@group(package)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_python/DESCRIPTION.rst' + - '/cuda_python/LICENSE' + - '/cuda_python/pyproject.toml' + - '/cuda_python/setup.py' + - '/cuda_python/README.md' + - '/README.md' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-sdist-bindings'] + + docs: + script: 'rm -rf docs/build docs/source/generated && cd docs && ./build_docs.sh ${DOCS_BUILD_ARGS:-}' + inputs: ['@group(docs)'] + options: + cache: false + tags: ['ci-docs'] + + benchmark: + script: 'python -m pip install -e ../cuda_pathfinder -e . --group test && python -m pip install pyperf && cd ../benchmarks/cuda_bindings && python run_pyperf.py' + inputs: ['@group(package)', '@group(benchmarks)'] + options: + cache: false + + ci-test-assets: + deps: ['build-cython-tests'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(linuxTestInfra)' + - '@group(windowsTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + options: + cache: false + tags: ['ci-test-assets-current'] + + ci-test-linux: + deps: ['test-installed', 'benchmark-smoke'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(linuxTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + options: + cache: false + tags: ['ci-test-linux'] + + ci-test-windows: + deps: ['test-installed'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(windowsTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + options: + cache: false + tags: ['ci-test-windows'] diff --git a/cuda_core/moon.yml b/cuda_core/moon.yml new file mode 100644 index 00000000000..0876774cedb --- /dev/null +++ b/cuda_core/moon.yml @@ -0,0 +1,310 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +$schema: https://moonrepo.dev/schemas/v2/project.json + +dependsOn: ['bindings'] + +fileGroups: + package: + - 'cuda/**/*' + - '!cuda/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!cuda/**/*.{md,svg}' + - 'build_hooks.py' + - 'DESCRIPTION.rst' + - 'LICENSE' + - 'MANIFEST.in' + - 'NOTICE' + - 'pyproject.toml' + - 'setup.py' + - '.git_archival.txt' + - '/.git_archival.txt' + tests: + - 'tests/**/*' + - 'examples/**/*' + - '!{tests,examples}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - 'pytest.ini' + docs: + - 'docs/**/*' + - '!docs/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + sharedTestInfra: + - '/cuda_python_test_helpers/**/*' + - '/benchmarks/**/*' + - '!/{benchmarks,cuda_python_test_helpers}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!/benchmarks/**/*.{md,svg}' + - '/ci/test-matrix.yml' + - '/ci/tools/download-wheels' + - '/ci/tools/run-tests' + linuxTestInfra: + - '/.github/workflows/test-wheel-linux.yml' + - '/ci/tools/guess_latest.sh' + - '/ci/tools/install_gpu_driver.sh' + - '/ci/tools/setup-sanitizer' + windowsTestInfra: + - '/.github/workflows/test-wheel-windows.yml' + - '/ci/tools/configure_driver_mode.ps1' + - '/ci/tools/install_gpu_driver.ps1' + upstreamPackageVersions: + - '/cuda_pathfinder/.git_archival.txt' + - '/cuda_bindings/.git_archival.txt' + +tasks: + test: + script: | + python -m pip install -e ../cuda_pathfinder -e ../cuda_bindings -e . --group test + bash tests/cython/build_tests.sh + python -m pytest . --override-ini norecursedirs="" + inputs: ['@group(package)', '@group(tests)', '/cuda_python_test_helpers/**/*'] + options: + cache: false + windowsShell: 'bash' + + test-installed: + script: | + set -euo pipefail + cd .. + temporary_wheels=() + for wheel in cuda_pathfinder/dist/*.whl; do + staged="cuda_pathfinder/$(basename "$wheel")" + if [[ ! -e "$staged" ]]; then + cp "$wheel" "$staged" + temporary_wheels+=("$staged") + fi + done + trap 'rm -f "${temporary_wheels[@]}"' EXIT + ci/tools/run-tests core + deps: ['wheel', 'build-cython-tests', 'build-test-binaries'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + options: + cache: false + internal: true + mutex: 'ci-python-gpu' + windowsShell: 'bash' + + build-cython-tests: + script: | + set -euo pipefail + core_wheels=(dist/cu${CUDA_CORE_BUILD_MAJOR:?}/*.whl) + if [[ ! -e "${core_wheels[0]}" ]]; then + core_wheels=(dist/*.whl) + fi + python -m pip install ../cuda_pathfinder/dist/*.whl ../cuda_bindings/dist/*.whl "${core_wheels[@]}" --group ./pyproject.toml:test + bash tests/cython/build_tests.sh + deps: ['wheel'] + inputs: ['@group(package)', 'tests/cython/**/*', '/cuda_bindings/cuda/**/*', '/cuda_python_test_helpers/**/*'] + options: + cache: false + internal: true + mutex: 'ci-python-build' + windowsShell: 'bash' + + build-test-binaries: + command: 'python' + args: ['tests/test_binaries/build_test_binaries.py'] + inputs: ['tests/test_binaries/build_test_binaries.py', 'tests/test_binaries/saxpy.cu'] + options: + cache: false + internal: true + mutex: 'ci-python-build' + + wheel: + script: | + cd .. + cuda_major=${CUDA_CORE_BUILD_MAJOR:?} + mkdir -p "cuda_core/dist/cu${cuda_major}" + python -m cibuildwheel cuda_core --output-dir "cuda_core/dist/cu${cuda_major}" + shopt -s nullglob + wheels=("cuda_core/dist/cu${cuda_major}"/*.whl) + test "${#wheels[@]}" -eq 1 + if [[ "${wheels[0]}" != *.cu${cuda_major}.whl ]]; then + mv "${wheels[0]}" "${wheels[0]%.whl}.cu${cuda_major}.whl" + fi + deps: ['bindings:wheel'] + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/ci/tools/merge_cuda_core_wheels.py' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-wheel-consumers', 'ci-wheel-multi-ctk'] + + wheel-merge: + script: 'python ../ci/tools/merge_cuda_core_wheels.py dist/cu12/*.whl dist/cu13/*.whl --output-dir dist' + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - 'dist/cu12/*.whl' + - 'dist/cu13/*.whl' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/ci/tools/merge_cuda_core_wheels.py' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-wheel-finalize'] + + sdist: + script: 'mkdir -p dist && export PIP_FIND_LINKS="../cuda_pathfinder/dist ../cuda_bindings/dist" PIP_PRE=1 && python -m build --sdist --outdir dist . && python -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz' + deps: ['bindings:sdist'] + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-sdist-consumers'] + + docs: + script: 'rm -rf docs/build docs/source/generated && cd docs && ./build_docs.sh ${DOCS_BUILD_ARGS:-}' + inputs: ['@group(docs)'] + options: + cache: false + tags: ['ci-docs'] + + api-check: + script: 'uvx griffe check cuda.core --search . --find-stubs-packages --against "${CUDA_CORE_API_REF:?}" --format github 2>&1' + inputs: ['@group(package)', '/.github/actions/griffe-api-check/action.yml'] + options: + cache: false + tags: ['ci-api'] + + ci-test-assets: + deps: ['build-cython-tests'] + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(linuxTestInfra)' + - '@group(windowsTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + options: + cache: false + tags: ['ci-test-assets-current'] + + ci-test-binaries: + deps: ['build-test-binaries'] + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(linuxTestInfra)' + - '@group(windowsTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + options: + cache: false + tags: ['ci-test-assets-previous'] + + ci-test-linux: + deps: ['test-installed'] + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(linuxTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + options: + cache: false + tags: ['ci-test-linux'] + + ci-test-windows: + deps: ['test-installed'] + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(windowsTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + options: + cache: false + tags: ['ci-test-windows'] diff --git a/cuda_pathfinder/moon.yml b/cuda_pathfinder/moon.yml new file mode 100644 index 00000000000..f898fc5661d --- /dev/null +++ b/cuda_pathfinder/moon.yml @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +$schema: https://moonrepo.dev/schemas/v2/project.json + +fileGroups: + package: + - 'cuda/**/*' + - '!cuda/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!cuda/**/*.{md,svg}' + - 'DESCRIPTION.rst' + - 'LICENSE' + - 'pyproject.toml' + - '.git_archival.txt' + - '/.git_archival.txt' + tests: + - 'tests/**/*' + - 'examples/**/*' + - '!{tests,examples}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + docs: + - 'docs/**/*' + - '!docs/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + sharedTestInfra: + - '/cuda_python_test_helpers/**/*' + - '/benchmarks/**/*' + - '!/{benchmarks,cuda_python_test_helpers}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!/benchmarks/**/*.{md,svg}' + - '/ci/test-matrix.yml' + - '/ci/tools/download-wheels' + - '/ci/tools/run-tests' + linuxTestInfra: + - '/.github/workflows/test-wheel-linux.yml' + - '/ci/tools/guess_latest.sh' + - '/ci/tools/install_gpu_driver.sh' + - '/ci/tools/setup-sanitizer' + windowsTestInfra: + - '/.github/workflows/test-wheel-windows.yml' + - '/ci/tools/configure_driver_mode.ps1' + - '/ci/tools/install_gpu_driver.ps1' + +tasks: + test: + script: 'python -m pip install -e . --group test && python -m pytest tests/' + inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)'] + options: + cache: false + windowsShell: 'bash' + + test-installed: + script: | + set -euo pipefail + cd .. + temporary_wheels=() + for wheel in cuda_pathfinder/dist/*.whl; do + staged="cuda_pathfinder/$(basename "$wheel")" + if [[ ! -e "$staged" ]]; then + cp "$wheel" "$staged" + temporary_wheels+=("$staged") + fi + done + trap 'rm -f "${temporary_wheels[@]}"' EXIT + CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS=see_what_works CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS=see_what_works CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS=see_what_works ci/tools/run-tests pathfinder + python -m pip install --only-binary=:all: -v cuda_pathfinder/*.whl --group "./cuda_pathfinder/pyproject.toml:test-cu${TEST_CUDA_MAJOR:?}" + CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS=all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS=all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS=all_must_work ci/tools/run-tests pathfinder + deps: ['wheel'] + inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)'] + options: + cache: false + internal: true + mutex: 'ci-python-gpu' + windowsShell: 'bash' + + wheel: + script: 'mkdir -p dist && python -m pip wheel -v --no-deps --wheel-dir dist .' + inputs: ['@group(package)'] + options: + cache: false + windowsShell: 'bash' + tags: ['ci-wheel-foundation'] + + sdist: + script: 'mkdir -p dist && python -m build --sdist --outdir dist . && python -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz' + inputs: ['@group(package)'] + options: + cache: false + windowsShell: 'bash' + tags: ['ci-sdist-foundation'] + + docs: + script: 'rm -rf docs/build docs/source/generated && cd docs && ./build_docs.sh ${DOCS_BUILD_ARGS:-}' + inputs: ['@group(docs)'] + options: + cache: false + tags: ['ci-docs'] + + ci-test-linux: + deps: ['test-installed'] + inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)', '@group(linuxTestInfra)'] + options: + cache: false + tags: ['ci-test-linux'] + + ci-test-windows: + deps: ['test-installed'] + inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)', '@group(windowsTestInfra)'] + options: + cache: false + tags: ['ci-test-windows'] diff --git a/cuda_python/moon.yml b/cuda_python/moon.yml new file mode 100644 index 00000000000..52999bb9ca1 --- /dev/null +++ b/cuda_python/moon.yml @@ -0,0 +1,194 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +$schema: https://moonrepo.dev/schemas/v2/project.json + +dependsOn: ['bindings'] + +fileGroups: + package: + - 'DESCRIPTION.rst' + - 'LICENSE' + - 'pyproject.toml' + - 'setup.py' + - 'README.md' + - '/README.md' + - '/cuda_pathfinder/.git_archival.txt' + - '/cuda_bindings/.git_archival.txt' + - '/.git_archival.txt' + tests: + - '/tests/integration/**/*' + - '!/tests/integration/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + docs: + - 'docs/**/*' + - '!docs/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + sharedTestInfra: + - '/cuda_python_test_helpers/**/*' + - '/benchmarks/**/*' + - '!/{benchmarks,cuda_python_test_helpers}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!/benchmarks/**/*.{md,svg}' + - '/cuda_core/.git_archival.txt' + - '/ci/test-matrix.yml' + - '/ci/tools/download-wheels' + - '/ci/tools/run-tests' + linuxTestInfra: + - '/.github/workflows/test-wheel-linux.yml' + - '/ci/tools/guess_latest.sh' + - '/ci/tools/install_gpu_driver.sh' + - '/ci/tools/setup-sanitizer' + windowsTestInfra: + - '/.github/workflows/test-wheel-windows.yml' + - '/ci/tools/configure_driver_mode.ps1' + - '/ci/tools/install_gpu_driver.ps1' + +tasks: + test-installed: + script: | + set -euo pipefail + core_wheels=(../cuda_core/dist/*.whl) + if [[ ! -e "${core_wheels[0]}" ]]; then + core_wheels=(../cuda_core/dist/cu${CUDA_CORE_BUILD_MAJOR:?}/*.whl) + fi + python -m pip install --only-binary=:all: ../cuda_pathfinder/dist/*.whl ../cuda_bindings/dist/*.whl "${core_wheels[@]}" dist/*.whl + deps: ['wheel', 'core:wheel'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/cuda_core/cuda/**/*' + - '/cuda_core/build_hooks.py' + - '/cuda_core/DESCRIPTION.rst' + - '/cuda_core/LICENSE' + - '/cuda_core/MANIFEST.in' + - '/cuda_core/NOTICE' + - '/cuda_core/pyproject.toml' + - '/cuda_core/setup.py' + options: + cache: false + internal: true + mutex: 'ci-python-gpu' + windowsShell: 'bash' + + wheel: + script: 'mkdir -p dist && python -m pip wheel -v --no-deps --wheel-dir dist .' + deps: ['bindings:wheel'] + inputs: + - '@group(package)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/cuda_python/README.md' + - '/README.md' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-wheel-consumers'] + + sdist: + script: 'mkdir -p dist && python -m build --sdist --outdir dist . && python -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz' + inputs: + - '@group(package)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/cuda_python/README.md' + - '/README.md' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-sdist-consumers'] + + docs: + script: 'rm -rf docs/build docs/source/generated && cd docs && ./build_docs.sh ${DOCS_BUILD_ARGS:-}' + inputs: ['@group(docs)'] + options: + cache: false + tags: ['ci-docs'] + + ci-test-linux: + deps: ['test-installed'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(linuxTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/cuda_core/cuda/**/*' + - '/cuda_core/build_hooks.py' + - '/cuda_core/DESCRIPTION.rst' + - '/cuda_core/LICENSE' + - '/cuda_core/MANIFEST.in' + - '/cuda_core/NOTICE' + - '/cuda_core/pyproject.toml' + - '/cuda_core/setup.py' + options: + cache: false + tags: ['ci-test-linux'] + + ci-test-windows: + deps: ['test-installed'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(windowsTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/cuda_core/cuda/**/*' + - '/cuda_core/build_hooks.py' + - '/cuda_core/DESCRIPTION.rst' + - '/cuda_core/LICENSE' + - '/cuda_core/MANIFEST.in' + - '/cuda_core/NOTICE' + - '/cuda_core/pyproject.toml' + - '/cuda_core/setup.py' + options: + cache: false + tags: ['ci-test-windows'] diff --git a/moon.yml b/moon.yml new file mode 100644 index 00000000000..2670648d385 --- /dev/null +++ b/moon.yml @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +$schema: https://moonrepo.dev/schemas/v2/project.json + +tasks: + test: + deps: + - 'pathfinder:test' + - 'bindings:test' + - 'core:test' + inputs: [] + options: + cache: false + + docs: + script: | + rm -rf cuda_python/docs/build/html/cuda-bindings cuda_python/docs/build/html/cuda-core cuda_python/docs/build/html/cuda-pathfinder + mkdir -p cuda_python/docs/build/html/cuda-bindings cuda_python/docs/build/html/cuda-core cuda_python/docs/build/html/cuda-pathfinder + cp -R cuda_bindings/docs/build/html/. cuda_python/docs/build/html/cuda-bindings/ + cp -R cuda_core/docs/build/html/. cuda_python/docs/build/html/cuda-core/ + cp -R cuda_pathfinder/docs/build/html/. cuda_python/docs/build/html/cuda-pathfinder/ + deps: + - 'pathfinder:docs' + - 'bindings:docs' + - 'core:docs' + - 'metapackage:docs' + inputs: [] + options: + cache: false + tags: ['ci-docs'] + + ci-ignore: + inputs: + - '/.agents/**/*' + - '/.coveragerc' + - '/.gitattributes' + - '/.gitignore' + - '/.github/**/*' + - '!/.github/actions/**/*' + - '!/.github/workflows/**/*' + - '/.mailmap' + - '/.pre-commit-config.yaml' + - '/.spdx-ignore' + - '/**/AGENTS.md' + - '/**/CLAUDE.md' + - '/**/pixi.lock' + - '/**/pixi.toml' + - '/**/*.md' + - '/**/*.svg' + - '!/README.md' + - '!/cuda_python/README.md' + - '/context7.json' + - '/greptile.json' + - '/LICENSE' + - '/ruff.toml' + - '/toolshed/**/*' + options: + cache: false + tags: ['ci-ignore'] + + ci-fallback: + deps: + - 'pathfinder:wheel' + - 'bindings:wheel' + - 'core:wheel' + - 'metapackage:wheel' + - 'core:wheel-merge' + - 'pathfinder:sdist' + - 'bindings:sdist' + - 'core:sdist' + - 'metapackage:sdist' + - 'pathfinder:ci-test-linux' + - 'pathfinder:ci-test-windows' + - 'bindings:ci-test-linux' + - 'bindings:ci-test-windows' + - 'bindings:ci-test-assets' + - 'core:ci-test-linux' + - 'core:ci-test-windows' + - 'core:ci-test-assets' + - 'core:ci-test-binaries' + - 'metapackage:ci-test-linux' + - 'metapackage:ci-test-windows' + - 'core:api-check' + - 'docs' + inputs: + - '/.github/actions/**/*' + - '/.github/workflows/build-docs.yml' + - '/.github/workflows/build-wheel.yml' + - '/.github/workflows/ci-nightly.yml' + - '/.github/workflows/ci-pixi-source-test.yml' + - '/.github/workflows/ci.yml' + - '/.github/workflows/coverage.yml' + - '/.github/workflows/release*.yml' + - '/.github/workflows/test-sdist-linux.yml' + - '/.github/workflows/test-sdist-windows.yml' + - '/.moon/workspace.yml' + - '/moon.yml' + - '/cuda_pathfinder/moon.yml' + - '/cuda_bindings/moon.yml' + - '/cuda_core/moon.yml' + - '/cuda_python/moon.yml' + - '/ci/tools/env-vars' + - '/ci/versions.yml' + - '/pyproject.toml' + - '/pytest.ini' + - '/tests/test_moon_ci.py' + options: + cache: false + tags: ['ci-force-all'] diff --git a/tests/test_moon_ci.py b/tests/test_moon_ci.py new file mode 100644 index 00000000000..d7124f36076 --- /dev/null +++ b/tests/test_moon_ci.py @@ -0,0 +1,600 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Behavior checks for the Moon-owned selective CI graph.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import textwrap +from pathlib import Path +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +MOON = shutil.which("moon") +BASH = shutil.which("bash") + +VISIBLE_TASKS = { + "root": {"test", "docs", "ci-ignore", "ci-fallback"}, + "pathfinder": {"test", "docs", "wheel", "sdist", "ci-test-linux", "ci-test-windows"}, + "bindings": { + "test", + "docs", + "wheel", + "sdist", + "benchmark", + "ci-test-linux", + "ci-test-windows", + "ci-test-assets", + }, + "core": { + "test", + "docs", + "wheel", + "wheel-merge", + "sdist", + "api-check", + "ci-test-linux", + "ci-test-windows", + "ci-test-assets", + "ci-test-binaries", + }, + "metapackage": {"docs", "wheel", "sdist", "ci-test-linux", "ci-test-windows"}, +} +INTERNAL_TASKS = { + "pathfinder:test-installed", + "bindings:test-installed", + "bindings:build-cython-tests", + "bindings:benchmark-smoke", + "core:test-installed", + "core:build-cython-tests", + "core:build-test-binaries", + "metapackage:test-installed", +} +ALL_ROUTES = { + f"{project}:ci-test-{os_name}" + for project in ("pathfinder", "bindings", "core", "metapackage") + for os_name in ("linux", "windows") +} +LINUX_ROUTES = {target for target in ALL_ROUTES if target.endswith("-linux")} +WINDOWS_ROUTES = {target for target in ALL_ROUTES if target.endswith("-windows")} +ASSET_ROUTES = {"bindings:ci-test-assets", "core:ci-test-assets", "core:ci-test-binaries"} +PRODUCERS = { + f"{project}:{kind}" for project in ("pathfinder", "bindings", "core", "metapackage") for kind in ("wheel", "sdist") +} | {"core:wheel-merge"} +CI_TASKS = ( + ALL_ROUTES + | ASSET_ROUTES + | PRODUCERS + | {f"{project}:docs" for project in ("root", "pathfinder", "bindings", "core", "metapackage")} + | {"core:api-check", "root:ci-ignore", "root:ci-fallback"} +) + + +def run_moon(*args: str, stdin: str | None = None) -> subprocess.CompletedProcess[str]: + assert MOON is not None + return subprocess.run( # noqa: S603 - MOON resolves to the pinned executable. + [MOON, *args], + cwd=ROOT, + input=stdin, + text=True, + check=False, + capture_output=True, + ) + + +def moon_json(*args: str, stdin: str | None = None) -> dict[str, Any]: + result = run_moon(*args, stdin=stdin) + result.check_returncode() + return json.loads(result.stdout) + + +def targets(payload: dict[str, Any]) -> set[str]: + return {f"{project}:{task}" for project, project_tasks in payload["tasks"].items() for task in project_tasks} + + +def affected(*paths: str) -> set[str]: + payload = moon_json( + "query", + "tasks", + "--affected", + "stdin", + "--upstream", + "none", + "--downstream", + "none", + stdin="".join(f"{path}\n" for path in paths), + ) + return targets(payload) & CI_TASKS + + +def task_graph(target: str) -> dict[str, dict[str, Any]]: + payload = moon_json("task-graph", target, "--json") + return {task["target"]: task for task in payload["data"].values()} + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def workflow_step_script(path: str, name: str) -> str: + lines = read(path).splitlines() + start = lines.index(f" - name: {name}") + run = next(index for index in range(start, len(lines)) if lines[index].strip() == "run: |") + end = next( + (index for index in range(run + 1, len(lines)) if lines[index].startswith(" - name:")), + len(lines), + ) + return textwrap.dedent("\n".join(lines[run + 1 : end])).replace("${{ github.repository }}", "NVIDIA/cuda-python") + + +def baseline_artifacts(*, merge_base: str, expired: str | None = None) -> list[dict[str, object]]: + names = ["cuda-pathfinder-wheel", "cuda-python-wheel"] + for version in ("3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15", "3.15t"): + python = version.replace(".", "") + for platform in ("linux-64", "linux-aarch64", "win-64"): + names.append(f"cuda-bindings-python{python}-cuda13.3.0-{platform}-{merge_base}") + names.append(f"cuda-core-python{python}-{platform}-{merge_base}") + return [{"name": name, "expired": name == expired} for name in names] + + +@pytest.mark.skipif(MOON is None, reason="Moon 2.5.1 is required") +@pytest.mark.agent_authored(model="gpt-5") +class TestMoonCi: + def test_workspace_is_pinned_and_visible_inventory_is_exact(self) -> None: + assert run_moon("--version").stdout.strip() == "moon 2.5.1" + payload = moon_json("query", "tasks") + assert {project: set(project_tasks) for project, project_tasks in payload["tasks"].items()} == VISIBLE_TASKS + assert not (targets(payload) & INTERNAL_TASKS) + + def test_internal_inventory_is_hidden_and_rejects_direct_execution(self) -> None: + graph = task_graph("root:ci-fallback") + internal = {target for target, task in graph.items() if task["options"]["internal"]} + assert internal == INTERNAL_TASKS + for target in sorted(INTERNAL_TASKS): + result = run_moon("run", target, "--upstream", "none", "--downstream", "none") + assert result.returncode != 0 + assert "Unknown task" in result.stderr + + def test_all_tasks_disable_caching_and_routes_are_commandless(self) -> None: + visible = moon_json("query", "tasks")["tasks"] + graph = task_graph("root:ci-fallback") + assert all(task["options"]["cache"] is False for tasks in visible.values() for task in tasks.values()) + assert all(task["options"]["cache"] is False for task in graph.values()) + for target in ALL_ROUTES | ASSET_ROUTES | {"root:ci-ignore", "root:ci-fallback", "root:test"}: + project, task = target.split(":") + assert visible[project][task]["command"] == "noop" + + def test_semantic_tag_inventory_is_exact(self) -> None: + payload = moon_json("query", "tasks") + actual = { + target: set(payload["tasks"][target.split(":")[0]][target.split(":")[1]].get("tags", [])) + for target in targets(payload) + if payload["tasks"][target.split(":")[0]][target.split(":")[1]].get("tags") + } + expected = { + "pathfinder:wheel": {"ci-wheel-foundation"}, + "bindings:wheel": {"ci-wheel-bindings"}, + "core:wheel": {"ci-wheel-consumers", "ci-wheel-multi-ctk"}, + "metapackage:wheel": {"ci-wheel-consumers"}, + "core:wheel-merge": {"ci-wheel-finalize"}, + "pathfinder:sdist": {"ci-sdist-foundation"}, + "bindings:sdist": {"ci-sdist-bindings"}, + "core:sdist": {"ci-sdist-consumers"}, + "metapackage:sdist": {"ci-sdist-consumers"}, + "bindings:ci-test-assets": {"ci-test-assets-current"}, + "core:ci-test-assets": {"ci-test-assets-current"}, + "core:ci-test-binaries": {"ci-test-assets-previous"}, + "core:api-check": {"ci-api"}, + "root:ci-ignore": {"ci-ignore"}, + "root:ci-fallback": {"ci-force-all"}, + } + expected.update({target: {"ci-test-linux"} for target in LINUX_ROUTES}) + expected.update({target: {"ci-test-windows"} for target in WINDOWS_ROUTES}) + expected.update({f"{project}:docs": {"ci-docs"} for project in VISIBLE_TASKS}) + assert actual == expected + + def test_package_source_impact_routes(self) -> None: + cases = { + "cuda_pathfinder/cuda/pathfinder/__init__.py": PRODUCERS | ALL_ROUTES | ASSET_ROUTES, + "cuda_bindings/cuda/bindings/__init__.py": { + "bindings:wheel", + "bindings:sdist", + "core:wheel", + "core:wheel-merge", + "core:sdist", + "metapackage:wheel", + "metapackage:sdist", + "bindings:ci-test-linux", + "bindings:ci-test-windows", + "core:ci-test-linux", + "core:ci-test-windows", + "metapackage:ci-test-linux", + "metapackage:ci-test-windows", + } + | ASSET_ROUTES, + "cuda_core/cuda/core/__init__.py": { + "core:wheel", + "core:wheel-merge", + "core:sdist", + "core:api-check", + "core:ci-test-linux", + "core:ci-test-windows", + "metapackage:ci-test-linux", + "metapackage:ci-test-windows", + "core:ci-test-assets", + "core:ci-test-binaries", + }, + "cuda_python/pyproject.toml": { + "bindings:wheel", + "bindings:sdist", + "metapackage:wheel", + "metapackage:sdist", + "metapackage:ci-test-linux", + "metapackage:ci-test-windows", + }, + } + cases["cuda_pathfinder/.git_archival.txt"] = cases["cuda_pathfinder/cuda/pathfinder/__init__.py"] + cases["cuda_bindings/.git_archival.txt"] = cases["cuda_bindings/cuda/bindings/__init__.py"] + cases["cuda_core/.git_archival.txt"] = cases["cuda_core/cuda/core/__init__.py"] + for path, expected in cases.items(): + assert affected(path) == expected, path + + def test_tests_helpers_benchmarks_and_os_infrastructure_impact(self) -> None: + cases = { + "cuda_pathfinder/tests/test_pathfinder.py": { + "pathfinder:ci-test-linux", + "pathfinder:ci-test-windows", + }, + "cuda_bindings/examples/0_Introduction/vectorAddDrv.py": { + "bindings:ci-test-linux", + "bindings:ci-test-windows", + "bindings:ci-test-assets", + }, + "cuda_core/tests/test_device.py": { + "core:ci-test-linux", + "core:ci-test-windows", + "core:ci-test-assets", + "core:ci-test-binaries", + }, + "cuda_python_test_helpers/pyproject.toml": ALL_ROUTES | ASSET_ROUTES, + "benchmarks/cuda_bindings/run_pyperf.py": ALL_ROUTES | ASSET_ROUTES, + "benchmarks/cuda_bindings/compare.py": ALL_ROUTES | ASSET_ROUTES, + "benchmarks/cuda_core/runtime.py": ALL_ROUTES | ASSET_ROUTES, + ".github/workflows/test-wheel-linux.yml": LINUX_ROUTES | ASSET_ROUTES, + ".github/workflows/test-wheel-windows.yml": WINDOWS_ROUTES | ASSET_ROUTES, + "ci/tools/guess_latest.sh": LINUX_ROUTES | ASSET_ROUTES, + } + for path, expected in cases.items(): + assert affected(path) == expected, path + + def test_docs_ignored_unknown_and_fallback_ownership(self) -> None: + assert affected("cuda_core/docs/source/index.rst") == {"core:docs"} + for path in ( + ".coveragerc", + ".github/ISSUE_TEMPLATE/bug_report.yml", + ".github/labeler.yml", + ".pre-commit-config.yaml", + "CONTRIBUTING.md", + "context7.json", + "cuda_core/pixi.toml", + "cuda_core/tests/AGENTS.md", + "diagram.svg", + "greptile.json", + "new-area/pixi.lock", + "ruff.toml", + "toolshed/README.md", + ): + assert affected(path) == {"root:ci-ignore"} + assert affected(".github/workflows/ci.yml") == {"root:ci-fallback"} + assert affected("an-entirely-new-path.txt") == set() + gate = read(".github/workflows/ci.yml") + assert 'length == 0 or any(.[]; .target == "root:ci-fallback")' in gate + + fallback = task_graph("root:ci-fallback")["root:ci-fallback"] + assert {dep["target"] for dep in fallback["deps"]} == ( + PRODUCERS | ALL_ROUTES | ASSET_ROUTES | {"core:api-check", "root:docs"} + ) + for path in (".moon/workspace.yml", "moon.yml", "cuda_core/moon.yml", "ci/versions.yml"): + assert "root:ci-fallback" in affected(path) + + def test_mixed_changes_and_symlink_consumers(self) -> None: + assert affected("cuda_core/docs/source/index.rst", "cuda_bindings/tests/test_api.py") == { + "core:docs", + "bindings:ci-test-linux", + "bindings:ci-test-windows", + "bindings:ci-test-assets", + } + expected_readme = { + "bindings:wheel", + "bindings:sdist", + "metapackage:wheel", + "metapackage:sdist", + "metapackage:ci-test-linux", + "metapackage:ci-test-windows", + } + assert affected("README.md") == expected_readme + assert affected("cuda_python/README.md") == expected_readme + assert affected(".git_archival.txt") == PRODUCERS | ALL_ROUTES | ASSET_ROUTES | {"core:api-check"} + + def test_editable_tests_do_not_build_wheels(self) -> None: + for target in ("pathfinder:test", "bindings:test", "core:test"): + graph = task_graph(target) + assert set(graph) == {target} + script = graph[target]["script"] + assert "pip install -e" in script + assert "pip wheel" not in script + assert "cibuildwheel" not in script + + def test_local_core_wheel_builds_current_dependency_chain(self) -> None: + assert set(task_graph("core:wheel")) == { + "pathfinder:wheel", + "bindings:wheel", + "core:wheel", + } + + def test_ci_routes_have_only_hidden_direct_executors(self) -> None: + expected = { + "pathfinder:ci-test-linux": {"pathfinder:test-installed"}, + "pathfinder:ci-test-windows": {"pathfinder:test-installed"}, + "bindings:ci-test-linux": {"bindings:test-installed", "bindings:benchmark-smoke"}, + "bindings:ci-test-windows": {"bindings:test-installed"}, + "core:ci-test-linux": {"core:test-installed"}, + "core:ci-test-windows": {"core:test-installed"}, + "metapackage:ci-test-linux": {"metapackage:test-installed"}, + "metapackage:ci-test-windows": {"metapackage:test-installed"}, + } + fallback = task_graph("root:ci-fallback") + for route, direct_targets in expected.items(): + actual = {dep["target"] for dep in fallback[route]["deps"]} + assert actual == direct_targets + assert all(fallback[target]["options"]["internal"] for target in actual) + assert all(fallback[target].get("deps") for target in actual) + for workflow in (".github/workflows/test-wheel-linux.yml", ".github/workflows/test-wheel-windows.yml"): + assert 'moon run "${target_args[@]}" --upstream direct --downstream none' in read(workflow) + + def test_build_traversal_stages_dependencies_and_runs_exact_targets(self) -> None: + workflow = read(".github/workflows/build-wheel.yml") + for phase in ( + "WHEEL_FOUNDATION_TARGETS", + "WHEEL_BINDINGS_TARGETS", + "WHEEL_CONSUMER_TARGETS", + "WHEEL_MULTI_CTK_TARGETS", + "WHEEL_FINALIZE_TARGETS", + ): + assert phase in workflow + assert workflow.count("--upstream none --downstream none") >= 5 + assert workflow.count("--upstream direct --downstream none") >= 2 + assert "Download reusable cuda.pathfinder wheel" in workflow + assert "Download reusable cuda.bindings wheel" in workflow + assert workflow.count("python -m pip install cibuildwheel twine wheel") == 2 + + def test_native_assets_follow_the_selected_os(self, tmp_path: Path) -> None: + assert BASH is not None + script = workflow_step_script(".github/workflows/build-wheel.yml", "Resolve Moon phase targets") + common = { + "WHEEL_FOUNDATION_TARGETS": "[]", + "WHEEL_BINDINGS_TARGETS": "[]", + "WHEEL_CONSUMER_TARGETS": "[]", + "WHEEL_MULTI_CTK_TARGETS": "[]", + "WHEEL_FINALIZE_TARGETS": "[]", + "TEST_ASSETS_CURRENT_TARGETS": '["bindings:ci-test-assets","core:ci-test-assets"]', + "TEST_ASSETS_PREVIOUS_TARGETS": '["core:ci-test-binaries"]', + } + cases = { + "linux-selected": { + "TEST_LINUX_TARGETS": '["core:ci-test-linux"]', + "TEST_WINDOWS_TARGETS": "[]", + "linux-64": "true", + "win-64": "false", + }, + "windows-selected": { + "TEST_LINUX_TARGETS": "[]", + "TEST_WINDOWS_TARGETS": '["core:ci-test-windows"]', + "linux-64": "false", + "win-64": "true", + }, + } + for case_name, case in cases.items(): + for platform in ("linux-64", "win-64"): + output = tmp_path / f"{case_name}-{platform}.env" + env = ( + os.environ + | common + | { + "HOST_PLATFORM": platform, + "GITHUB_ENV": str(output), + "TEST_LINUX_TARGETS": case["TEST_LINUX_TARGETS"], + "TEST_WINDOWS_TARGETS": case["TEST_WINDOWS_TARGETS"], + } + ) + result = subprocess.run( # noqa: S603 - controlled repository script. + [BASH, "-c", script], + cwd=ROOT, + env=env, + text=True, + check=False, + capture_output=True, + ) + assert result.returncode == 0, (case_name, platform, result.stderr) + values = dict(line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines()) + assert values["TEST_BINDINGS"] == case[platform] + assert values["TEST_CORE_CURRENT"] == case[platform] + assert values["TEST_CORE_PREVIOUS"] == case[platform] + + workflow = read(".github/workflows/ci.yml") + linux_arm = workflow.split(" build-linux-aarch64:", 1)[1].split(" build-windows:", 1)[0] + windows = workflow.split(" build-windows:", 1)[1].split(" test-sdist-linux:", 1)[0] + assert "ci-test-assets" not in linux_arm + assert "ci-test-assets" not in windows + + def test_core_uses_one_target_in_both_toolkits_then_merges(self) -> None: + graph = task_graph("root:ci-fallback") + assert set(graph["core:wheel"]["tags"]) == {"ci-wheel-consumers", "ci-wheel-multi-ctk"} + assert not graph["core:wheel-merge"].get("deps") + merger = graph["core:wheel-merge"]["script"] + assert "dist/cu12/*.whl dist/cu13/*.whl" in merger + assert "merge_cuda_core_wheels.py" in merger + + def test_only_merged_core_wheel_is_in_baseline_artifact(self) -> None: + workflow = read(".github/workflows/build-wheel.yml") + assert "name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}" in workflow + assert "path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl" in workflow + assert "path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu" not in workflow + for name in ("cuda-pathfinder-wheel", "cuda-python-wheel"): + assert f"name: {name}" in workflow + + def test_baseline_reuse_requires_one_exact_successful_complete_set(self) -> None: + workflow = read(".github/workflows/ci.yml") + for contract in ( + '--commit "${merge_base}"', + "--event push", + "--status success", + "if [[ $(jq 'length' <<< \"$runs\") -ne 1 ]]", + '"${run_sha}" != "${merge_base}"', + "length == 1 and .[0].expired == false", + "if (( ${#missing[@]} != 0 ))", + 'baseline_run_id=""', + 'baseline_sha=""', + ): + assert contract in workflow + assert "cuda-pathfinder-wheel cuda-python-wheel" in workflow + assert "CUDA_BINDINGS_ARTIFACT_BASENAME" in read(".github/workflows/build-wheel.yml") + assert "CUDA_CORE_ARTIFACT_BASENAME" in read(".github/workflows/build-wheel.yml") + assert "uvx --from pytest pytest -q tests/test_moon_ci.py" in workflow + + def test_baseline_reuse_behaviors(self, tmp_path: Path) -> None: + assert BASH is not None + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + gh = fake_bin / "gh" + gh.write_text( + """#!/usr/bin/env bash +printf '%s\\n' "$*" >> "$MOCK_GH_LOG" +if [[ "$1 $2" == "run list" ]]; then + printf '%s\\n' "$MOCK_RUNS" + exit "$MOCK_RUN_STATUS" +fi +if [[ "$1" == "api" ]]; then + printf '%s\\n' "$MOCK_ARTIFACTS" + exit "$MOCK_ARTIFACT_STATUS" +fi +exit 2 +""", + encoding="utf-8", + ) + yq = fake_bin / "yq" + yq.write_text( + """#!/usr/bin/env bash +if [[ "$1" == "-r" ]]; then + printf '%s\\n' 3.10 3.11 3.12 3.13 3.14 3.14t 3.15 3.15t +else + printf '%s\\n' 13.3.0 +fi +""", + encoding="utf-8", + ) + os.chmod(gh, 0o700) + os.chmod(yq, 0o700) + + merge_base = "exact-base" + complete = baseline_artifacts(merge_base=merge_base) + cases = { + "complete": { + "runs": [{"databaseId": 42, "headSha": merge_base}], + "artifacts": complete, + "accepted": True, + }, + "incomplete": { + "runs": [{"databaseId": 42, "headSha": merge_base}], + "artifacts": complete[:-1], + "accepted": False, + }, + "expired": { + "runs": [{"databaseId": 42, "headSha": merge_base}], + "artifacts": baseline_artifacts(merge_base=merge_base, expired="cuda-pathfinder-wheel"), + "accepted": False, + }, + "failed-run": {"runs": [], "artifacts": complete, "accepted": False}, + "wrong-sha": { + "runs": [{"databaseId": 42, "headSha": "another-sha"}], + "artifacts": complete, + "accepted": False, + }, + "duplicate": { + "runs": [{"databaseId": 42, "headSha": merge_base}], + "artifacts": [*complete, complete[0]], + "accepted": False, + }, + "lookup-failure": { + "runs": [{"databaseId": 42, "headSha": merge_base}], + "artifacts": complete, + "accepted": False, + "run_status": 1, + }, + } + script = workflow_step_script(".github/workflows/ci.yml", "Resolve reusable base artifacts") + for name, case in cases.items(): + output = tmp_path / f"{name}.output" + summary = tmp_path / f"{name}.summary" + log = tmp_path / f"{name}.gh.log" + output.touch() + summary.touch() + env = os.environ | { + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "BASE_REF": "main", + "MERGE_BASE": merge_base, + "GITHUB_OUTPUT": str(output), + "GITHUB_STEP_SUMMARY": str(summary), + "MOCK_GH_LOG": str(log), + "MOCK_RUNS": json.dumps(case["runs"]), + "MOCK_ARTIFACTS": "\n".join(json.dumps(item) for item in case["artifacts"]), + "MOCK_RUN_STATUS": str(case.get("run_status", 0)), + "MOCK_ARTIFACT_STATUS": "0", + } + result = subprocess.run( # noqa: S603 - controlled script and fake tools. + [BASH, "-c", script], + cwd=ROOT, + env=env, + text=True, + check=False, + capture_output=True, + ) + assert result.returncode == 0, (name, result.stderr) + accepted = "run_id=42" in output.read_text(encoding="utf-8") + assert accepted is case["accepted"], name + if not case["accepted"]: + assert "No complete reusable artifact set" in summary.read_text(encoding="utf-8") + + complete_log = (tmp_path / "complete.gh.log").read_text(encoding="utf-8") + for argument in ("--commit exact-base", "--event push", "--status success"): + assert argument in complete_log + + def test_docs_select_component_or_parallel_aggregate_layout(self) -> None: + workflow = read(".github/workflows/build-docs.yml") + assert "all) targets='[\"root:docs\"]'" in workflow + for project in ("pathfinder", "bindings", "core", "metapackage"): + assert f'"{project}:docs"' in workflow + assert "DOCS_BUILD_ARGS" in workflow + assert "--upstream deep --downstream none" in workflow + assert "DOCS_USE_MOON" in workflow + assert "./build_all_docs.sh latest-only" in workflow + assert "./build_docs.sh latest-only" in workflow + root_docs = task_graph("root:docs") + assert {dep["target"] for dep in root_docs["root:docs"]["deps"]} == { + "pathfinder:docs", + "bindings:docs", + "core:docs", + "metapackage:docs", + } + script = root_docs["root:docs"]["script"] + for destination in ("cuda-bindings", "cuda-core", "cuda-pathfinder"): + assert f"cuda_python/docs/build/html/{destination}" in script + for project in ("pathfinder", "bindings", "core", "metapackage"): + assert "${DOCS_BUILD_ARGS:-}" in root_docs[f"{project}:docs"]["script"] diff --git a/toolshed/check_spdx.py b/toolshed/check_spdx.py index d4c9430673c..ce422aef997 100644 --- a/toolshed/check_spdx.py +++ b/toolshed/check_spdx.py @@ -23,6 +23,7 @@ TOP_LEVEL_DIRS_LICENSE_IDENTIFIERS = { ".agents": "Apache-2.0", ".github": "Apache-2.0", + ".moon": "Apache-2.0", "benchmarks": "Apache-2.0", "ci": "Apache-2.0", "cuda_bindings": "Apache-2.0", @@ -32,6 +33,7 @@ "cuda_python_test_helpers": "Apache-2.0", "qa": "LicenseRef-NVIDIA-SOFTWARE-LICENSE", "scripts": "Apache-2.0", + "tests": "Apache-2.0", "toolshed": "Apache-2.0", } From 139c2c49a3300a99b74ec6aa21d0230be8fdc8c0 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Thu, 20 Aug 2026 01:51:50 -0400 Subject: [PATCH 12/13] ci: separate editable installs from Moon tests --- cuda_bindings/moon.yml | 13 ++++++-- cuda_core/moon.yml | 10 +++++- cuda_pathfinder/moon.yml | 10 +++++- tests/test_moon_ci.py | 67 ++++++++++++++++++++++++++++++++++------ 4 files changed, 87 insertions(+), 13 deletions(-) diff --git a/cuda_bindings/moon.yml b/cuda_bindings/moon.yml index d38add85af1..d21b05fa280 100644 --- a/cuda_bindings/moon.yml +++ b/cuda_bindings/moon.yml @@ -55,12 +55,20 @@ fileGroups: - '/ci/tools/install_gpu_driver.ps1' tasks: + install: + script: 'python -m pip install -e . pyperf --group test' + deps: ['pathfinder:install'] + inputs: ['@group(package)'] + options: + cache: false + windowsShell: 'bash' + test: script: | - python -m pip install -e ../cuda_pathfinder -e . --group test bash tests/cython/build_tests.sh python -m pytest . --override-ini norecursedirs=examples python -m pytest ../benchmarks/cuda_bindings/tests/ + deps: ['install'] inputs: ['@group(package)', '@group(tests)', '@group(benchmarks)', '/cuda_python_test_helpers/**/*'] options: cache: false @@ -163,7 +171,8 @@ tasks: tags: ['ci-docs'] benchmark: - script: 'python -m pip install -e ../cuda_pathfinder -e . --group test && python -m pip install pyperf && cd ../benchmarks/cuda_bindings && python run_pyperf.py' + script: 'cd ../benchmarks/cuda_bindings && python run_pyperf.py' + deps: ['install'] inputs: ['@group(package)', '@group(benchmarks)'] options: cache: false diff --git a/cuda_core/moon.yml b/cuda_core/moon.yml index 0876774cedb..c4b5b47f455 100644 --- a/cuda_core/moon.yml +++ b/cuda_core/moon.yml @@ -50,11 +50,19 @@ fileGroups: - '/cuda_bindings/.git_archival.txt' tasks: + install: + script: 'python -m pip install -e . --group test' + deps: ['bindings:install'] + inputs: ['@group(package)'] + options: + cache: false + windowsShell: 'bash' + test: script: | - python -m pip install -e ../cuda_pathfinder -e ../cuda_bindings -e . --group test bash tests/cython/build_tests.sh python -m pytest . --override-ini norecursedirs="" + deps: ['install'] inputs: ['@group(package)', '@group(tests)', '/cuda_python_test_helpers/**/*'] options: cache: false diff --git a/cuda_pathfinder/moon.yml b/cuda_pathfinder/moon.yml index f898fc5661d..c164576c52c 100644 --- a/cuda_pathfinder/moon.yml +++ b/cuda_pathfinder/moon.yml @@ -40,8 +40,16 @@ fileGroups: - '/ci/tools/install_gpu_driver.ps1' tasks: + install: + script: 'python -m pip install -e . --group test' + inputs: ['@group(package)'] + options: + cache: false + windowsShell: 'bash' + test: - script: 'python -m pip install -e . --group test && python -m pytest tests/' + script: 'python -m pytest tests/' + deps: ['install'] inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)'] options: cache: false diff --git a/tests/test_moon_ci.py b/tests/test_moon_ci.py index d7124f36076..3dc06a0ebce 100644 --- a/tests/test_moon_ci.py +++ b/tests/test_moon_ci.py @@ -22,8 +22,9 @@ VISIBLE_TASKS = { "root": {"test", "docs", "ci-ignore", "ci-fallback"}, - "pathfinder": {"test", "docs", "wheel", "sdist", "ci-test-linux", "ci-test-windows"}, + "pathfinder": {"install", "test", "docs", "wheel", "sdist", "ci-test-linux", "ci-test-windows"}, "bindings": { + "install", "test", "docs", "wheel", @@ -34,6 +35,7 @@ "ci-test-assets", }, "core": { + "install", "test", "docs", "wheel", @@ -167,7 +169,15 @@ def test_all_tasks_disable_caching_and_routes_are_commandless(self) -> None: graph = task_graph("root:ci-fallback") assert all(task["options"]["cache"] is False for tasks in visible.values() for task in tasks.values()) assert all(task["options"]["cache"] is False for task in graph.values()) - for target in ALL_ROUTES | ASSET_ROUTES | {"root:ci-ignore", "root:ci-fallback", "root:test"}: + for target in ( + ALL_ROUTES + | ASSET_ROUTES + | { + "root:ci-ignore", + "root:ci-fallback", + "root:test", + } + ): project, task = target.split(":") assert visible[project][task]["command"] == "noop" @@ -323,14 +333,53 @@ def test_mixed_changes_and_symlink_consumers(self) -> None: assert affected("cuda_python/README.md") == expected_readme assert affected(".git_archival.txt") == PRODUCERS | ALL_ROUTES | ASSET_ROUTES | {"core:api-check"} - def test_editable_tests_do_not_build_wheels(self) -> None: - for target in ("pathfinder:test", "bindings:test", "core:test"): - graph = task_graph(target) - assert set(graph) == {target} + def test_editable_installs_are_first_class_dependencies(self) -> None: + expected_install_deps = { + "pathfinder:install": set(), + "bindings:install": {"pathfinder:install"}, + "core:install": {"bindings:install"}, + } + graph = task_graph("core:install") + assert set(graph) == set(expected_install_deps) + for target, expected in expected_install_deps.items(): + assert {dep["target"] for dep in graph[target].get("deps", [])} == expected script = graph[target]["script"] - assert "pip install -e" in script - assert "pip wheel" not in script - assert "cibuildwheel" not in script + assert "pip install -e ." in script + assert "../cuda_" not in script + + expected_test_graphs = { + "pathfinder:test": {"pathfinder:install", "pathfinder:test"}, + "bindings:test": {"pathfinder:install", "bindings:install", "bindings:test"}, + "core:test": { + "pathfinder:install", + "bindings:install", + "core:install", + "core:test", + }, + } + for target, expected in expected_test_graphs.items(): + graph = task_graph(target) + assert set(graph) == expected + test_script = graph[target]["script"] + assert "pip install" not in test_script + assert all("wheel" not in graph_target for graph_target in graph) + assert all("cibuildwheel" not in task["script"] for task in graph.values()) + + root_test_graph = task_graph("root:test") + assert {dep["target"] for dep in root_test_graph["root:test"]["deps"]} == { + "pathfinder:test", + "bindings:test", + "core:test", + } + assert root_test_graph["root:test"]["options"]["runDepsInParallel"] is True + + benchmark_graph = task_graph("bindings:benchmark") + assert set(benchmark_graph) == { + "pathfinder:install", + "bindings:install", + "bindings:benchmark", + } + assert "pip install" not in benchmark_graph["bindings:benchmark"]["script"] def test_local_core_wheel_builds_current_dependency_chain(self) -> None: assert set(task_graph("core:wheel")) == { From ceaae51ecbe7c3d761c6b63611ec7bfcad0b15d1 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Thu, 20 Aug 2026 02:02:32 -0400 Subject: [PATCH 13/13] Revert Moon CI migration from PR #2467 Moon migration belongs in PR #2659. Restore the pre-Moon selective-CI planner and workflows from 727ef5994e. --- .github/workflows/build-docs.yml | 149 ++---- .github/workflows/build-wheel.yml | 291 +++++----- .github/workflows/ci.yml | 341 +++--------- .github/workflows/test-sdist-linux.yml | 93 ++-- .github/workflows/test-sdist-windows.yml | 84 +-- .github/workflows/test-wheel-linux.yml | 121 +++-- .github/workflows/test-wheel-windows.yml | 123 +++-- .gitignore | 1 - .moon/workspace.yml | 15 - ci/test-matrix.yml | 6 + ci/tools/compute_ci_plan.py | 213 ++++++++ ci/tools/tests/test_compute_ci_plan.py | 181 +++++++ cuda_bindings/moon.yml | 224 -------- cuda_core/moon.yml | 318 ----------- cuda_pathfinder/moon.yml | 117 ---- cuda_python/moon.yml | 194 ------- moon.yml | 111 ---- tests/test_moon_ci.py | 649 ----------------------- toolshed/check_spdx.py | 2 - 19 files changed, 848 insertions(+), 2385 deletions(-) delete mode 100644 .moon/workspace.yml create mode 100644 ci/tools/compute_ci_plan.py create mode 100644 ci/tools/tests/test_compute_ci_plan.py delete mode 100644 cuda_bindings/moon.yml delete mode 100644 cuda_core/moon.yml delete mode 100644 cuda_pathfinder/moon.yml delete mode 100644 cuda_python/moon.yml delete mode 100644 moon.yml delete mode 100644 tests/test_moon_ci.py diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 8f3c58ed67a..7bb70809556 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -18,11 +18,6 @@ on: # - cuda-python # - cuda-pathfinder # - all - targets: - description: "JSON array of exact Moon docs targets. Empty derives targets from component." - required: false - default: "" - type: string git-tag: description: "Target git tag to build docs for" required: false @@ -65,39 +60,6 @@ jobs: fetch-depth: 1 ref: ${{ inputs.git-tag }} - - name: Resolve Moon docs targets - env: - REQUESTED_TARGETS: ${{ inputs.targets }} - COMPONENT: ${{ inputs.component }} - run: | - if [[ -n "$REQUESTED_TARGETS" ]]; then - targets="$REQUESTED_TARGETS" - else - case "$COMPONENT" in - all) targets='["root:docs"]' ;; - cuda-pathfinder) targets='["pathfinder:docs"]' ;; - cuda-bindings) targets='["bindings:docs"]' ;; - cuda-core) targets='["core:docs"]' ;; - cuda-python) targets='["metapackage:docs"]' ;; - *) echo "error: unsupported docs component: $COMPONENT" >&2; exit 1 ;; - esac - fi - jq -e ' - type == "array" and length > 0 and - all(.[]; - . == "root:docs" or - . == "pathfinder:docs" or - . == "bindings:docs" or - . == "core:docs" or - . == "metapackage:docs") - ' <<< "$targets" >/dev/null - echo "DOCS_TARGETS=$(jq -c . <<< "$targets")" >> "$GITHUB_ENV" - if [[ -f .moon/workspace.yml && -f moon.yml ]]; then - echo "DOCS_USE_MOON=true" >> "$GITHUB_ENV" - else - echo "DOCS_USE_MOON=false" >> "$GITHUB_ENV" - fi - - name: Read build CTK version run: | if [[ -f ci/versions.yml ]]; then @@ -132,12 +94,6 @@ jobs: conda config --show-sources conda config --show - - name: Install Moon - if: ${{ env.DOCS_USE_MOON == 'true' }} - run: | - curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash - echo "$HOME/.moon/bin" >> "$GITHUB_PATH" - # WAR: Building the doc currently requires CTK installed (NVIDIA/cuda-python#326,327) - name: Set up mini CTK uses: ./.github/actions/fetch_ctk @@ -176,7 +132,7 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel - path: ./cuda_python/dist + path: . run-id: ${{ inputs.run-id }} github-token: ${{ github.token }} @@ -189,7 +145,7 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel - path: ./cuda_pathfinder/dist + path: ./cuda_pathfinder run-id: ${{ inputs.run-id }} github-token: ${{ github.token }} @@ -244,7 +200,7 @@ jobs: - name: Install all packages run: | - pushd cuda_pathfinder/dist + pushd cuda_pathfinder pip install *.whl popd @@ -258,7 +214,7 @@ jobs: # Subpackages are already installed from CI artifacts above. # --no-deps avoids re-resolving cuda-core from PyPI during tag releases. - pip install --no-deps cuda_python/dist/*.whl + pip install --no-deps cuda_python*.whl # This step sets the PR_NUMBER/BUILD_LATEST/BUILD_PREVIEW env vars. - name: Get PR number @@ -271,79 +227,42 @@ jobs: # create an empty folder for removal use mkdir -p artifacts/empty_docs - - name: Build selected docs - if: ${{ env.DOCS_USE_MOON == 'true' }} - env: - DOCS_BUILD_ARGS: ${{ !inputs.is-release && 'latest-only' || '' }} - run: | - mapfile -t targets < <(jq -r '.[]' <<< "$DOCS_TARGETS") - moon run "${targets[@]}" --upstream deep --downstream none - - # Release workflows may check out tags created before Moon was added. - - name: Build selected docs from a legacy tag - if: ${{ env.DOCS_USE_MOON != 'true' }} + - name: Build all docs + if: ${{ inputs.component == 'all' }} run: | - if [[ "${{ inputs.component }}" == "all" ]]; then - pushd cuda_python/docs - if [[ "${{ inputs.is-release }}" == "false" ]]; then - ./build_all_docs.sh latest-only - else - ./build_all_docs.sh - rm -rf build/html/latest - fi - popd + pushd cuda_python/docs/ + if [[ "${{ inputs.is-release }}" == "false" ]]; then + ./build_all_docs.sh latest-only else - component="${{ inputs.component }}" - component=${component//-/_} - pushd "$component/docs" - if [[ "${{ inputs.is-release }}" == "false" ]]; then - ./build_docs.sh latest-only - else - ./build_docs.sh - rm -rf build/html/latest - fi - popd + ./build_all_docs.sh + # At release time, we don't want to update the latest docs + rm -rf build/html/latest fi + ls -l build + popd + mv cuda_python/docs/build/html/* artifacts/docs/ - - name: Assemble selected docs + - name: Build component docs + if: ${{ inputs.component != 'all' }} run: | - if jq -e 'index("root:docs") != null' <<< "$DOCS_TARGETS" >/dev/null; then - if [[ "${{ inputs.is-release }}" == "true" ]]; then - rm -rf cuda_python/docs/build/html/latest - fi - ls -l cuda_python/docs/build - mv cuda_python/docs/build/html/* artifacts/docs/ - exit 0 + COMPONENT=$(echo "${{ inputs.component }}" | tr '-' '_') + pushd ${COMPONENT}/docs/ + if [[ "${{ inputs.is-release }}" == "false" ]]; then + ./build_docs.sh latest-only + else + ./build_docs.sh + # At release time, we don't want to update the latest docs + rm -rf build/html/latest fi - - while IFS= read -r target; do - case "$target" in - pathfinder:docs) - component=cuda_pathfinder - destination=cuda-pathfinder - ;; - bindings:docs) - component=cuda_bindings - destination=cuda-bindings - ;; - core:docs) - component=cuda_core - destination=cuda-core - ;; - metapackage:docs) - component=cuda_python - destination= - ;; - esac - if [[ "${{ inputs.is-release }}" == "true" ]]; then - rm -rf "$component/docs/build/html/latest" - fi - ls -l "$component/docs/build" - if [[ -n "$destination" ]]; then - mkdir -p "artifacts/docs/$destination" - fi - mv "$component"/docs/build/html/* "artifacts/docs/$destination" - done < <(jq -r '.[]' <<< "$DOCS_TARGETS") + ls -l build + popd + if [[ "${{ inputs.component }}" != "cuda-python" ]]; then + TARGET="${{ inputs.component }}" + mkdir -p artifacts/docs/${TARGET} + else + TARGET="" + fi + mv ${COMPONENT}/docs/build/html/* artifacts/docs/${TARGET} - name: Write rendered docs file list if: ${{ !inputs.is-release && github.ref_name != 'main' && !startsWith(github.ref_name, 'release/') }} diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 2285f8e985d..7e233244843 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -31,16 +31,12 @@ permissions: jobs: build: env: - WHEEL_FOUNDATION_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-foundation']) || '["pathfinder:wheel"]' }} - WHEEL_BINDINGS_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-bindings']) || '["bindings:wheel"]' }} - WHEEL_CONSUMER_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-consumers']) || '["core:wheel","metapackage:wheel"]' }} - WHEEL_MULTI_CTK_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-multi-ctk']) || '["core:wheel"]' }} - WHEEL_FINALIZE_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-finalize']) || '["core:wheel-merge"]' }} - TEST_ASSETS_CURRENT_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-test-assets-current']) || '["bindings:ci-test-assets","core:ci-test-assets"]' }} - TEST_ASSETS_PREVIOUS_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-test-assets-previous']) || '["core:ci-test-binaries"]' }} - TEST_LINUX_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-test-linux']) || '["pathfinder:ci-test-linux"]' }} - TEST_WINDOWS_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-test-windows']) || '["pathfinder:ci-test-windows"]' }} - HOST_PLATFORM: ${{ inputs.host-platform }} + BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} + BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} + BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} + BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} + TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} + TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} BASELINE_RUN_ID: ${{ inputs.workplan != '' && fromJSON(inputs.workplan).baseline.run_id || '' }} BASELINE_SHA: ${{ inputs.workplan != '' && fromJSON(inputs.workplan).baseline.sha || '' }} strategy: @@ -68,63 +64,6 @@ jobs: fetch-depth: 0 filter: blob:none - - name: Install target resolver dependencies - if: ${{ startsWith(inputs.host-platform, 'linux') }} - uses: ./.github/actions/install_unix_deps - with: - dependencies: "jq" - dependent_exes: "jq" - - - name: Resolve Moon phase targets - run: | - for value in \ - "$WHEEL_FOUNDATION_TARGETS" \ - "$WHEEL_BINDINGS_TARGETS" \ - "$WHEEL_CONSUMER_TARGETS" \ - "$WHEEL_MULTI_CTK_TARGETS" \ - "$WHEEL_FINALIZE_TARGETS" \ - "$TEST_ASSETS_CURRENT_TARGETS" \ - "$TEST_ASSETS_PREVIOUS_TARGETS" \ - "$TEST_LINUX_TARGETS" \ - "$TEST_WINDOWS_TARGETS"; do - jq -e 'type == "array" and all(.[]; type == "string")' <<< "$value" >/dev/null - done - - has_target() { - jq -e --arg target "$2" 'index($target) != null' <<< "$1" >/dev/null - } - if [[ "$HOST_PLATFORM" == "win-64" ]]; then - platform_test_targets="$TEST_WINDOWS_TARGETS" - else - platform_test_targets="$TEST_LINUX_TARGETS" - fi - run_test_assets=$(jq -r 'length > 0' <<< "$platform_test_targets") - { - echo "BUILD_PATHFINDER=$(has_target "$WHEEL_FOUNDATION_TARGETS" pathfinder:wheel && echo true || echo false)" - echo "BUILD_BINDINGS=$(has_target "$WHEEL_BINDINGS_TARGETS" bindings:wheel && echo true || echo false)" - echo "BUILD_CORE_CURRENT=$(has_target "$WHEEL_CONSUMER_TARGETS" core:wheel && echo true || echo false)" - echo "BUILD_CORE_PREVIOUS=$(has_target "$WHEEL_MULTI_CTK_TARGETS" core:wheel && echo true || echo false)" - echo "FINALIZE_CORE=$(has_target "$WHEEL_FINALIZE_TARGETS" core:wheel-merge && echo true || echo false)" - if has_target "$WHEEL_CONSUMER_TARGETS" core:wheel || \ - has_target "$WHEEL_MULTI_CTK_TARGETS" core:wheel || \ - has_target "$WHEEL_FINALIZE_TARGETS" core:wheel-merge; then - echo "BUILD_CORE=true" - else - echo "BUILD_CORE=false" - fi - echo "BUILD_PYTHON=$(has_target "$WHEEL_CONSUMER_TARGETS" metapackage:wheel && echo true || echo false)" - echo "TEST_BINDINGS=$(if [[ "$run_test_assets" == "true" ]]; then has_target "$TEST_ASSETS_CURRENT_TARGETS" bindings:ci-test-assets && echo true || echo false; else echo false; fi)" - echo "TEST_CORE_CURRENT=$(if [[ "$run_test_assets" == "true" ]]; then has_target "$TEST_ASSETS_CURRENT_TARGETS" core:ci-test-assets && echo true || echo false; else echo false; fi)" - echo "TEST_CORE_PREVIOUS=$(if [[ "$run_test_assets" == "true" ]]; then has_target "$TEST_ASSETS_PREVIOUS_TARGETS" core:ci-test-binaries && echo true || echo false; else echo false; fi)" - if [[ "$run_test_assets" == "true" ]] && \ - (has_target "$TEST_ASSETS_CURRENT_TARGETS" core:ci-test-assets || \ - has_target "$TEST_ASSETS_PREVIOUS_TARGETS" core:ci-test-binaries); then - echo "TEST_CORE=true" - else - echo "TEST_CORE=false" - fi - } >> "$GITHUB_ENV" - - name: Install latest rapidsai/sccache if: ${{ startsWith(inputs.host-platform, 'linux') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} run: | @@ -160,12 +99,6 @@ jobs: # see https://github.com/actions/setup-python/issues/871 python-version: "3.12" - - name: Install Moon and build tools - run: | - curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash - echo "$HOME/.moon/bin" >> "$GITHUB_PATH" - python -m pip install cibuildwheel twine wheel - - name: Set up MSVC if: ${{ startsWith(inputs.host-platform, 'win') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') }} uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 @@ -205,19 +138,24 @@ jobs: run: | env + - name: Install twine + run: | + pip install twine + # To keep the build workflow simple, all matrix jobs will build a wheel for later use within this workflow. - name: Build and check cuda.pathfinder wheel if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | - mapfile -t targets < <(jq -r '.[]' <<< "$WHEEL_FOUNDATION_TARGETS") - moon run "${targets[@]}" --upstream none --downstream none + pushd cuda_pathfinder + pip wheel -v --no-deps . + popd - name: Download reusable cuda.pathfinder wheel if: ${{ env.BUILD_PATHFINDER != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel - path: cuda_pathfinder/dist + path: cuda_pathfinder github-token: ${{ github.token }} run-id: ${{ env.BASELINE_RUN_ID }} @@ -228,20 +166,20 @@ jobs: else export CHOWN="sudo chown" fi - $CHOWN -R $(whoami) cuda_pathfinder/dist/*.whl - ls -lahR cuda_pathfinder/dist + $CHOWN -R $(whoami) cuda_pathfinder/*.whl + ls -lahR cuda_pathfinder # We only need/want a single pure python wheel, pick linux-64 index 0. # This is what we will use for testing & releasing. - name: Check cuda.pathfinder wheel if: ${{ env.BUILD_PATHFINDER == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} run: | - twine check --strict cuda_pathfinder/dist/*.whl + twine check --strict cuda_pathfinder/*.whl - name: Constrain builds to the local cuda.pathfinder wheel if: ${{ env.BUILD_BINDINGS == 'true' }} run: | - pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) + pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test -f "${pathfinder_wheels[0]}" mkdir -p wheel-constraints @@ -257,7 +195,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: cuda-pathfinder-wheel - path: cuda_pathfinder/dist/*.whl + path: cuda_pathfinder/*.whl if-no-files-found: error - name: Set up mini CTK @@ -270,9 +208,10 @@ jobs: - name: Build cuda.bindings wheel if: ${{ env.BUILD_BINDINGS == 'true' }} - run: | - mapfile -t targets < <(jq -r '.[]' <<< "$WHEEL_BINDINGS_TARGETS") - moon run "${targets[@]}" --upstream none --downstream none + uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 + with: + package-dir: ./cuda_bindings/ + output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} env: CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"' @@ -346,9 +285,9 @@ jobs: twine check --strict ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl - name: Constrain cuda.core to the local cuda.bindings wheel - if: ${{ env.BUILD_CORE_CURRENT == 'true' }} + if: ${{ env.BUILD_CORE == 'true' }} run: | - pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) + pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) bindings_wheels=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-"${BUILD_CUDA_MAJOR}".*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test "${#bindings_wheels[@]}" -eq 1 @@ -374,19 +313,13 @@ jobs: path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl if-no-files-found: error - - name: Build current-context consumer wheels - if: ${{ env.BUILD_CORE_CURRENT == 'true' || (env.BUILD_PYTHON == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64') }} - run: | - targets="$WHEEL_CONSUMER_TARGETS" - if [[ "${{ inputs.host-platform }}" != "linux-64" || "${{ strategy.job-index }}" != "0" ]]; then - targets=$(jq -c 'map(select(. != "metapackage:wheel"))' <<< "$targets") - fi - mapfile -t target_args < <(jq -r '.[]' <<< "$targets") - if (( ${#target_args[@]} != 0 )); then - moon run "${target_args[@]}" --upstream none --downstream none - fi + - name: Build cuda.core wheel + if: ${{ env.BUILD_CORE == 'true' }} + uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 + with: + package-dir: ./cuda_core/ + output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} env: - CUDA_CORE_BUILD_MAJOR: ${{ env.BUILD_CUDA_MAJOR }} CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"' CIBW_BEFORE_BUILD_WINDOWS: 'python -m pip install --upgrade "pip>=25.3" delvewheel' @@ -429,15 +362,15 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.core) - if: ${{ env.BUILD_CORE_CURRENT == 'true' && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_CORE == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_core.json label: "cuda.core" build-step: "Build cuda.core wheel" - - name: List the cuda.core artifacts directory - if: ${{ env.BUILD_CORE_CURRENT == 'true' }} + - name: List the cuda.core artifacts directory and rename + if: ${{ env.BUILD_CORE == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == win* ]]; then export CHOWN=chown @@ -445,6 +378,18 @@ jobs: export CHOWN="sudo chown" fi $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + + # Rename wheel to include CUDA version suffix + mkdir -p "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}" + for wheel in ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl; do + if [[ -f "${wheel}" ]]; then + base_name=$(basename "${wheel}" .whl) + new_name="${base_name}.cu${BUILD_CUDA_MAJOR}.whl" + mv "${wheel}" "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}/${new_name}" + echo "Renamed wheel to: ${new_name}" + fi + done + ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - name: Download reusable cuda.core wheel @@ -456,12 +401,21 @@ jobs: github-token: ${{ github.token }} run-id: ${{ env.BASELINE_RUN_ID }} + # We only need/want a single pure python wheel, pick linux-64 index 0. + - name: Build and check cuda-python wheel + if: ${{ env.BUILD_PYTHON == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} + run: | + pushd cuda_python + pip wheel -v --no-deps . + twine check --strict *.whl + popd + - name: Download reusable cuda-python wheel if: ${{ env.BUILD_PYTHON != 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel - path: cuda_python/dist + path: cuda_python github-token: ${{ github.token }} run-id: ${{ env.BASELINE_RUN_ID }} @@ -473,42 +427,38 @@ jobs: else export CHOWN="sudo chown" fi - $CHOWN -R $(whoami) cuda_python/dist/*.whl - ls -lahR cuda_python/dist + $CHOWN -R $(whoami) cuda_python/*.whl + ls -lahR cuda_python - name: Upload cuda-python build artifacts if: ${{ strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: cuda-python-wheel - path: cuda_python/dist/*.whl + path: cuda_python/*.whl if-no-files-found: error - name: Set up Python id: setup-python2 - if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} # TODO: remove allow-prereleases once 3.15 is officially supported allow-prereleases: ${{ startsWith(matrix.python-version, '3.15') }} - - name: Reinstall build tools for the selected Python - if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} - run: python -m pip install cibuildwheel twine wheel - - name: Enable Scientific Python Nightly Wheels for Python 3.15 - if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true') && startsWith(matrix.python-version, '3.15') }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && startsWith(matrix.python-version, '3.15') }} run: | echo "PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" >> "$GITHUB_ENV" echo "PIP_ONLY_BINARY=numpy" >> "$GITHUB_ENV" - name: verify free-threaded build - if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true') && endsWith(matrix.python-version, 't') }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && endsWith(matrix.python-version, 't') }} run: python -c 'import sys; assert not sys._is_gil_enabled()' - name: Set up Python include paths - if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == linux* ]]; then echo "CPLUS_INCLUDE_PATH=${Python3_ROOT_DIR}/include/python${{ matrix.python-version }}" >> $GITHUB_ENV @@ -519,41 +469,24 @@ jobs: echo "PY_EXT_SUFFIX=$(python -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")" >> $GITHUB_ENV - name: Install cuda.pathfinder (required for next step) - if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} run: | - pip install cuda_pathfinder/dist/*.whl + pip install cuda_pathfinder/*.whl - name: Hide GNU link.exe so Meson finds MSVC link.exe - if: ${{ startsWith(inputs.host-platform, 'win') && (env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true') }} + if: ${{ startsWith(inputs.host-platform, 'win') && (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') }} run: | if [ -f "/c/Program Files/Git/usr/bin/link.exe" ]; then mv "/c/Program Files/Git/usr/bin/link.exe" "/c/Program Files/Git/usr/bin/link.exe.bak" fi - - name: Install wheels for current-context native test assets - if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} - run: | - if [[ "$TEST_BINDINGS" == "true" ]]; then - pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl --group ./cuda_bindings/pyproject.toml:test - fi - if [[ "$TEST_CORE_CURRENT" == "true" ]]; then - pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl - if [[ "$BUILD_CORE_CURRENT" == "true" ]]; then - core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}" -maxdepth 1 -type f -name '*.whl' -print -quit) - else - core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" -maxdepth 1 -type f -name '*.whl' -print -quit) - fi - test -n "$core_wheel" - pip install "$core_wheel" --group ./cuda_core/pyproject.toml:test - fi - - - name: Build current-context native test assets - if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} - env: - CUDA_CORE_BUILD_MAJOR: ${{ env.BUILD_CUDA_MAJOR }} + - name: Build cuda.bindings Cython tests + if: ${{ env.TEST_BINDINGS == 'true' }} run: | - mapfile -t targets < <(jq -r '.[]' <<< "$TEST_ASSETS_CURRENT_TARGETS") - moon run "${targets[@]}" --upstream direct --downstream none + pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl --group ./cuda_bindings/pyproject.toml:test + pushd ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }} + bash build_tests.sh + popd - name: Upload cuda.bindings Cython tests if: ${{ env.TEST_BINDINGS == 'true' }} @@ -563,8 +496,26 @@ jobs: path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }}/test_*${{ env.PY_EXT_SUFFIX }} if-no-files-found: error + - name: Build cuda.core Cython tests + if: ${{ env.TEST_CORE == 'true' }} + run: | + pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl + if ${{ env.BUILD_CORE == 'true' }}; then + core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}" -maxdepth 1 -type f -name '*.whl' -print -quit) + else + core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" -maxdepth 1 -type f -name '*.whl' -print -quit) + fi + if [[ -z "${core_wheel}" ]]; then + echo "No cuda.core wheel found" >&2 + exit 1 + fi + pip install "${core_wheel}" --group ./cuda_core/pyproject.toml:test + pushd ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }} + bash build_tests.sh + popd + - name: Upload cuda.core Cython tests - if: ${{ env.TEST_CORE_CURRENT == 'true' }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests @@ -573,7 +524,7 @@ jobs: # Note: This overwrites CUDA_PATH etc - name: Set up mini CTK - if: ${{ env.BUILD_CORE_PREVIOUS == 'true' || env.TEST_CORE_PREVIOUS == 'true' }} + if: ${{ env.BUILD_CORE == 'true' || env.TEST_CORE == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -581,14 +532,14 @@ jobs: cuda-version: ${{ inputs.prev-cuda-version }} cuda-path: "./cuda_toolkit_prev" - - name: Build previous-context native test assets - if: ${{ env.TEST_CORE_PREVIOUS == 'true' }} + - name: Build cuda.core test binaries + if: ${{ env.TEST_CORE == 'true' }} run: | - mapfile -t targets < <(jq -r '.[]' <<< "$TEST_ASSETS_PREVIOUS_TARGETS") - moon run "${targets[@]}" --upstream direct --downstream none + nvcc --version + python "${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/build_test_binaries.py" - name: Upload cuda.core test binaries - if: ${{ env.TEST_CORE_PREVIOUS == 'true' }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries @@ -599,7 +550,7 @@ jobs: if-no-files-found: error - name: Download cuda.bindings build artifacts from the prior branch - if: ${{ env.BUILD_CORE_PREVIOUS == 'true' }} + if: ${{ env.BUILD_CORE == 'true' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -635,9 +586,9 @@ jobs: rmdir "${OLD_ARTIFACT_DIR}" - name: Constrain previous cuda.core to the downloaded cuda.bindings wheel - if: ${{ env.BUILD_CORE_PREVIOUS == 'true' }} + if: ${{ env.BUILD_CORE == 'true' }} run: | - pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) + pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) bindings_wheels=(cuda_bindings/dist-prev/cuda_bindings-"${BUILD_PREV_CUDA_MAJOR}".*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test "${#bindings_wheels[@]}" -eq 1 @@ -656,13 +607,13 @@ jobs: printf 'cuda-bindings @ %s\n' "${bindings_uri}" } | tee wheel-constraints/cuda-core-prev.txt - - name: Build previous-context cuda.core wheel - if: ${{ env.BUILD_CORE_PREVIOUS == 'true' }} - run: | - mapfile -t targets < <(jq -r '.[]' <<< "$WHEEL_MULTI_CTK_TARGETS") - moon run "${targets[@]}" --upstream none --downstream none + - name: Build cuda.core wheel + if: ${{ env.BUILD_CORE == 'true' }} + uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 + with: + package-dir: ./cuda_core/ + output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} env: - CUDA_CORE_BUILD_MAJOR: ${{ env.BUILD_PREV_CUDA_MAJOR }} CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"' CIBW_BEFORE_BUILD_WINDOWS: 'python -m pip install --upgrade "pip>=25.3" delvewheel' @@ -705,15 +656,15 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.core prev) - if: ${{ env.BUILD_CORE_PREVIOUS == 'true' && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_CORE == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_core_prev.json label: "cuda.core (prev CTK)" build-step: "Build cuda.core wheel" - - name: List the previous-context cuda.core artifacts - if: ${{ env.BUILD_CORE_PREVIOUS == 'true' }} + - name: List the cuda.core artifacts directory and rename + if: ${{ env.BUILD_CORE == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == win* ]]; then export CHOWN=chown @@ -723,14 +674,30 @@ jobs: $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }} ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + # Rename wheel to include CUDA version suffix + mkdir -p "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_PREV_CUDA_MAJOR}" + for wheel in ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl; do + if [[ -f "${wheel}" ]]; then + base_name=$(basename "${wheel}" .whl) + new_name="${base_name}.cu${BUILD_PREV_CUDA_MAJOR}.whl" + mv "${wheel}" "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_PREV_CUDA_MAJOR}/${new_name}" + echo "Renamed wheel to: ${new_name}" + fi + done + + ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + - name: Merge cuda.core wheels - if: ${{ env.FINALIZE_CORE == 'true' }} + if: ${{ env.BUILD_CORE == 'true' }} run: | - mapfile -t targets < <(jq -r '.[]' <<< "$WHEEL_FINALIZE_TARGETS") - moon run "${targets[@]}" --upstream none --downstream none + pip install wheel + python ci/tools/merge_cuda_core_wheels.py \ + "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cu"${BUILD_CUDA_MAJOR}"/cuda_core*.whl \ + "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cu"${BUILD_PREV_CUDA_MAJOR}"/cuda_core*.whl \ + --output-dir "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" - name: Check cuda.core wheel - if: ${{ env.FINALIZE_CORE == 'true' }} + if: ${{ env.BUILD_CORE == 'true' }} run: | twine check --strict ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a65710331df..2a1068fb7b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,9 +80,25 @@ jobs: echo "doc_only=${doc_only}" >> "$GITHUB_OUTPUT" echo "base_ref=${base_ref}" >> "$GITHUB_OUTPUT" - # Moon owns file ownership, package impact, and the task graph. This job only - # establishes whether trusted artifacts may be reused and groups Moon's - # directly affected tasks by semantic CI phase for the heterogeneous runners. + # Detect which packages were touched by the PR so downstream build and test + # jobs can avoid rebuilding/retesting packages unaffected by the change. + # See issue #299. + # + # Dependency graph (verified in pyproject.toml files): + # cuda_pathfinder -> (no internal deps) + # cuda_bindings -> cuda_pathfinder + # cuda_core -> cuda_pathfinder, cuda_bindings + # cuda_python -> cuda_pathfinder, cuda_bindings, cuda_core (meta package) + # + # A change to cuda_pathfinder (or shared infra) forces a rebuild of every + # downstream module. A change to cuda_bindings forces rebuild of cuda_core. + # A change to cuda_core alone skips rebuilding/retesting cuda_bindings and + # cuda_pathfinder, but still retests the downstream cuda-python metapackage. + # Shared build/orchestration changes run the full pipeline; test-only CI + # infrastructure runs every test suite without rebuilding package wheels. + # On push to main, tag refs, schedule, or workflow_dispatch events we + # unconditionally run everything because there is no meaningful "changed + # paths" baseline for those events. detect-changes: runs-on: ubuntu-latest needs: should-skip @@ -100,11 +116,6 @@ jobs: fetch-depth: 0 filter: blob:none - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - enable-cache: false - - name: Resolve PR merge base id: merge-base if: ${{ startsWith(github.ref_name, 'pull-request/') }} @@ -150,7 +161,7 @@ jobs: --event push \ --workflow ci.yml \ --status success \ - --limit 100 \ + --limit 1 \ --json databaseId,headSha); then unavailable fi @@ -158,26 +169,21 @@ jobs: # Reuse only artifacts produced from the exact commit used as the # PR diff base. Using the latest base-branch run is unsafe for a PR # that was opened before newer changes landed on that branch. - if [[ $(jq 'length' <<< "$runs") -ne 1 ]]; then - unavailable - fi run_id=$(jq -r '.[0].databaseId // empty' <<< "$runs") run_sha=$(jq -r '.[0].headSha // empty' <<< "$runs") if [[ -z "${run_id}" || "${run_sha}" != "${merge_base}" ]]; then unavailable fi - if ! artifacts=$(gh api \ + if ! artifact_names=$(gh api \ "repos/${{ github.repository }}/actions/runs/${run_id}/artifacts?per_page=100" \ --paginate \ - --jq '.artifacts[] | {name, expired}'); then + --jq '.artifacts[] | select(.expired == false) | .name'); then unavailable fi has_artifact() { - jq -se --arg name "$1" \ - '[.[] | select(.name == $name)] | length == 1 and .[0].expired == false' \ - <<< "$artifacts" >/dev/null + grep -Fxq "$1" <<< "$artifact_names" } missing=() @@ -189,17 +195,20 @@ jobs: if ! python_versions=$(yq -r '.jobs.build.strategy.matrix."python-version"[]' .github/workflows/build-wheel.yml); then unavailable fi - if [[ -z "${python_versions}" ]]; then + if ! platforms=$(yq -r '.platforms[]' ci/test-matrix.yml); then + unavailable + fi + if [[ -z "${python_versions}" || -z "${platforms}" ]]; then unavailable fi while IFS= read -r python_version; do python=${python_version//./} - for platform in linux-64 linux-aarch64 win-64; do + while IFS= read -r platform; do binding="cuda-bindings-python${python}-cuda${cuda_version}-${platform}-${merge_base}" core="cuda-core-python${python}-${platform}-${merge_base}" has_artifact "$binding" || missing+=("$binding") has_artifact "$core" || missing+=("$core") - done + done <<< "${platforms}" done <<< "${python_versions}" if (( ${#missing[@]} != 0 )); then @@ -213,104 +222,19 @@ jobs: echo "Reusable artifacts: run \`${run_id}\` at \`${merge_base}\` on \`${BASE_REF}\`." } >> "$GITHUB_STEP_SUMMARY" - - name: Compute Moon CI workplan + - name: Test CI workplan planner + run: python3 -m unittest ci/tools/tests/test_compute_ci_plan.py + + - name: Compute CI workplan id: workplan env: MERGE_BASE: ${{ steps.merge-base.outputs.sha }} BASELINE_RUN_ID: ${{ steps.baseline.outputs.run_id }} - DOC_ONLY: ${{ needs.should-skip.outputs.doc-only }} run: | set -euo pipefail - curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash - export PATH="$HOME/.moon/bin:$PATH" - uvx --from pytest pytest -q tests/test_moon_ci.py - - semantic_tags='[ - "ci-wheel-foundation", - "ci-wheel-bindings", - "ci-wheel-consumers", - "ci-wheel-multi-ctk", - "ci-wheel-finalize", - "ci-sdist-foundation", - "ci-sdist-bindings", - "ci-sdist-consumers", - "ci-test-assets-current", - "ci-test-assets-previous", - "ci-test-linux", - "ci-test-windows", - "ci-docs", - "ci-api", - "ci-ignore", - "ci-force-all" - ]' - - visible_tasks() { - jq -c '[ - .tasks - | to_entries[] as $project - | $project.value - | to_entries[] - | select(.value.options.internal != true) - | { - target: "\($project.key):\(.key)", - tags: (.value.tags // []) - } - ]' - } - - force_all=false - selected='[]' - if [[ -z "${MERGE_BASE}" || -z "${BASELINE_RUN_ID}" ]]; then - force_all=true - else - git diff --no-renames --name-only -z "${MERGE_BASE}"...HEAD > changed-paths - while IFS= read -r -d '' path; do - [[ -n "${path}" ]] || continue - result=$(printf '%s\n' "${path}" | moon query tasks --affected stdin --upstream none --downstream none) - owned=$(visible_tasks <<< "${result}") - if jq -e 'length == 0 or any(.[]; .target == "root:ci-fallback")' <<< "${owned}" >/dev/null; then - force_all=true - break - fi - selected=$(jq -cn \ - --argjson current "${selected}" \ - --argjson next "${owned}" \ - '$current + $next | unique_by(.target)') - done < changed-paths - fi - - if [[ "${force_all}" == "true" ]]; then - selected=$(moon query tasks | visible_tasks) - baseline_run_id="" - baseline_sha="" - else - baseline_run_id="${BASELINE_RUN_ID}" - baseline_sha="${MERGE_BASE}" - fi - - # Preserve the established [doc-only] behavior: build the complete - # documentation site even when the changed paths do not own docs. - if [[ "${DOC_ONLY}" == "true" ]]; then - docs=$(moon query tasks | visible_tasks | jq -c '[.[] | select(.tags | index("ci-docs"))]') - selected=$(jq -cn \ - --argjson current "${selected}" \ - --argjson docs "${docs}" \ - '$current + $docs | unique_by(.target)') - fi - - targets=$(jq -cn \ - --argjson tags "${semantic_tags}" \ - --argjson selected "${selected}" \ - 'reduce $tags[] as $tag ({}; - .[$tag] = ([$selected[] - | select(.tags | index($tag)) - | .target] | unique | sort))') - workplan=$(jq -cn \ - --argjson targets "${targets}" \ - --arg merge_base "${MERGE_BASE}" \ - --arg baseline_run_id "${baseline_run_id}" \ - --arg baseline_sha "${baseline_sha}" \ - '{targets: $targets, merge_base: $merge_base, baseline: {run_id: $baseline_run_id, sha: $baseline_sha}}') + workplan=$(python3 ci/tools/compute_ci_plan.py \ + --merge-base "$MERGE_BASE" \ + --baseline-run-id "$BASELINE_RUN_ID") echo "workplan=$workplan" >> "$GITHUB_OUTPUT" { echo @@ -324,7 +248,7 @@ jobs: name: API check (cuda_core vs. latest release) if: >- ${{ !fromJSON(needs.should-skip.outputs.skip) && - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-api'][0] }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks }} runs-on: ubuntu-latest needs: - should-skip @@ -362,28 +286,20 @@ jobs: git fetch --depth=1 --filter=blob:none origin \ "refs/tags/${{ steps.latest-tag.outputs.tag }}:refs/tags/${{ steps.latest-tag.outputs.tag }}" - - name: Install Moon 2.5.1 - shell: bash --noprofile --norc -euo pipefail {0} - run: | - curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash - echo "$HOME/.moon/bin" >> "$GITHUB_PATH" - - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - enable-cache: false - - name: Check cuda_core public API - env: - CUDA_CORE_API_REF: ${{ steps.latest-tag.outputs.tag }} - run: moon run core:api-check --upstream none --downstream none + id: griffe + uses: ./.github/actions/griffe-api-check + with: + package-name: cuda.core + package-dir: cuda_core + merge-base: ${{ steps.latest-tag.outputs.tag }} api-check-core-vs-base: name: API check (cuda_core vs. merge base) if: >- ${{ startsWith(github.ref_name, 'pull-request/') && !fromJSON(needs.should-skip.outputs.skip) && - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-api'][0] }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks }} runs-on: ubuntu-latest needs: - should-skip @@ -403,21 +319,13 @@ jobs: git fetch --depth=1 --filter=blob:none origin \ "${{ fromJSON(needs.detect-changes.outputs.workplan).merge_base }}" - - name: Install Moon 2.5.1 - shell: bash --noprofile --norc -euo pipefail {0} - run: | - curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash - echo "$HOME/.moon/bin" >> "$GITHUB_PATH" - - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - enable-cache: false - - name: Check cuda_core public API - env: - CUDA_CORE_API_REF: ${{ fromJSON(needs.detect-changes.outputs.workplan).merge_base }} - run: moon run core:api-check --upstream none --downstream none + id: griffe + uses: ./.github/actions/griffe-api-check + with: + package-name: cuda.core + package-dir: cuda_core + merge-base: ${{ fromJSON(needs.detect-changes.outputs.workplan).merge_base }} # NOTE: Build jobs are intentionally split by platform rather than using a single # matrix. This lets each test job consume its platform-specific artifacts as @@ -437,21 +345,7 @@ jobs: host-platform: - linux-64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: >- - ${{ github.repository_owner == 'nvidia' && - !fromJSON(needs.should-skip.outputs.skip) && - (fromJSON(needs.should-skip.outputs.doc-only) || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-foundation'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-bindings'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-consumers'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-multi-ctk'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-finalize'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-foundation'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-bindings'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-consumers'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-windows'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-docs'][0]) }} + if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) }} permissions: actions: read contents: read @@ -478,12 +372,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-foundation'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-bindings'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-consumers'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-multi-ctk'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-finalize'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux'][0]) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }} permissions: actions: read contents: read @@ -510,15 +399,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-foundation'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-bindings'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-consumers'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-multi-ctk'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-finalize'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-foundation'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-bindings'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-consumers'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-windows'][0]) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows }} permissions: actions: read contents: read @@ -546,9 +427,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-foundation'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-bindings'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-consumers'][0]) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests }} permissions: actions: read contents: read @@ -557,7 +436,7 @@ jobs: with: host-platform: linux-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See test-sdist-linux for why sdist test jobs are split by platform. test-sdist-windows: @@ -571,9 +450,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-foundation'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-bindings'][0] || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-consumers'][0]) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests }} permissions: actions: read contents: read @@ -582,7 +459,7 @@ jobs: with: host-platform: win-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # NOTE: Test jobs are split by platform for the same reason as build jobs (see # build-linux-64). Keep these job definitions textually identical except for: @@ -597,9 +474,8 @@ jobs: - linux-64 name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && - !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux'][0] }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }} permissions: actions: read contents: read # This is required for actions/checkout @@ -615,7 +491,7 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux']) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See test-linux-64 for why test jobs are split by platform. test-linux-aarch64: @@ -626,9 +502,8 @@ jobs: - linux-aarch64 name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && - !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux'][0] }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }} permissions: actions: read contents: read # This is required for actions/checkout @@ -645,7 +520,7 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux']) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See test-linux-64 for why test jobs are split by platform. test-windows: @@ -656,9 +531,8 @@ jobs: - win-64 name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && - !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-windows'][0] }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows }} permissions: actions: read contents: read # This is required for actions/checkout @@ -675,14 +549,11 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-windows']) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} doc: name: Docs - if: ${{ github.repository_owner == 'nvidia' && - !fromJSON(needs.should-skip.outputs.skip) && - (fromJSON(needs.should-skip.outputs.doc-only) || - fromJSON(needs.detect-changes.outputs.workplan).targets['ci-docs'][0]) }} + if: ${{ github.repository_owner == 'nvidia' }} # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: id-token: write @@ -690,14 +561,11 @@ jobs: pull-requests: write needs: - ci-vars - - should-skip - - detect-changes - build-linux-64 secrets: inherit uses: ./.github/workflows/build-docs.yml with: is-release: ${{ github.ref_type == 'tag' }} - targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets['ci-docs']) }} precommit-windows: name: Pre-commit on Windows @@ -755,7 +623,6 @@ jobs: - name: Exit env: NEEDS_JSON: ${{ toJSON(needs) }} - WORKPLAN: ${{ needs.detect-changes.outputs.workplan }} run: | # GitHub treats `result == 'skipped'` as success for required # status checks (see CCCL gate comment + cccl#605). The previous @@ -772,22 +639,10 @@ jobs: fi doc_only="${{ needs.should-skip.outputs.doc-only }}" - wheel_selected=$(jq -r '([ - .targets["ci-wheel-foundation"][], - .targets["ci-wheel-bindings"][], - .targets["ci-wheel-consumers"][], - .targets["ci-wheel-multi-ctk"][], - .targets["ci-wheel-finalize"][] - ] | length > 0)' <<< "$WORKPLAN") - sdist_selected=$(jq -r '([ - .targets["ci-sdist-foundation"][], - .targets["ci-sdist-bindings"][], - .targets["ci-sdist-consumers"][] - ] | length > 0)' <<< "$WORKPLAN") - linux_selected=$(jq -r '.targets["ci-test-linux"] | length > 0' <<< "$WORKPLAN") - windows_selected=$(jq -r '.targets["ci-test-windows"] | length > 0' <<< "$WORKPLAN") - docs_selected=$(jq -r '.targets["ci-docs"] | length > 0' <<< "$WORKPLAN") - run_core_api_check=$(jq -r '.targets["ci-api"] | length > 0' <<< "$WORKPLAN") + linux_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux || false }}" + windows_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows || false }}" + build_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests || false }}" + run_core_api_check="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks || false }}" is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { @@ -800,62 +655,38 @@ jobs: fi } - # Control jobs and Windows pre-commit always run. + # Control jobs, the universal linux build, docs, and Windows + # pre-commit checks always run. check_result "ci-vars" "success" check_result "should-skip" "success" check_result "detect-changes" "success" + check_result "build-linux-64" "success" + check_result "doc" "success" check_result "precommit-windows" "success" - # Build jobs copy forward the complete trusted artifact set whenever - # downstream work needs it, even if no package wheel is rebuilt. - expected="skipped" - if [[ "$doc_only" == "true" || "$wheel_selected" == "true" || - "$sdist_selected" == "true" || "$linux_selected" == "true" || - "$windows_selected" == "true" || "$docs_selected" == "true" ]]; then - expected="success" - fi - check_result "build-linux-64" "$expected" - - expected="skipped" - if [[ "$doc_only" != "true" && - ( "$wheel_selected" == "true" || "$linux_selected" == "true" ) ]]; then - expected="success" + # Optional platform builds and wheel tests share the platform plan. + linux_expected="skipped" + if [[ "$doc_only" != "true" && "$linux_selected" == "true" ]]; then + linux_expected="success" fi - check_result "build-linux-aarch64" "$expected" - - expected="skipped" - if [[ "$doc_only" != "true" && - ( "$wheel_selected" == "true" || "$sdist_selected" == "true" || - "$windows_selected" == "true" ) ]]; then - expected="success" + windows_expected="skipped" + if [[ "$doc_only" != "true" && "$windows_selected" == "true" ]]; then + windows_expected="success" fi - check_result "build-windows" "$expected" + check_result "build-linux-aarch64" "$linux_expected" + check_result "build-windows" "$windows_expected" + # Sdist tests follow build selection; wheel tests follow the platform plan. expected="skipped" - if [[ "$doc_only" != "true" && "$sdist_selected" == "true" ]]; then + if [[ "$doc_only" != "true" && "$build_selected" == "true" ]]; then expected="success" fi check_result "test-sdist-linux" "$expected" check_result "test-sdist-windows" "$expected" - expected="skipped" - if [[ "$doc_only" != "true" && "$linux_selected" == "true" ]]; then - expected="success" - fi - check_result "test-linux-64" "$expected" - check_result "test-linux-aarch64" "$expected" - - expected="skipped" - if [[ "$doc_only" != "true" && "$windows_selected" == "true" ]]; then - expected="success" - fi - check_result "test-windows" "$expected" - - expected="skipped" - if [[ "$doc_only" == "true" || "$docs_selected" == "true" ]]; then - expected="success" - fi - check_result "doc" "$expected" + check_result "test-linux-64" "$linux_expected" + check_result "test-linux-aarch64" "$linux_expected" + check_result "test-windows" "$windows_expected" # API compatibility checks run for cuda_core source changes and for # conservative full runs when reusable base artifacts are unavailable. diff --git a/.github/workflows/test-sdist-linux.yml b/.github/workflows/test-sdist-linux.yml index 00722002da8..ba7cdfc6ef1 100644 --- a/.github/workflows/test-sdist-linux.yml +++ b/.github/workflows/test-sdist-linux.yml @@ -11,8 +11,8 @@ on: cuda-version: required: true type: string - targets: - description: JSON object keyed by semantic Moon tags. An empty value builds everything. + workplan: + description: JSON workplan. An empty value builds everything. required: false default: "" type: string @@ -28,8 +28,12 @@ permissions: jobs: test-sdist: name: Test sdist builds + if: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).jobs.sdist_tests }} env: - SEMANTIC_TARGETS: ${{ inputs.targets != '' && inputs.targets || '{"ci-sdist-foundation":["pathfinder:sdist"],"ci-sdist-bindings":["bindings:sdist"],"ci-sdist-consumers":["core:sdist","metapackage:sdist"]}' }} + BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} + BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} + BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} + BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} timeout-minutes: 60 runs-on: linux-amd64-cpu8 steps: @@ -41,59 +45,26 @@ jobs: fetch-depth: 0 filter: blob:none - - name: Install target resolver dependencies - uses: ./.github/actions/install_unix_deps - with: - dependencies: "jq" - dependent_exes: "jq" - - - name: Resolve Moon sdist targets - run: | - jq -e ' - . as $root | - type == "object" and - all(["ci-sdist-foundation", "ci-sdist-bindings", "ci-sdist-consumers"][]; - . as $tag | - (($root[$tag] // []) | type == "array" and all(.[]; type == "string"))) - ' <<< "$SEMANTIC_TARGETS" >/dev/null - foundation=$(jq -c '.["ci-sdist-foundation"] // []' <<< "$SEMANTIC_TARGETS") - bindings=$(jq -c '.["ci-sdist-bindings"] // []' <<< "$SEMANTIC_TARGETS") - consumers=$(jq -c '.["ci-sdist-consumers"] // []' <<< "$SEMANTIC_TARGETS") - all_targets=$(jq -cn \ - --argjson foundation "$foundation" \ - --argjson bindings "$bindings" \ - --argjson consumers "$consumers" \ - '$foundation + $bindings + $consumers') - has_target() { - jq -e --arg target "$1" 'index($target) != null' <<< "$all_targets" >/dev/null - } - { - echo "SDIST_FOUNDATION_TARGETS=$foundation" - echo "SDIST_BINDINGS_TARGETS=$bindings" - echo "SDIST_CONSUMER_TARGETS=$consumers" - echo "BUILD_PATHFINDER=$(has_target pathfinder:sdist && echo true || echo false)" - echo "BUILD_BINDINGS=$(has_target bindings:sdist && echo true || echo false)" - echo "BUILD_CORE=$(has_target core:sdist && echo true || echo false)" - echo "BUILD_PYTHON=$(has_target metapackage:sdist && echo true || echo false)" - } >> "$GITHUB_ENV" - - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" - - name: Install Moon and build tools - run: | - curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash - echo "$HOME/.moon/bin" >> "$GITHUB_PATH" - python -m pip install "pip>=25.3" build + - name: Install build tools + run: python -m pip install "pip>=25.3" build # Pure Python packages -- no CTK needed. - - name: Build foundation sdists and wheels-from-sdists + - name: Build cuda.pathfinder sdist and wheel-from-sdist if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | - mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_FOUNDATION_TARGETS") - moon run "${targets[@]}" --upstream none --downstream none + python -m build --sdist cuda_pathfinder/ + pip wheel --no-deps --wheel-dir cuda_pathfinder/dist cuda_pathfinder/dist/*.tar.gz + + - name: Build cuda-python sdist and wheel-from-sdist + if: ${{ env.BUILD_PYTHON == 'true' }} + run: | + python -m build --sdist cuda_python/ + pip wheel --no-deps --wheel-dir cuda_python/dist cuda_python/dist/*.tar.gz - name: Download cuda.pathfinder wheel if: ${{ env.BUILD_PATHFINDER != 'true' && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} @@ -147,7 +118,7 @@ jobs: # cuda_bindings/setup.py parses CUDA headers at import time, so CUDA_PATH # (set by fetch_ctk) must be available for both sdist and wheel builds. - - name: Build bindings sdists and wheels-from-sdists + - name: Build cuda.bindings sdist and wheel-from-sdist if: ${{ env.BUILD_BINDINGS == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) @@ -155,8 +126,8 @@ jobs: export CXX="sccache c++" export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-bindings.txt" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_BINDINGS_TARGETS") - moon run "${targets[@]}" --upstream none --downstream none + python -m build --sdist cuda_bindings/ + pip wheel --no-deps --wheel-dir cuda_bindings/dist cuda_bindings/dist/*.tar.gz - name: Download cuda.bindings wheel if: ${{ env.BUILD_BINDINGS != 'true' && env.BUILD_CORE == 'true' }} @@ -186,19 +157,17 @@ jobs: # cuda_core sdist delegates to setuptools (no CTK needed), but # wheel-from-sdist needs CTK and cuda-bindings (dynamic build dep via # get_requires_for_build_wheel in build_hooks.py). - - name: Build consumer sdists and wheels-from-sdists - if: ${{ env.BUILD_CORE == 'true' || env.BUILD_PYTHON == 'true' }} + - name: Build cuda.core sdist and wheel-from-sdist + if: ${{ env.BUILD_CORE == 'true' }} run: | - if [[ "$BUILD_CORE" == "true" ]]; then - export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) - export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" - export CC="sccache cc" - export CXX="sccache c++" - export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-core.txt" - export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - fi - mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_CONSUMER_TARGETS") - moon run "${targets[@]}" --upstream none --downstream none + export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) + export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" + export CC="sccache cc" + export CXX="sccache c++" + export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-core.txt" + export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" + python -m build --sdist cuda_core/ + pip wheel --no-deps --wheel-dir cuda_core/dist cuda_core/dist/*.tar.gz - name: Show sccache stats if: ${{ always() && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} diff --git a/.github/workflows/test-sdist-windows.yml b/.github/workflows/test-sdist-windows.yml index 623ce85fec1..a0594800eba 100644 --- a/.github/workflows/test-sdist-windows.yml +++ b/.github/workflows/test-sdist-windows.yml @@ -17,8 +17,8 @@ on: cuda-version: required: true type: string - targets: - description: JSON object keyed by semantic Moon tags. An empty value builds everything. + workplan: + description: JSON workplan. An empty value builds everything. required: false default: "" type: string @@ -34,8 +34,12 @@ permissions: jobs: test-sdist: name: Test sdist builds + if: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).jobs.sdist_tests }} env: - SEMANTIC_TARGETS: ${{ inputs.targets != '' && inputs.targets || '{"ci-sdist-foundation":["pathfinder:sdist"],"ci-sdist-bindings":["bindings:sdist"],"ci-sdist-consumers":["core:sdist","metapackage:sdist"]}' }} + BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} + BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} + BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} + BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} timeout-minutes: 60 runs-on: windows-2022 steps: @@ -47,37 +51,6 @@ jobs: fetch-depth: 0 filter: blob:none - - name: Resolve Moon sdist targets - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - jq -e ' - . as $root | - type == "object" and - all(["ci-sdist-foundation", "ci-sdist-bindings", "ci-sdist-consumers"][]; - . as $tag | - (($root[$tag] // []) | type == "array" and all(.[]; type == "string"))) - ' <<< "$SEMANTIC_TARGETS" >/dev/null - foundation=$(jq -c '.["ci-sdist-foundation"] // []' <<< "$SEMANTIC_TARGETS") - bindings=$(jq -c '.["ci-sdist-bindings"] // []' <<< "$SEMANTIC_TARGETS") - consumers=$(jq -c '.["ci-sdist-consumers"] // []' <<< "$SEMANTIC_TARGETS") - all_targets=$(jq -cn \ - --argjson foundation "$foundation" \ - --argjson bindings "$bindings" \ - --argjson consumers "$consumers" \ - '$foundation + $bindings + $consumers') - has_target() { - jq -e --arg target "$1" 'index($target) != null' <<< "$all_targets" >/dev/null - } - { - echo "SDIST_FOUNDATION_TARGETS=$foundation" - echo "SDIST_BINDINGS_TARGETS=$bindings" - echo "SDIST_CONSUMER_TARGETS=$consumers" - echo "BUILD_PATHFINDER=$(has_target pathfinder:sdist && echo true || echo false)" - echo "BUILD_BINDINGS=$(has_target bindings:sdist && echo true || echo false)" - echo "BUILD_CORE=$(has_target core:sdist && echo true || echo false)" - echo "BUILD_PYTHON=$(has_target metapackage:sdist && echo true || echo false)" - } >> "$GITHUB_ENV" - - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: @@ -87,18 +60,21 @@ jobs: if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 - - name: Install Moon and build tools - run: | - curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash - echo "$HOME/.moon/bin" >> "$GITHUB_PATH" - python -m pip install "pip>=25.3" build + - name: Install build tools + run: python -m pip install "pip>=25.3" build # Pure Python packages -- no CTK needed. - - name: Build foundation sdists and wheels-from-sdists + - name: Build cuda.pathfinder sdist and wheel-from-sdist if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | - mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_FOUNDATION_TARGETS") - moon run "${targets[@]}" --upstream none --downstream none + python -m build --sdist cuda_pathfinder/ + pip wheel --no-deps --wheel-dir cuda_pathfinder/dist cuda_pathfinder/dist/*.tar.gz + + - name: Build cuda-python sdist and wheel-from-sdist + if: ${{ env.BUILD_PYTHON == 'true' }} + run: | + python -m build --sdist cuda_python/ + pip wheel --no-deps --wheel-dir cuda_python/dist cuda_python/dist/*.tar.gz - name: Download cuda.pathfinder wheel if: ${{ env.BUILD_PATHFINDER != 'true' && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} @@ -132,14 +108,14 @@ jobs: # (set by fetch_ctk) must be available for both sdist and wheel builds. # Constraint paths are passed as native Windows paths because the pip # subprocesses run outside Git Bash. - - name: Build bindings sdists and wheels-from-sdists + - name: Build cuda.bindings sdist and wheel-from-sdist if: ${{ env.BUILD_BINDINGS == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-bindings.txt")" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_BINDINGS_TARGETS") - moon run "${targets[@]}" --upstream none --downstream none + python -m build --sdist cuda_bindings/ + pip wheel --no-deps --wheel-dir cuda_bindings/dist cuda_bindings/dist/*.tar.gz - name: Download cuda.bindings wheel if: ${{ env.BUILD_BINDINGS != 'true' && env.BUILD_CORE == 'true' }} @@ -169,14 +145,12 @@ jobs: # cuda_core sdist delegates to setuptools (no CTK needed), but # wheel-from-sdist needs CTK and cuda-bindings (dynamic build dep via # get_requires_for_build_wheel in build_hooks.py). - - name: Build consumer sdists and wheels-from-sdists - if: ${{ env.BUILD_CORE == 'true' || env.BUILD_PYTHON == 'true' }} + - name: Build cuda.core sdist and wheel-from-sdist + if: ${{ env.BUILD_CORE == 'true' }} run: | - if [[ "$BUILD_CORE" == "true" ]]; then - export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) - export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" - export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-core.txt")" - export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - fi - mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_CONSUMER_TARGETS") - moon run "${targets[@]}" --upstream none --downstream none + export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) + export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" + export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-core.txt")" + export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" + python -m build --sdist cuda_core/ + pip wheel --no-deps --wheel-dir cuda_core/dist cuda_core/dist/*.tar.gz diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index dd99ae55384..814e4e756b0 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -22,8 +22,8 @@ on: nruns: type: number default: 1 - targets: - description: JSON array of exact Moon Linux test route targets. An empty value tests everything. + workplan: + description: JSON workplan. An empty value tests everything. type: string default: "" run-id: @@ -99,7 +99,10 @@ jobs: test: env: - MOON_TARGETS: ${{ inputs.targets != '' && inputs.targets || '["pathfinder:ci-test-linux","bindings:ci-test-linux","core:ci-test-linux","metapackage:ci-test-linux"]' }} + TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }} + TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} + TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} + TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }} name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }}${{ matrix.FLAVOR && format(', {0}', matrix.FLAVOR) || '' }}${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }} timeout-minutes: 60 needs: compute-matrix @@ -140,19 +143,6 @@ jobs: dependencies: "jq wget libgl1 libegl1 g++ util-linux" dependent_exes: "jq wget" - - name: Resolve Moon test targets - run: | - jq -e 'type == "array" and all(.[]; type == "string")' <<< "$MOON_TARGETS" >/dev/null - has_target() { - jq -e --arg target "$1" 'index($target) != null' <<< "$MOON_TARGETS" >/dev/null - } - { - echo "TEST_PATHFINDER=$(has_target pathfinder:ci-test-linux && echo true || echo false)" - echo "TEST_BINDINGS=$(has_target bindings:ci-test-linux && echo true || echo false)" - echo "TEST_CORE=$(has_target core:ci-test-linux && echo true || echo false)" - echo "TEST_PYTHON=$(has_target metapackage:ci-test-linux && echo true || echo false)" - } >> "$GITHUB_ENV" - - name: Install GPU driver if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} env: @@ -257,24 +247,11 @@ jobs: rmdir cuda-python-wheel fi - - name: Stage pure wheels for Moon - if: ${{ inputs.test-mode == 'standard' }} - run: | - mkdir -p cuda_pathfinder/dist cuda_python/dist - pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) - if (( ${#pathfinder_wheels[@]} != 0 )) && [[ -f "${pathfinder_wheels[0]}" ]]; then - cp "${pathfinder_wheels[@]}" cuda_pathfinder/dist/ - fi - python_wheels=(cuda_python-*.whl) - if (( ${#python_wheels[@]} != 0 )) && [[ -f "${python_wheels[0]}" ]]; then - cp "${python_wheels[@]}" cuda_python/dist/ - fi - - name: Display structure of downloaded cuda-python artifacts if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'published' }} run: | pwd - find cuda_pathfinder cuda_python -maxdepth 2 -type f -name '*.whl' -print + ls -lah cuda_python*.whl cuda_pathfinder/ - name: Display structure of downloaded cuda.bindings artifacts if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && @@ -299,7 +276,7 @@ jobs: ls -lahR $CUDA_BINDINGS_CYTHON_TESTS_DIR - name: Download cuda.core build artifacts - if: ${{ env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} @@ -308,7 +285,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core build artifacts - if: ${{ env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} + if: ${{ env.TEST_CORE == 'true' }} run: | pwd ls -lahR $CUDA_CORE_ARTIFACTS_DIR @@ -353,12 +330,6 @@ jobs: # we use self-hosted runners on which setup-python behaves weirdly (Python include can't be found)... AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache" - - name: Install Moon - if: ${{ inputs.test-mode == 'standard' }} - run: | - curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash - echo "$HOME/.moon/bin" >> "$GITHUB_PATH" - - name: Enable Scientific Python Nightly Wheels for Python 3.15 if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && startsWith(matrix.PY_VER, '3.15') }} @@ -390,24 +361,74 @@ jobs: - name: Set up test repetition on nightly runs run: echo "PYTEST_ADDOPTS=\"--count=${{ inputs.nruns }}\"" >> "$GITHUB_ENV" - # ── Standard test route (skipped for nightly modes) ── - - name: Run selected installed-wheel tests - if: ${{ inputs.test-mode == 'standard' }} + # ── Standard test steps (skipped for nightly modes) ── + - name: Run cuda.pathfinder tests with see_what_works + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} + env: + CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works + CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works + CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: see_what_works + run: run-tests pathfinder + + - name: Run cuda.bindings tests + if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + run: run-tests bindings + + - name: Run cuda.bindings benchmarks (smoke test) + if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} run: | - targets="$MOON_TARGETS" - if [[ "$SKIP_CUDA_BINDINGS_TEST" != "0" ]]; then - targets=$(jq -c 'map(select(. != "bindings:ci-test-linux"))' <<< "$targets") - fi - if [[ "$BINDINGS_SOURCE" != "main" ]]; then - targets=$(jq -c 'map(select(. != "metapackage:ci-test-linux"))' <<< "$targets") + pip install pyperf + pushd benchmarks/cuda_bindings + python run_pyperf.py --debug-single-value + popd + + - name: Run cuda.core tests + if: ${{ inputs.test-mode == 'standard' && env.TEST_CORE == 'true' }} + env: + CUDA_VER: ${{ matrix.CUDA_VER }} + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + run: run-tests core + + - name: Ensure cuda-python installable + if: ${{ inputs.test-mode == 'standard' && env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} + run: | + # Package suites install their own dependencies. A metapackage-only + # run has no preceding suite, so install the exact local internal + # wheels in one transaction while resolving released dependencies + # such as cuda-core from the package index. + if ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }}; then + dependency_args=(--no-deps) + else + dependency_args=( + ./cuda_pathfinder/cuda_pathfinder-*.whl + "${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-*.whl + ) fi - mapfile -t target_args < <(jq -r '.[]' <<< "$targets") - if (( ${#target_args[@]} != 0 )); then - moon run "${target_args[@]}" --upstream direct --downstream none + python_requirements=(cuda_python*.whl) + if [[ "${{ matrix.LOCAL_CTK }}" != 1 ]]; then + python_requirements=("${python_requirements[@]/%/[all]}") fi + pip install --only-binary=:all: "${dependency_args[@]}" "${python_requirements[@]}" + + - name: Install cuda.pathfinder extra wheels for testing + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} + run: | + set -euo pipefail + pushd cuda_pathfinder + pip install --only-binary=:all: -v ./*.whl --group "test-cu${TEST_CUDA_MAJOR}" + pip list + popd + + - name: Run cuda.pathfinder tests with all_must_work + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} + env: + CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work + CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work + CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: all_must_work + run: run-tests pathfinder # ── Nightly: install wheels + optional dep together ── - name: Install cuda-python wheels + PyTorch diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 1ee5600042d..3da9c180dd2 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -22,8 +22,8 @@ on: nruns: type: number default: 1 - targets: - description: JSON array of exact Moon Windows test route targets. An empty value tests everything. + workplan: + description: JSON workplan. An empty value tests everything. type: string default: "" run-id: @@ -89,7 +89,10 @@ jobs: test: env: - MOON_TARGETS: ${{ inputs.targets != '' && inputs.targets || '["pathfinder:ci-test-windows","bindings:ci-test-windows","core:ci-test-windows","metapackage:ci-test-windows"]' }} + TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }} + TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} + TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} + TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }} name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }} (${{ matrix.DRIVER_MODE }})${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }} timeout-minutes: 60 # The build stage could fail but we want the CI to keep moving. @@ -105,20 +108,6 @@ jobs: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Resolve Moon test targets - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - jq -e 'type == "array" and all(.[]; type == "string")' <<< "$MOON_TARGETS" >/dev/null - has_target() { - jq -e --arg target "$1" 'index($target) != null' <<< "$MOON_TARGETS" >/dev/null - } - { - echo "TEST_PATHFINDER=$(has_target pathfinder:ci-test-windows && echo true || echo false)" - echo "TEST_BINDINGS=$(has_target bindings:ci-test-windows && echo true || echo false)" - echo "TEST_CORE=$(has_target core:ci-test-windows && echo true || echo false)" - echo "TEST_PYTHON=$(has_target metapackage:ci-test-windows && echo true || echo false)" - } >> "$GITHUB_ENV" - - name: Setup proxy cache uses: nv-gha-runners/setup-proxy-cache@main continue-on-error: true @@ -238,25 +227,11 @@ jobs: rmdir cuda-python-wheel fi - - name: Stage pure wheels for Moon - if: ${{ inputs.test-mode == 'standard' }} - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - mkdir -p cuda_pathfinder/dist cuda_python/dist - pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) - if (( ${#pathfinder_wheels[@]} != 0 )) && [[ -f "${pathfinder_wheels[0]}" ]]; then - cp "${pathfinder_wheels[@]}" cuda_pathfinder/dist/ - fi - python_wheels=(cuda_python-*.whl) - if (( ${#python_wheels[@]} != 0 )) && [[ -f "${python_wheels[0]}" ]]; then - cp "${python_wheels[@]}" cuda_python/dist/ - fi - - name: Display structure of downloaded cuda-python artifacts if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'published' }} run: | Get-Location - Get-ChildItem -Recurse cuda_pathfinder,cuda_python -Filter *.whl | Select-Object Mode, LastWriteTime, Length, FullName + Get-ChildItem cuda_python*.whl | Select-Object Mode, LastWriteTime, Length, FullName - name: Display structure of downloaded cuda.bindings artifacts if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && @@ -281,7 +256,7 @@ jobs: Get-ChildItem -Recurse -Force $env:CUDA_BINDINGS_CYTHON_TESTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.core build artifacts - if: ${{ env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} @@ -290,7 +265,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core build artifacts - if: ${{ env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} + if: ${{ env.TEST_CORE == 'true' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_CORE_ARTIFACTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName @@ -332,13 +307,6 @@ jobs: # TODO: remove allow-prereleases once 3.15 is officially supported allow-prereleases: ${{ startsWith(matrix.PY_VER, '3.15') }} - - name: Install Moon - if: ${{ inputs.test-mode == 'standard' }} - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash - echo "$HOME/.moon/bin" >> "$GITHUB_PATH" - - name: Enable Scientific Python Nightly Wheels for Python 3.15 if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && startsWith(matrix.PY_VER, '3.15') }} @@ -368,25 +336,70 @@ jobs: shell: bash --noprofile --norc -xeuo pipefail {0} run: echo "PYTEST_ADDOPTS=\"--count=${{ inputs.nruns }}\"" >> "$GITHUB_ENV" - # ── Standard test route (skipped for nightly modes) ── - - name: Run selected installed-wheel tests - if: ${{ inputs.test-mode == 'standard' }} + # ── Standard test steps (skipped for nightly modes) ── + - name: Run cuda.pathfinder tests with see_what_works + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} + env: + CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works + CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works + CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: see_what_works + shell: bash --noprofile --norc -xeuo pipefail {0} + run: run-tests pathfinder + + - name: Run cuda.bindings tests + if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} + env: + CUDA_VER: ${{ matrix.CUDA_VER }} + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: run-tests bindings + + - name: Run cuda.core tests + if: ${{ inputs.test-mode == 'standard' && env.TEST_CORE == 'true' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} shell: bash --noprofile --norc -xeuo pipefail {0} + run: run-tests core + + - name: Ensure cuda-python installable + if: ${{ inputs.test-mode == 'standard' && env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} run: | - targets="$MOON_TARGETS" - if [[ "$SKIP_CUDA_BINDINGS_TEST" != "0" ]]; then - targets=$(jq -c 'map(select(. != "bindings:ci-test-windows"))' <<< "$targets") - fi - if [[ "$BINDINGS_SOURCE" != "main" ]]; then - targets=$(jq -c 'map(select(. != "metapackage:ci-test-windows"))' <<< "$targets") - fi - mapfile -t target_args < <(jq -r '.[]' <<< "$targets") - if (( ${#target_args[@]} != 0 )); then - moon run "${target_args[@]}" --upstream direct --downstream none - fi + # Package suites install their own dependencies. A metapackage-only + # run has no preceding suite, so install the exact local internal + # wheels in one transaction while resolving released dependencies + # such as cuda-core from the package index. + if ('${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }}' -eq 'true') { + $dependencyArgs = @('--no-deps') + } else { + $dependencyArgs = @( + (Get-Item ./cuda_pathfinder/cuda_pathfinder-*.whl).FullName + (Get-Item "$env:CUDA_BINDINGS_ARTIFACTS_DIR/cuda_bindings-*.whl").FullName + ) + } + $pythonRequirements = @((Get-Item ./cuda_python*.whl).FullName) + if ('${{ matrix.LOCAL_CTK }}' -ne '1') { + $pythonRequirements = @($pythonRequirements | ForEach-Object { "$($_)[all]" }) + } + pip install --only-binary=:all: @dependencyArgs @pythonRequirements + + - name: Install cuda.pathfinder extra wheels for testing + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + pushd cuda_pathfinder + pip install --only-binary=:all: -v ./*.whl --group "test-cu${TEST_CUDA_MAJOR}" + pip list + popd + + - name: Run cuda.pathfinder tests with all_must_work + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} + env: + CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work + CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work + CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: all_must_work + shell: bash --noprofile --norc -xeuo pipefail {0} + run: run-tests pathfinder # ── Nightly: install wheels + optional dep together ── - name: Install Visual C++ Redistributable (required by PyTorch on Windows) diff --git a/.gitignore b/.gitignore index 4824d472ec6..6b6a7dfc0b5 100644 --- a/.gitignore +++ b/.gitignore @@ -182,4 +182,3 @@ cython_debug/ # Cursor .cursorrules .claude/settings.local.json -.moon/cache/ diff --git a/.moon/workspace.yml b/.moon/workspace.yml deleted file mode 100644 index 02cc41d1447..00000000000 --- a/.moon/workspace.yml +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -projects: - root: '.' - pathfinder: 'cuda_pathfinder' - bindings: 'cuda_bindings' - core: 'cuda_core' - metapackage: 'cuda_python' - -vcs: - defaultBranch: 'main' - -versionConstraint: '=2.5.1' diff --git a/ci/test-matrix.yml b/ci/test-matrix.yml index 563774494e9..66f3196ab68 100644 --- a/ci/test-matrix.yml +++ b/ci/test-matrix.yml @@ -32,6 +32,12 @@ # ENV: { CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM: '1' } # ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.12.1', TORCH_CUDA: 'cu126' } +# Host platforms that produce wheel artifacts in the main CI workflow. +platforms: + - linux-64 + - linux-aarch64 + - win-64 + linux: pull-request: # linux-64 diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py new file mode 100644 index 00000000000..9f94c67173d --- /dev/null +++ b/ci/tools/compute_ci_plan.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Compute the CI build and test workplan for a pull request.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path, PurePosixPath + +REPO_ROOT = Path(__file__).resolve().parents[2] +MODULES = ("pathfinder", "bindings", "core", "python") +PLATFORMS = ("linux", "windows") +PACKAGE_MODULES = {f"cuda_{module}": module for module in MODULES} + +# Source changes have different build and test consumers. In particular, +# cuda-python source needs a same-version bindings wheel, while a core-only +# change can reuse the baseline cuda-python wheel. +SOURCE_IMPACT = { + "pathfinder": (set(MODULES), set(MODULES)), + "bindings": ({"bindings", "core", "python"}, {"bindings", "core", "python"}), + "core": ({"core"}, {"core", "python"}), + "python": ({"bindings", "python"}, {"python"}), +} + +IGNORED_BASENAMES = {"AGENTS.md", "CLAUDE.md", "pixi.lock", "pixi.toml"} +IGNORED_SUFFIXES = {".md", ".svg"} +IGNORED_PATHS = { + ".coveragerc", + ".gitignore", + ".pre-commit-config.yaml", + ".spdx-ignore", + "LICENSE", + "context7.json", + "greptile.json", + "ruff.toml", +} +IGNORED_PREFIXES = (".agents/", "toolshed/") + +# Only infrastructure exclusive to one OS belongs here; other CI paths force a full run. +TEST_INFRA_PLATFORMS = { + ".github/workflows/test-wheel-linux.yml": "linux", + ".github/workflows/test-wheel-windows.yml": "windows", + "ci/tools/configure_driver_mode.ps1": "windows", + "ci/tools/guess_latest.sh": "linux", + "ci/tools/install_gpu_driver.ps1": "windows", + "ci/tools/install_gpu_driver.sh": "linux", + "ci/tools/setup-sanitizer": "linux", +} + + +def compute_workplan( + paths: list[str], + *, + merge_base: str, + baseline_run_id: str, + linked_paths: set[str] | None = None, +) -> dict[str, object]: + """Return the final CI decisions for the supplied changed paths.""" + linked_paths = linked_paths or set() + source_changes: set[str] = set() + test_changes: set[str] = set() + test_platforms: set[str] = set() + force_all = not merge_base or not baseline_run_id + + if not force_all: + for path in paths: + path_parts = PurePosixPath(path).parts + if not path_parts: + continue + + if platform := TEST_INFRA_PLATFORMS.get(path): + test_platforms.add(platform) + continue + + if path_parts[0] == "ci" or ( + len(path_parts) >= 2 and path_parts[:2] in {(".github", "actions"), (".github", "workflows")} + ): + force_all = True + break + + if path_parts[0] == ".github" or path_parts[-1] in IGNORED_BASENAMES: + continue + + module = PACKAGE_MODULES.get(path_parts[0]) + if module is not None and len(path_parts) > 1: + relative = path_parts[1:] + if relative[0] == "docs": + continue + if ( + any(part in {"test", "tests"} for part in relative[:-1]) + or relative[0] == "examples" + or (module == "core" and relative == ("pytest.ini",)) + ): + test_changes.add(module) + elif PurePosixPath(path).suffix in IGNORED_SUFFIXES and path not in linked_paths: + continue + else: + source_changes.add(module) + continue + + is_test_path = any(part in {"test", "tests"} for part in path_parts[:-1]) + if is_test_path: + test_changes.update(MODULES) + elif ( + path in IGNORED_PATHS + or PurePosixPath(path).suffix in IGNORED_SUFFIXES + or path.startswith(IGNORED_PREFIXES) + ): + continue + elif path_parts[0] in {"benchmarks", "cuda_python_test_helpers"}: + test_changes.update(MODULES) + else: + force_all = True + break + + if force_all: + builds = set(MODULES) + tests = set(MODULES) + test_platforms = set(PLATFORMS) + else: + builds: set[str] = set() + tests = set(MODULES) if test_platforms else set(test_changes) + for module in source_changes: + build_impact, test_impact = SOURCE_IMPACT[module] + builds.update(build_impact) + tests.update(test_impact) + if source_changes or test_changes: + test_platforms.update(PLATFORMS) + + modules = { + module: { + "needs_build": module in builds, + "needs_test": module in tests, + } + for module in MODULES + } + return { + "modules": modules, + "jobs": { + # These gates cover both optional artifact builds and wheel tests. + "platforms": {platform: platform in test_platforms for platform in PLATFORMS}, + "sdist_tests": bool(builds), + "core_api_checks": force_all or "core" in source_changes, + }, + "merge_base": merge_base, + "baseline": { + "run_id": baseline_run_id if not force_all else "", + "sha": merge_base if not force_all else "", + }, + } + + +def _git_output(*args: str) -> bytes: + return subprocess.check_output( # noqa: S603 - argv is passed directly without a shell. + ["git", *args], # noqa: S607 + cwd=REPO_ROOT, + ) + + +def _changed_paths(merge_base: str) -> tuple[list[str], set[str]]: + output = _git_output("diff", "--no-renames", "--name-only", "-z", merge_base, "HEAD") + paths = [path.decode("utf-8", errors="surrogateescape") for path in output.split(b"\0") if path] + head_symlinks = _tracked_symlink_paths("HEAD") + # Base links preserve the packaging impact of deleted or replaced symlinks. + linked_paths = set(head_symlinks) | set(_tracked_symlink_paths(merge_base)) + return _expand_linked_paths(paths, head_symlinks, root=REPO_ROOT), linked_paths + + +def _tracked_symlink_paths(ref: str) -> list[str]: + output = _git_output("ls-tree", "--full-tree", "-r", "-z", ref) + return [ + entry.partition(b"\t")[2].decode("utf-8", errors="surrogateescape") + for entry in output.split(b"\0") + if entry.startswith(b"120000 ") + ] + + +def _expand_linked_paths(paths: list[str], symlink_paths: list[str], *, root: Path) -> list[str]: + """Include tracked symlinks whose resolved targets changed.""" + resolved_paths = {(root / path).resolve(strict=False) for path in paths} + expanded = list(paths) + selected = set(paths) + expanded.extend( + path for path in symlink_paths if path not in selected and (root / path).resolve(strict=False) in resolved_paths + ) + return expanded + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--merge-base", default="") + parser.add_argument("--baseline-run-id", default="") + args = parser.parse_args() + + reusable_baseline = bool(args.merge_base and args.baseline_run_id) + paths, linked_paths = _changed_paths(args.merge_base) if reusable_baseline else ([], set()) + plan = compute_workplan( + paths, + merge_base=args.merge_base, + baseline_run_id=args.baseline_run_id, + linked_paths=linked_paths, + ) + print(json.dumps(plan, separators=(",", ":"), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/ci/tools/tests/test_compute_ci_plan.py b/ci/tools/tests/test_compute_ci_plan.py new file mode 100644 index 00000000000..79a83394dfa --- /dev/null +++ b/ci/tools/tests/test_compute_ci_plan.py @@ -0,0 +1,181 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from ci.tools.compute_ci_plan import _expand_linked_paths, compute_workplan + +ALL_MODULES = {"pathfinder", "bindings", "core", "python"} +ALL_PLATFORMS = {"linux", "windows"} + + +def plan_for( + *paths: str, + baseline: bool = True, + linked_paths: set[str] | None = None, +) -> dict[str, object]: + return compute_workplan( + list(paths), + merge_base="base", + baseline_run_id="123" if baseline else "", + linked_paths=linked_paths, + ) + + +def selected(plan: dict[str, object], key: str) -> set[str]: + modules = plan["modules"] + assert isinstance(modules, dict) + return {name for name, decision in modules.items() if decision[key]} + + +def selected_platforms(plan: dict[str, object]) -> set[str]: + jobs = plan["jobs"] + assert isinstance(jobs, dict) + platforms = jobs["platforms"] + assert isinstance(platforms, dict) + assert set(platforms) == ALL_PLATFORMS + return {name for name, enabled in platforms.items() if enabled} + + +class ComputeWorkplanTest(unittest.TestCase): + def test_path_impacts(self) -> None: + cases = { + "cuda_pathfinder/cuda/pathfinder/_loader.py": (ALL_MODULES, ALL_MODULES, False), + "cuda_bindings/cuda/bindings/driver.pyx": ( + {"bindings", "core", "python"}, + {"bindings", "core", "python"}, + False, + ), + "cuda_core/cuda/core/_device.py": ({"core"}, {"core", "python"}, True), + "cuda_core/cuda/core/examples/demo.py": ({"core"}, {"core", "python"}, True), + "cuda_python/pyproject.toml": ({"bindings", "python"}, {"python"}, False), + "cuda_pathfinder/tests/test_loader.py": (set(), {"pathfinder"}, False), + "cuda_bindings/examples/0_Introduction/vectorAddDrv.py": (set(), {"bindings"}, False), + "cuda_bindings/tests/README.md": (set(), {"bindings"}, False), + "cuda_core/pytest.ini": (set(), {"core"}, False), + "cuda_python/tests/test_import.py": (set(), {"python"}, False), + "cuda_python_test_helpers/cuda_python_test_helpers/cuda_utils.py": ( + set(), + ALL_MODULES, + False, + ), + "benchmarks/cuda_bindings/run_pyperf.py": (set(), ALL_MODULES, False), + "benchmarks/cuda_core/runner.py": (set(), ALL_MODULES, False), + "ci/tools/run-tests": (ALL_MODULES, ALL_MODULES, True), + "ci/versions.yml": (ALL_MODULES, ALL_MODULES, True), + "pytest.ini": (ALL_MODULES, ALL_MODULES, True), + } + + for path, (builds, tests, core_api) in cases.items(): + with self.subTest(path=path): + plan = plan_for(path) + assert selected(plan, "needs_build") == builds + assert selected(plan, "needs_test") == tests + assert selected_platforms(plan) == ALL_PLATFORMS + assert plan["jobs"]["sdist_tests"] == bool(builds) + assert plan["jobs"]["core_api_checks"] == core_api + + def test_test_infrastructure_platforms(self) -> None: + cases = { + ".github/workflows/test-wheel-linux.yml": {"linux"}, + ".github/workflows/test-wheel-windows.yml": {"windows"}, + "ci/tools/configure_driver_mode.ps1": {"windows"}, + "ci/tools/guess_latest.sh": {"linux"}, + "ci/tools/install_gpu_driver.ps1": {"windows"}, + "ci/tools/install_gpu_driver.sh": {"linux"}, + "ci/tools/setup-sanitizer": {"linux"}, + } + + for path, platforms in cases.items(): + with self.subTest(path=path): + plan = plan_for(path) + assert not selected(plan, "needs_build") + assert selected(plan, "needs_test") == ALL_MODULES + assert selected_platforms(plan) == platforms + assert not plan["jobs"]["sdist_tests"] + assert not plan["jobs"]["core_api_checks"] + + mixed_plan = plan_for("ci/tools/install_gpu_driver.sh", "ci/tools/install_gpu_driver.ps1") + assert selected_platforms(mixed_plan) == ALL_PLATFORMS + + source_plan = plan_for("ci/tools/install_gpu_driver.sh", "cuda_python/pyproject.toml") + assert selected_platforms(source_plan) == ALL_PLATFORMS + + def test_ignored_paths_select_no_work(self) -> None: + for path in ( + "cuda_core/docs/index.rst", + "cuda_core/pixi.toml", + "cuda_core/tests/fixtures/pixi.toml", + "benchmarks/cuda_bindings/pixi.toml", + "benchmarks/cuda_bindings/AGENTS.md", + "cuda_core/cuda/core/_cpp/DESIGN.md", + "cuda_bindings/README.md", + "cuda_core/README.md", + "new-area/pixi.toml", + "notes.md", + "diagram.svg", + ".github/labeler.yml", + ".github/ISSUE_TEMPLATE/bug.yml", + ): + with self.subTest(path=path): + plan = plan_for(path) + assert not selected(plan, "needs_build") + assert not selected(plan, "needs_test") + assert not selected_platforms(plan) + + def test_unknown_path_and_missing_baseline_force_all(self) -> None: + for plan in ( + plan_for("new-top-level-file"), + plan_for("new-area/config.toml"), + plan_for(".github/workflows/new-main-ci-workflow.yml"), + plan_for(".github/actions/doc_preview/action.yml"), + plan_for("ci/ci-pipeline.svg"), + plan_for("cuda_core/docs/index.rst", baseline=False), + compute_workplan([], merge_base="", baseline_run_id="123"), + ): + assert selected(plan, "needs_build") == ALL_MODULES + assert selected(plan, "needs_test") == ALL_MODULES + assert selected_platforms(plan) == ALL_PLATFORMS + assert plan["jobs"]["core_api_checks"] + assert plan["baseline"] == {"run_id": "", "sha": ""} + + def test_mixed_changes_are_combined(self) -> None: + plan = plan_for("cuda_core/tests/test_device.py", "cuda_python/pyproject.toml") + assert selected(plan, "needs_build") == {"bindings", "python"} + assert selected(plan, "needs_test") == {"core", "python"} + assert selected_platforms(plan) == ALL_PLATFORMS + assert plan["jobs"]["sdist_tests"] + assert plan["baseline"] == {"run_id": "123", "sha": "base"} + + def test_changed_symlink_targets_include_their_consumers(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "cuda_python").mkdir() + (root / "cuda_core").mkdir() + (root / "README.md").write_text("readme", encoding="utf-8") + (root / "cuda_python" / "README.md").symlink_to("../README.md") + (root / "cuda_core" / "README.md").symlink_to("../README.md") + + paths = _expand_linked_paths( + ["README.md"], + ["cuda_python/README.md"], + root=root, + ) + + assert paths == ["README.md", "cuda_python/README.md"] + plan = plan_for(*paths, linked_paths={"cuda_python/README.md"}) + assert selected(plan, "needs_build") == {"bindings", "python"} + assert selected(plan, "needs_test") == {"python"} + + removed_link = plan_for("cuda_python/README.md", linked_paths={"cuda_python/README.md"}) + assert selected(removed_link, "needs_build") == {"bindings", "python"} + assert selected(removed_link, "needs_test") == {"python"} + + +if __name__ == "__main__": + unittest.main() diff --git a/cuda_bindings/moon.yml b/cuda_bindings/moon.yml deleted file mode 100644 index d21b05fa280..00000000000 --- a/cuda_bindings/moon.yml +++ /dev/null @@ -1,224 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -$schema: https://moonrepo.dev/schemas/v2/project.json - -dependsOn: ['pathfinder'] - -fileGroups: - package: - - 'cuda/**/*' - - '!cuda/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - - '!cuda/**/*.{md,svg}' - - 'build_hooks.py' - - 'DESCRIPTION.rst' - - 'LICENSE' - - 'MANIFEST.in' - - 'pyproject.toml' - - 'setup.py' - - '.git_archival.txt' - - '/.git_archival.txt' - - '/cuda_pathfinder/.git_archival.txt' - tests: - - 'tests/**/*' - - 'examples/**/*' - - '!{tests,examples}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - docs: - - 'docs/**/*' - - '!docs/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - benchmarks: - - '/benchmarks/cuda_bindings/benchmarks/**/*' - - '/benchmarks/cuda_bindings/runner/**/*' - - '/benchmarks/cuda_bindings/tests/**/*' - - '/benchmarks/cuda_bindings/compare.py' - - '/benchmarks/cuda_bindings/run_cpp.py' - - '/benchmarks/cuda_bindings/run_pyperf.py' - - '/benchmarks/cuda_bindings/pixi.lock' - - '/benchmarks/cuda_bindings/pixi.toml' - sharedTestInfra: - - '/cuda_python_test_helpers/**/*' - - '/benchmarks/**/*' - - '!/{benchmarks,cuda_python_test_helpers}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - - '!/benchmarks/**/*.{md,svg}' - - '/ci/test-matrix.yml' - - '/ci/tools/download-wheels' - - '/ci/tools/run-tests' - linuxTestInfra: - - '/.github/workflows/test-wheel-linux.yml' - - '/ci/tools/guess_latest.sh' - - '/ci/tools/install_gpu_driver.sh' - - '/ci/tools/setup-sanitizer' - windowsTestInfra: - - '/.github/workflows/test-wheel-windows.yml' - - '/ci/tools/configure_driver_mode.ps1' - - '/ci/tools/install_gpu_driver.ps1' - -tasks: - install: - script: 'python -m pip install -e . pyperf --group test' - deps: ['pathfinder:install'] - inputs: ['@group(package)'] - options: - cache: false - windowsShell: 'bash' - - test: - script: | - bash tests/cython/build_tests.sh - python -m pytest . --override-ini norecursedirs=examples - python -m pytest ../benchmarks/cuda_bindings/tests/ - deps: ['install'] - inputs: ['@group(package)', '@group(tests)', '@group(benchmarks)', '/cuda_python_test_helpers/**/*'] - options: - cache: false - windowsShell: 'bash' - - test-installed: - script: | - set -euo pipefail - cd .. - temporary_wheels=() - for wheel in cuda_pathfinder/dist/*.whl; do - staged="cuda_pathfinder/$(basename "$wheel")" - if [[ ! -e "$staged" ]]; then - cp "$wheel" "$staged" - temporary_wheels+=("$staged") - fi - done - trap 'rm -f "${temporary_wheels[@]}"' EXIT - ci/tools/run-tests bindings - deps: ['wheel', 'build-cython-tests'] - inputs: - - '@group(package)' - - '@group(tests)' - - '@group(sharedTestInfra)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - options: - cache: false - internal: true - mutex: 'ci-python-gpu' - windowsShell: 'bash' - - build-cython-tests: - script: 'python -m pip install ../cuda_pathfinder/dist/*.whl dist/*.whl --group ./pyproject.toml:test && bash tests/cython/build_tests.sh' - deps: ['wheel'] - inputs: ['@group(package)', 'tests/cython/**/*', '/cuda_python_test_helpers/**/*'] - options: - cache: false - internal: true - mutex: 'ci-python-build' - windowsShell: 'bash' - - benchmark-smoke: - script: 'python -m pip install ../cuda_pathfinder/dist/*.whl dist/*.whl pyperf --group ./pyproject.toml:test && cd ../benchmarks/cuda_bindings && python run_pyperf.py --debug-single-value' - deps: ['wheel'] - inputs: ['@group(package)', '@group(benchmarks)'] - options: - cache: false - internal: true - mutex: 'ci-python-gpu' - windowsShell: 'bash' - - wheel: - script: 'cd .. && mkdir -p cuda_bindings/dist && python -m cibuildwheel cuda_bindings --output-dir cuda_bindings/dist' - deps: ['pathfinder:wheel'] - inputs: - - '@group(package)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_python/DESCRIPTION.rst' - - '/cuda_python/LICENSE' - - '/cuda_python/pyproject.toml' - - '/cuda_python/setup.py' - - '/cuda_python/README.md' - - '/README.md' - options: - cache: false - windowsShell: 'bash' - tags: ['ci-wheel-bindings'] - - sdist: - script: 'mkdir -p dist && export PIP_FIND_LINKS=../cuda_pathfinder/dist PIP_PRE=1 && python -m build --sdist --outdir dist . && python -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz' - deps: ['pathfinder:sdist'] - inputs: - - '@group(package)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_python/DESCRIPTION.rst' - - '/cuda_python/LICENSE' - - '/cuda_python/pyproject.toml' - - '/cuda_python/setup.py' - - '/cuda_python/README.md' - - '/README.md' - options: - cache: false - windowsShell: 'bash' - tags: ['ci-sdist-bindings'] - - docs: - script: 'rm -rf docs/build docs/source/generated && cd docs && ./build_docs.sh ${DOCS_BUILD_ARGS:-}' - inputs: ['@group(docs)'] - options: - cache: false - tags: ['ci-docs'] - - benchmark: - script: 'cd ../benchmarks/cuda_bindings && python run_pyperf.py' - deps: ['install'] - inputs: ['@group(package)', '@group(benchmarks)'] - options: - cache: false - - ci-test-assets: - deps: ['build-cython-tests'] - inputs: - - '@group(package)' - - '@group(tests)' - - '@group(sharedTestInfra)' - - '@group(linuxTestInfra)' - - '@group(windowsTestInfra)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - options: - cache: false - tags: ['ci-test-assets-current'] - - ci-test-linux: - deps: ['test-installed', 'benchmark-smoke'] - inputs: - - '@group(package)' - - '@group(tests)' - - '@group(sharedTestInfra)' - - '@group(linuxTestInfra)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - options: - cache: false - tags: ['ci-test-linux'] - - ci-test-windows: - deps: ['test-installed'] - inputs: - - '@group(package)' - - '@group(tests)' - - '@group(sharedTestInfra)' - - '@group(windowsTestInfra)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - options: - cache: false - tags: ['ci-test-windows'] diff --git a/cuda_core/moon.yml b/cuda_core/moon.yml deleted file mode 100644 index c4b5b47f455..00000000000 --- a/cuda_core/moon.yml +++ /dev/null @@ -1,318 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -$schema: https://moonrepo.dev/schemas/v2/project.json - -dependsOn: ['bindings'] - -fileGroups: - package: - - 'cuda/**/*' - - '!cuda/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - - '!cuda/**/*.{md,svg}' - - 'build_hooks.py' - - 'DESCRIPTION.rst' - - 'LICENSE' - - 'MANIFEST.in' - - 'NOTICE' - - 'pyproject.toml' - - 'setup.py' - - '.git_archival.txt' - - '/.git_archival.txt' - tests: - - 'tests/**/*' - - 'examples/**/*' - - '!{tests,examples}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - - 'pytest.ini' - docs: - - 'docs/**/*' - - '!docs/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - sharedTestInfra: - - '/cuda_python_test_helpers/**/*' - - '/benchmarks/**/*' - - '!/{benchmarks,cuda_python_test_helpers}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - - '!/benchmarks/**/*.{md,svg}' - - '/ci/test-matrix.yml' - - '/ci/tools/download-wheels' - - '/ci/tools/run-tests' - linuxTestInfra: - - '/.github/workflows/test-wheel-linux.yml' - - '/ci/tools/guess_latest.sh' - - '/ci/tools/install_gpu_driver.sh' - - '/ci/tools/setup-sanitizer' - windowsTestInfra: - - '/.github/workflows/test-wheel-windows.yml' - - '/ci/tools/configure_driver_mode.ps1' - - '/ci/tools/install_gpu_driver.ps1' - upstreamPackageVersions: - - '/cuda_pathfinder/.git_archival.txt' - - '/cuda_bindings/.git_archival.txt' - -tasks: - install: - script: 'python -m pip install -e . --group test' - deps: ['bindings:install'] - inputs: ['@group(package)'] - options: - cache: false - windowsShell: 'bash' - - test: - script: | - bash tests/cython/build_tests.sh - python -m pytest . --override-ini norecursedirs="" - deps: ['install'] - inputs: ['@group(package)', '@group(tests)', '/cuda_python_test_helpers/**/*'] - options: - cache: false - windowsShell: 'bash' - - test-installed: - script: | - set -euo pipefail - cd .. - temporary_wheels=() - for wheel in cuda_pathfinder/dist/*.whl; do - staged="cuda_pathfinder/$(basename "$wheel")" - if [[ ! -e "$staged" ]]; then - cp "$wheel" "$staged" - temporary_wheels+=("$staged") - fi - done - trap 'rm -f "${temporary_wheels[@]}"' EXIT - ci/tools/run-tests core - deps: ['wheel', 'build-cython-tests', 'build-test-binaries'] - inputs: - - '@group(package)' - - '@group(tests)' - - '@group(sharedTestInfra)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - options: - cache: false - internal: true - mutex: 'ci-python-gpu' - windowsShell: 'bash' - - build-cython-tests: - script: | - set -euo pipefail - core_wheels=(dist/cu${CUDA_CORE_BUILD_MAJOR:?}/*.whl) - if [[ ! -e "${core_wheels[0]}" ]]; then - core_wheels=(dist/*.whl) - fi - python -m pip install ../cuda_pathfinder/dist/*.whl ../cuda_bindings/dist/*.whl "${core_wheels[@]}" --group ./pyproject.toml:test - bash tests/cython/build_tests.sh - deps: ['wheel'] - inputs: ['@group(package)', 'tests/cython/**/*', '/cuda_bindings/cuda/**/*', '/cuda_python_test_helpers/**/*'] - options: - cache: false - internal: true - mutex: 'ci-python-build' - windowsShell: 'bash' - - build-test-binaries: - command: 'python' - args: ['tests/test_binaries/build_test_binaries.py'] - inputs: ['tests/test_binaries/build_test_binaries.py', 'tests/test_binaries/saxpy.cu'] - options: - cache: false - internal: true - mutex: 'ci-python-build' - - wheel: - script: | - cd .. - cuda_major=${CUDA_CORE_BUILD_MAJOR:?} - mkdir -p "cuda_core/dist/cu${cuda_major}" - python -m cibuildwheel cuda_core --output-dir "cuda_core/dist/cu${cuda_major}" - shopt -s nullglob - wheels=("cuda_core/dist/cu${cuda_major}"/*.whl) - test "${#wheels[@]}" -eq 1 - if [[ "${wheels[0]}" != *.cu${cuda_major}.whl ]]; then - mv "${wheels[0]}" "${wheels[0]%.whl}.cu${cuda_major}.whl" - fi - deps: ['bindings:wheel'] - inputs: - - '@group(package)' - - '@group(upstreamPackageVersions)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - - '/ci/tools/merge_cuda_core_wheels.py' - options: - cache: false - windowsShell: 'bash' - tags: ['ci-wheel-consumers', 'ci-wheel-multi-ctk'] - - wheel-merge: - script: 'python ../ci/tools/merge_cuda_core_wheels.py dist/cu12/*.whl dist/cu13/*.whl --output-dir dist' - inputs: - - '@group(package)' - - '@group(upstreamPackageVersions)' - - 'dist/cu12/*.whl' - - 'dist/cu13/*.whl' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - - '/ci/tools/merge_cuda_core_wheels.py' - options: - cache: false - windowsShell: 'bash' - tags: ['ci-wheel-finalize'] - - sdist: - script: 'mkdir -p dist && export PIP_FIND_LINKS="../cuda_pathfinder/dist ../cuda_bindings/dist" PIP_PRE=1 && python -m build --sdist --outdir dist . && python -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz' - deps: ['bindings:sdist'] - inputs: - - '@group(package)' - - '@group(upstreamPackageVersions)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - options: - cache: false - windowsShell: 'bash' - tags: ['ci-sdist-consumers'] - - docs: - script: 'rm -rf docs/build docs/source/generated && cd docs && ./build_docs.sh ${DOCS_BUILD_ARGS:-}' - inputs: ['@group(docs)'] - options: - cache: false - tags: ['ci-docs'] - - api-check: - script: 'uvx griffe check cuda.core --search . --find-stubs-packages --against "${CUDA_CORE_API_REF:?}" --format github 2>&1' - inputs: ['@group(package)', '/.github/actions/griffe-api-check/action.yml'] - options: - cache: false - tags: ['ci-api'] - - ci-test-assets: - deps: ['build-cython-tests'] - inputs: - - '@group(package)' - - '@group(upstreamPackageVersions)' - - '@group(tests)' - - '@group(sharedTestInfra)' - - '@group(linuxTestInfra)' - - '@group(windowsTestInfra)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - options: - cache: false - tags: ['ci-test-assets-current'] - - ci-test-binaries: - deps: ['build-test-binaries'] - inputs: - - '@group(package)' - - '@group(upstreamPackageVersions)' - - '@group(tests)' - - '@group(sharedTestInfra)' - - '@group(linuxTestInfra)' - - '@group(windowsTestInfra)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - options: - cache: false - tags: ['ci-test-assets-previous'] - - ci-test-linux: - deps: ['test-installed'] - inputs: - - '@group(package)' - - '@group(upstreamPackageVersions)' - - '@group(tests)' - - '@group(sharedTestInfra)' - - '@group(linuxTestInfra)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - options: - cache: false - tags: ['ci-test-linux'] - - ci-test-windows: - deps: ['test-installed'] - inputs: - - '@group(package)' - - '@group(upstreamPackageVersions)' - - '@group(tests)' - - '@group(sharedTestInfra)' - - '@group(windowsTestInfra)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - options: - cache: false - tags: ['ci-test-windows'] diff --git a/cuda_pathfinder/moon.yml b/cuda_pathfinder/moon.yml deleted file mode 100644 index c164576c52c..00000000000 --- a/cuda_pathfinder/moon.yml +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -$schema: https://moonrepo.dev/schemas/v2/project.json - -fileGroups: - package: - - 'cuda/**/*' - - '!cuda/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - - '!cuda/**/*.{md,svg}' - - 'DESCRIPTION.rst' - - 'LICENSE' - - 'pyproject.toml' - - '.git_archival.txt' - - '/.git_archival.txt' - tests: - - 'tests/**/*' - - 'examples/**/*' - - '!{tests,examples}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - docs: - - 'docs/**/*' - - '!docs/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - sharedTestInfra: - - '/cuda_python_test_helpers/**/*' - - '/benchmarks/**/*' - - '!/{benchmarks,cuda_python_test_helpers}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - - '!/benchmarks/**/*.{md,svg}' - - '/ci/test-matrix.yml' - - '/ci/tools/download-wheels' - - '/ci/tools/run-tests' - linuxTestInfra: - - '/.github/workflows/test-wheel-linux.yml' - - '/ci/tools/guess_latest.sh' - - '/ci/tools/install_gpu_driver.sh' - - '/ci/tools/setup-sanitizer' - windowsTestInfra: - - '/.github/workflows/test-wheel-windows.yml' - - '/ci/tools/configure_driver_mode.ps1' - - '/ci/tools/install_gpu_driver.ps1' - -tasks: - install: - script: 'python -m pip install -e . --group test' - inputs: ['@group(package)'] - options: - cache: false - windowsShell: 'bash' - - test: - script: 'python -m pytest tests/' - deps: ['install'] - inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)'] - options: - cache: false - windowsShell: 'bash' - - test-installed: - script: | - set -euo pipefail - cd .. - temporary_wheels=() - for wheel in cuda_pathfinder/dist/*.whl; do - staged="cuda_pathfinder/$(basename "$wheel")" - if [[ ! -e "$staged" ]]; then - cp "$wheel" "$staged" - temporary_wheels+=("$staged") - fi - done - trap 'rm -f "${temporary_wheels[@]}"' EXIT - CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS=see_what_works CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS=see_what_works CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS=see_what_works ci/tools/run-tests pathfinder - python -m pip install --only-binary=:all: -v cuda_pathfinder/*.whl --group "./cuda_pathfinder/pyproject.toml:test-cu${TEST_CUDA_MAJOR:?}" - CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS=all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS=all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS=all_must_work ci/tools/run-tests pathfinder - deps: ['wheel'] - inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)'] - options: - cache: false - internal: true - mutex: 'ci-python-gpu' - windowsShell: 'bash' - - wheel: - script: 'mkdir -p dist && python -m pip wheel -v --no-deps --wheel-dir dist .' - inputs: ['@group(package)'] - options: - cache: false - windowsShell: 'bash' - tags: ['ci-wheel-foundation'] - - sdist: - script: 'mkdir -p dist && python -m build --sdist --outdir dist . && python -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz' - inputs: ['@group(package)'] - options: - cache: false - windowsShell: 'bash' - tags: ['ci-sdist-foundation'] - - docs: - script: 'rm -rf docs/build docs/source/generated && cd docs && ./build_docs.sh ${DOCS_BUILD_ARGS:-}' - inputs: ['@group(docs)'] - options: - cache: false - tags: ['ci-docs'] - - ci-test-linux: - deps: ['test-installed'] - inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)', '@group(linuxTestInfra)'] - options: - cache: false - tags: ['ci-test-linux'] - - ci-test-windows: - deps: ['test-installed'] - inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)', '@group(windowsTestInfra)'] - options: - cache: false - tags: ['ci-test-windows'] diff --git a/cuda_python/moon.yml b/cuda_python/moon.yml deleted file mode 100644 index 52999bb9ca1..00000000000 --- a/cuda_python/moon.yml +++ /dev/null @@ -1,194 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -$schema: https://moonrepo.dev/schemas/v2/project.json - -dependsOn: ['bindings'] - -fileGroups: - package: - - 'DESCRIPTION.rst' - - 'LICENSE' - - 'pyproject.toml' - - 'setup.py' - - 'README.md' - - '/README.md' - - '/cuda_pathfinder/.git_archival.txt' - - '/cuda_bindings/.git_archival.txt' - - '/.git_archival.txt' - tests: - - '/tests/integration/**/*' - - '!/tests/integration/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - docs: - - 'docs/**/*' - - '!docs/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - sharedTestInfra: - - '/cuda_python_test_helpers/**/*' - - '/benchmarks/**/*' - - '!/{benchmarks,cuda_python_test_helpers}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' - - '!/benchmarks/**/*.{md,svg}' - - '/cuda_core/.git_archival.txt' - - '/ci/test-matrix.yml' - - '/ci/tools/download-wheels' - - '/ci/tools/run-tests' - linuxTestInfra: - - '/.github/workflows/test-wheel-linux.yml' - - '/ci/tools/guess_latest.sh' - - '/ci/tools/install_gpu_driver.sh' - - '/ci/tools/setup-sanitizer' - windowsTestInfra: - - '/.github/workflows/test-wheel-windows.yml' - - '/ci/tools/configure_driver_mode.ps1' - - '/ci/tools/install_gpu_driver.ps1' - -tasks: - test-installed: - script: | - set -euo pipefail - core_wheels=(../cuda_core/dist/*.whl) - if [[ ! -e "${core_wheels[0]}" ]]; then - core_wheels=(../cuda_core/dist/cu${CUDA_CORE_BUILD_MAJOR:?}/*.whl) - fi - python -m pip install --only-binary=:all: ../cuda_pathfinder/dist/*.whl ../cuda_bindings/dist/*.whl "${core_wheels[@]}" dist/*.whl - deps: ['wheel', 'core:wheel'] - inputs: - - '@group(package)' - - '@group(tests)' - - '@group(sharedTestInfra)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - - '/cuda_core/cuda/**/*' - - '/cuda_core/build_hooks.py' - - '/cuda_core/DESCRIPTION.rst' - - '/cuda_core/LICENSE' - - '/cuda_core/MANIFEST.in' - - '/cuda_core/NOTICE' - - '/cuda_core/pyproject.toml' - - '/cuda_core/setup.py' - options: - cache: false - internal: true - mutex: 'ci-python-gpu' - windowsShell: 'bash' - - wheel: - script: 'mkdir -p dist && python -m pip wheel -v --no-deps --wheel-dir dist .' - deps: ['bindings:wheel'] - inputs: - - '@group(package)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - - '/cuda_python/README.md' - - '/README.md' - options: - cache: false - windowsShell: 'bash' - tags: ['ci-wheel-consumers'] - - sdist: - script: 'mkdir -p dist && python -m build --sdist --outdir dist . && python -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz' - inputs: - - '@group(package)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - - '/cuda_python/README.md' - - '/README.md' - options: - cache: false - windowsShell: 'bash' - tags: ['ci-sdist-consumers'] - - docs: - script: 'rm -rf docs/build docs/source/generated && cd docs && ./build_docs.sh ${DOCS_BUILD_ARGS:-}' - inputs: ['@group(docs)'] - options: - cache: false - tags: ['ci-docs'] - - ci-test-linux: - deps: ['test-installed'] - inputs: - - '@group(package)' - - '@group(tests)' - - '@group(sharedTestInfra)' - - '@group(linuxTestInfra)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - - '/cuda_core/cuda/**/*' - - '/cuda_core/build_hooks.py' - - '/cuda_core/DESCRIPTION.rst' - - '/cuda_core/LICENSE' - - '/cuda_core/MANIFEST.in' - - '/cuda_core/NOTICE' - - '/cuda_core/pyproject.toml' - - '/cuda_core/setup.py' - options: - cache: false - tags: ['ci-test-linux'] - - ci-test-windows: - deps: ['test-installed'] - inputs: - - '@group(package)' - - '@group(tests)' - - '@group(sharedTestInfra)' - - '@group(windowsTestInfra)' - - '/cuda_pathfinder/cuda/**/*' - - '/cuda_pathfinder/DESCRIPTION.rst' - - '/cuda_pathfinder/LICENSE' - - '/cuda_pathfinder/pyproject.toml' - - '/cuda_bindings/cuda/**/*' - - '/cuda_bindings/build_hooks.py' - - '/cuda_bindings/DESCRIPTION.rst' - - '/cuda_bindings/LICENSE' - - '/cuda_bindings/MANIFEST.in' - - '/cuda_bindings/pyproject.toml' - - '/cuda_bindings/setup.py' - - '/cuda_core/cuda/**/*' - - '/cuda_core/build_hooks.py' - - '/cuda_core/DESCRIPTION.rst' - - '/cuda_core/LICENSE' - - '/cuda_core/MANIFEST.in' - - '/cuda_core/NOTICE' - - '/cuda_core/pyproject.toml' - - '/cuda_core/setup.py' - options: - cache: false - tags: ['ci-test-windows'] diff --git a/moon.yml b/moon.yml deleted file mode 100644 index 2670648d385..00000000000 --- a/moon.yml +++ /dev/null @@ -1,111 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -$schema: https://moonrepo.dev/schemas/v2/project.json - -tasks: - test: - deps: - - 'pathfinder:test' - - 'bindings:test' - - 'core:test' - inputs: [] - options: - cache: false - - docs: - script: | - rm -rf cuda_python/docs/build/html/cuda-bindings cuda_python/docs/build/html/cuda-core cuda_python/docs/build/html/cuda-pathfinder - mkdir -p cuda_python/docs/build/html/cuda-bindings cuda_python/docs/build/html/cuda-core cuda_python/docs/build/html/cuda-pathfinder - cp -R cuda_bindings/docs/build/html/. cuda_python/docs/build/html/cuda-bindings/ - cp -R cuda_core/docs/build/html/. cuda_python/docs/build/html/cuda-core/ - cp -R cuda_pathfinder/docs/build/html/. cuda_python/docs/build/html/cuda-pathfinder/ - deps: - - 'pathfinder:docs' - - 'bindings:docs' - - 'core:docs' - - 'metapackage:docs' - inputs: [] - options: - cache: false - tags: ['ci-docs'] - - ci-ignore: - inputs: - - '/.agents/**/*' - - '/.coveragerc' - - '/.gitattributes' - - '/.gitignore' - - '/.github/**/*' - - '!/.github/actions/**/*' - - '!/.github/workflows/**/*' - - '/.mailmap' - - '/.pre-commit-config.yaml' - - '/.spdx-ignore' - - '/**/AGENTS.md' - - '/**/CLAUDE.md' - - '/**/pixi.lock' - - '/**/pixi.toml' - - '/**/*.md' - - '/**/*.svg' - - '!/README.md' - - '!/cuda_python/README.md' - - '/context7.json' - - '/greptile.json' - - '/LICENSE' - - '/ruff.toml' - - '/toolshed/**/*' - options: - cache: false - tags: ['ci-ignore'] - - ci-fallback: - deps: - - 'pathfinder:wheel' - - 'bindings:wheel' - - 'core:wheel' - - 'metapackage:wheel' - - 'core:wheel-merge' - - 'pathfinder:sdist' - - 'bindings:sdist' - - 'core:sdist' - - 'metapackage:sdist' - - 'pathfinder:ci-test-linux' - - 'pathfinder:ci-test-windows' - - 'bindings:ci-test-linux' - - 'bindings:ci-test-windows' - - 'bindings:ci-test-assets' - - 'core:ci-test-linux' - - 'core:ci-test-windows' - - 'core:ci-test-assets' - - 'core:ci-test-binaries' - - 'metapackage:ci-test-linux' - - 'metapackage:ci-test-windows' - - 'core:api-check' - - 'docs' - inputs: - - '/.github/actions/**/*' - - '/.github/workflows/build-docs.yml' - - '/.github/workflows/build-wheel.yml' - - '/.github/workflows/ci-nightly.yml' - - '/.github/workflows/ci-pixi-source-test.yml' - - '/.github/workflows/ci.yml' - - '/.github/workflows/coverage.yml' - - '/.github/workflows/release*.yml' - - '/.github/workflows/test-sdist-linux.yml' - - '/.github/workflows/test-sdist-windows.yml' - - '/.moon/workspace.yml' - - '/moon.yml' - - '/cuda_pathfinder/moon.yml' - - '/cuda_bindings/moon.yml' - - '/cuda_core/moon.yml' - - '/cuda_python/moon.yml' - - '/ci/tools/env-vars' - - '/ci/versions.yml' - - '/pyproject.toml' - - '/pytest.ini' - - '/tests/test_moon_ci.py' - options: - cache: false - tags: ['ci-force-all'] diff --git a/tests/test_moon_ci.py b/tests/test_moon_ci.py deleted file mode 100644 index 3dc06a0ebce..00000000000 --- a/tests/test_moon_ci.py +++ /dev/null @@ -1,649 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Behavior checks for the Moon-owned selective CI graph.""" - -from __future__ import annotations - -import json -import os -import shutil -import subprocess -import textwrap -from pathlib import Path -from typing import Any - -import pytest - -ROOT = Path(__file__).resolve().parents[1] -MOON = shutil.which("moon") -BASH = shutil.which("bash") - -VISIBLE_TASKS = { - "root": {"test", "docs", "ci-ignore", "ci-fallback"}, - "pathfinder": {"install", "test", "docs", "wheel", "sdist", "ci-test-linux", "ci-test-windows"}, - "bindings": { - "install", - "test", - "docs", - "wheel", - "sdist", - "benchmark", - "ci-test-linux", - "ci-test-windows", - "ci-test-assets", - }, - "core": { - "install", - "test", - "docs", - "wheel", - "wheel-merge", - "sdist", - "api-check", - "ci-test-linux", - "ci-test-windows", - "ci-test-assets", - "ci-test-binaries", - }, - "metapackage": {"docs", "wheel", "sdist", "ci-test-linux", "ci-test-windows"}, -} -INTERNAL_TASKS = { - "pathfinder:test-installed", - "bindings:test-installed", - "bindings:build-cython-tests", - "bindings:benchmark-smoke", - "core:test-installed", - "core:build-cython-tests", - "core:build-test-binaries", - "metapackage:test-installed", -} -ALL_ROUTES = { - f"{project}:ci-test-{os_name}" - for project in ("pathfinder", "bindings", "core", "metapackage") - for os_name in ("linux", "windows") -} -LINUX_ROUTES = {target for target in ALL_ROUTES if target.endswith("-linux")} -WINDOWS_ROUTES = {target for target in ALL_ROUTES if target.endswith("-windows")} -ASSET_ROUTES = {"bindings:ci-test-assets", "core:ci-test-assets", "core:ci-test-binaries"} -PRODUCERS = { - f"{project}:{kind}" for project in ("pathfinder", "bindings", "core", "metapackage") for kind in ("wheel", "sdist") -} | {"core:wheel-merge"} -CI_TASKS = ( - ALL_ROUTES - | ASSET_ROUTES - | PRODUCERS - | {f"{project}:docs" for project in ("root", "pathfinder", "bindings", "core", "metapackage")} - | {"core:api-check", "root:ci-ignore", "root:ci-fallback"} -) - - -def run_moon(*args: str, stdin: str | None = None) -> subprocess.CompletedProcess[str]: - assert MOON is not None - return subprocess.run( # noqa: S603 - MOON resolves to the pinned executable. - [MOON, *args], - cwd=ROOT, - input=stdin, - text=True, - check=False, - capture_output=True, - ) - - -def moon_json(*args: str, stdin: str | None = None) -> dict[str, Any]: - result = run_moon(*args, stdin=stdin) - result.check_returncode() - return json.loads(result.stdout) - - -def targets(payload: dict[str, Any]) -> set[str]: - return {f"{project}:{task}" for project, project_tasks in payload["tasks"].items() for task in project_tasks} - - -def affected(*paths: str) -> set[str]: - payload = moon_json( - "query", - "tasks", - "--affected", - "stdin", - "--upstream", - "none", - "--downstream", - "none", - stdin="".join(f"{path}\n" for path in paths), - ) - return targets(payload) & CI_TASKS - - -def task_graph(target: str) -> dict[str, dict[str, Any]]: - payload = moon_json("task-graph", target, "--json") - return {task["target"]: task for task in payload["data"].values()} - - -def read(path: str) -> str: - return (ROOT / path).read_text(encoding="utf-8") - - -def workflow_step_script(path: str, name: str) -> str: - lines = read(path).splitlines() - start = lines.index(f" - name: {name}") - run = next(index for index in range(start, len(lines)) if lines[index].strip() == "run: |") - end = next( - (index for index in range(run + 1, len(lines)) if lines[index].startswith(" - name:")), - len(lines), - ) - return textwrap.dedent("\n".join(lines[run + 1 : end])).replace("${{ github.repository }}", "NVIDIA/cuda-python") - - -def baseline_artifacts(*, merge_base: str, expired: str | None = None) -> list[dict[str, object]]: - names = ["cuda-pathfinder-wheel", "cuda-python-wheel"] - for version in ("3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15", "3.15t"): - python = version.replace(".", "") - for platform in ("linux-64", "linux-aarch64", "win-64"): - names.append(f"cuda-bindings-python{python}-cuda13.3.0-{platform}-{merge_base}") - names.append(f"cuda-core-python{python}-{platform}-{merge_base}") - return [{"name": name, "expired": name == expired} for name in names] - - -@pytest.mark.skipif(MOON is None, reason="Moon 2.5.1 is required") -@pytest.mark.agent_authored(model="gpt-5") -class TestMoonCi: - def test_workspace_is_pinned_and_visible_inventory_is_exact(self) -> None: - assert run_moon("--version").stdout.strip() == "moon 2.5.1" - payload = moon_json("query", "tasks") - assert {project: set(project_tasks) for project, project_tasks in payload["tasks"].items()} == VISIBLE_TASKS - assert not (targets(payload) & INTERNAL_TASKS) - - def test_internal_inventory_is_hidden_and_rejects_direct_execution(self) -> None: - graph = task_graph("root:ci-fallback") - internal = {target for target, task in graph.items() if task["options"]["internal"]} - assert internal == INTERNAL_TASKS - for target in sorted(INTERNAL_TASKS): - result = run_moon("run", target, "--upstream", "none", "--downstream", "none") - assert result.returncode != 0 - assert "Unknown task" in result.stderr - - def test_all_tasks_disable_caching_and_routes_are_commandless(self) -> None: - visible = moon_json("query", "tasks")["tasks"] - graph = task_graph("root:ci-fallback") - assert all(task["options"]["cache"] is False for tasks in visible.values() for task in tasks.values()) - assert all(task["options"]["cache"] is False for task in graph.values()) - for target in ( - ALL_ROUTES - | ASSET_ROUTES - | { - "root:ci-ignore", - "root:ci-fallback", - "root:test", - } - ): - project, task = target.split(":") - assert visible[project][task]["command"] == "noop" - - def test_semantic_tag_inventory_is_exact(self) -> None: - payload = moon_json("query", "tasks") - actual = { - target: set(payload["tasks"][target.split(":")[0]][target.split(":")[1]].get("tags", [])) - for target in targets(payload) - if payload["tasks"][target.split(":")[0]][target.split(":")[1]].get("tags") - } - expected = { - "pathfinder:wheel": {"ci-wheel-foundation"}, - "bindings:wheel": {"ci-wheel-bindings"}, - "core:wheel": {"ci-wheel-consumers", "ci-wheel-multi-ctk"}, - "metapackage:wheel": {"ci-wheel-consumers"}, - "core:wheel-merge": {"ci-wheel-finalize"}, - "pathfinder:sdist": {"ci-sdist-foundation"}, - "bindings:sdist": {"ci-sdist-bindings"}, - "core:sdist": {"ci-sdist-consumers"}, - "metapackage:sdist": {"ci-sdist-consumers"}, - "bindings:ci-test-assets": {"ci-test-assets-current"}, - "core:ci-test-assets": {"ci-test-assets-current"}, - "core:ci-test-binaries": {"ci-test-assets-previous"}, - "core:api-check": {"ci-api"}, - "root:ci-ignore": {"ci-ignore"}, - "root:ci-fallback": {"ci-force-all"}, - } - expected.update({target: {"ci-test-linux"} for target in LINUX_ROUTES}) - expected.update({target: {"ci-test-windows"} for target in WINDOWS_ROUTES}) - expected.update({f"{project}:docs": {"ci-docs"} for project in VISIBLE_TASKS}) - assert actual == expected - - def test_package_source_impact_routes(self) -> None: - cases = { - "cuda_pathfinder/cuda/pathfinder/__init__.py": PRODUCERS | ALL_ROUTES | ASSET_ROUTES, - "cuda_bindings/cuda/bindings/__init__.py": { - "bindings:wheel", - "bindings:sdist", - "core:wheel", - "core:wheel-merge", - "core:sdist", - "metapackage:wheel", - "metapackage:sdist", - "bindings:ci-test-linux", - "bindings:ci-test-windows", - "core:ci-test-linux", - "core:ci-test-windows", - "metapackage:ci-test-linux", - "metapackage:ci-test-windows", - } - | ASSET_ROUTES, - "cuda_core/cuda/core/__init__.py": { - "core:wheel", - "core:wheel-merge", - "core:sdist", - "core:api-check", - "core:ci-test-linux", - "core:ci-test-windows", - "metapackage:ci-test-linux", - "metapackage:ci-test-windows", - "core:ci-test-assets", - "core:ci-test-binaries", - }, - "cuda_python/pyproject.toml": { - "bindings:wheel", - "bindings:sdist", - "metapackage:wheel", - "metapackage:sdist", - "metapackage:ci-test-linux", - "metapackage:ci-test-windows", - }, - } - cases["cuda_pathfinder/.git_archival.txt"] = cases["cuda_pathfinder/cuda/pathfinder/__init__.py"] - cases["cuda_bindings/.git_archival.txt"] = cases["cuda_bindings/cuda/bindings/__init__.py"] - cases["cuda_core/.git_archival.txt"] = cases["cuda_core/cuda/core/__init__.py"] - for path, expected in cases.items(): - assert affected(path) == expected, path - - def test_tests_helpers_benchmarks_and_os_infrastructure_impact(self) -> None: - cases = { - "cuda_pathfinder/tests/test_pathfinder.py": { - "pathfinder:ci-test-linux", - "pathfinder:ci-test-windows", - }, - "cuda_bindings/examples/0_Introduction/vectorAddDrv.py": { - "bindings:ci-test-linux", - "bindings:ci-test-windows", - "bindings:ci-test-assets", - }, - "cuda_core/tests/test_device.py": { - "core:ci-test-linux", - "core:ci-test-windows", - "core:ci-test-assets", - "core:ci-test-binaries", - }, - "cuda_python_test_helpers/pyproject.toml": ALL_ROUTES | ASSET_ROUTES, - "benchmarks/cuda_bindings/run_pyperf.py": ALL_ROUTES | ASSET_ROUTES, - "benchmarks/cuda_bindings/compare.py": ALL_ROUTES | ASSET_ROUTES, - "benchmarks/cuda_core/runtime.py": ALL_ROUTES | ASSET_ROUTES, - ".github/workflows/test-wheel-linux.yml": LINUX_ROUTES | ASSET_ROUTES, - ".github/workflows/test-wheel-windows.yml": WINDOWS_ROUTES | ASSET_ROUTES, - "ci/tools/guess_latest.sh": LINUX_ROUTES | ASSET_ROUTES, - } - for path, expected in cases.items(): - assert affected(path) == expected, path - - def test_docs_ignored_unknown_and_fallback_ownership(self) -> None: - assert affected("cuda_core/docs/source/index.rst") == {"core:docs"} - for path in ( - ".coveragerc", - ".github/ISSUE_TEMPLATE/bug_report.yml", - ".github/labeler.yml", - ".pre-commit-config.yaml", - "CONTRIBUTING.md", - "context7.json", - "cuda_core/pixi.toml", - "cuda_core/tests/AGENTS.md", - "diagram.svg", - "greptile.json", - "new-area/pixi.lock", - "ruff.toml", - "toolshed/README.md", - ): - assert affected(path) == {"root:ci-ignore"} - assert affected(".github/workflows/ci.yml") == {"root:ci-fallback"} - assert affected("an-entirely-new-path.txt") == set() - gate = read(".github/workflows/ci.yml") - assert 'length == 0 or any(.[]; .target == "root:ci-fallback")' in gate - - fallback = task_graph("root:ci-fallback")["root:ci-fallback"] - assert {dep["target"] for dep in fallback["deps"]} == ( - PRODUCERS | ALL_ROUTES | ASSET_ROUTES | {"core:api-check", "root:docs"} - ) - for path in (".moon/workspace.yml", "moon.yml", "cuda_core/moon.yml", "ci/versions.yml"): - assert "root:ci-fallback" in affected(path) - - def test_mixed_changes_and_symlink_consumers(self) -> None: - assert affected("cuda_core/docs/source/index.rst", "cuda_bindings/tests/test_api.py") == { - "core:docs", - "bindings:ci-test-linux", - "bindings:ci-test-windows", - "bindings:ci-test-assets", - } - expected_readme = { - "bindings:wheel", - "bindings:sdist", - "metapackage:wheel", - "metapackage:sdist", - "metapackage:ci-test-linux", - "metapackage:ci-test-windows", - } - assert affected("README.md") == expected_readme - assert affected("cuda_python/README.md") == expected_readme - assert affected(".git_archival.txt") == PRODUCERS | ALL_ROUTES | ASSET_ROUTES | {"core:api-check"} - - def test_editable_installs_are_first_class_dependencies(self) -> None: - expected_install_deps = { - "pathfinder:install": set(), - "bindings:install": {"pathfinder:install"}, - "core:install": {"bindings:install"}, - } - graph = task_graph("core:install") - assert set(graph) == set(expected_install_deps) - for target, expected in expected_install_deps.items(): - assert {dep["target"] for dep in graph[target].get("deps", [])} == expected - script = graph[target]["script"] - assert "pip install -e ." in script - assert "../cuda_" not in script - - expected_test_graphs = { - "pathfinder:test": {"pathfinder:install", "pathfinder:test"}, - "bindings:test": {"pathfinder:install", "bindings:install", "bindings:test"}, - "core:test": { - "pathfinder:install", - "bindings:install", - "core:install", - "core:test", - }, - } - for target, expected in expected_test_graphs.items(): - graph = task_graph(target) - assert set(graph) == expected - test_script = graph[target]["script"] - assert "pip install" not in test_script - assert all("wheel" not in graph_target for graph_target in graph) - assert all("cibuildwheel" not in task["script"] for task in graph.values()) - - root_test_graph = task_graph("root:test") - assert {dep["target"] for dep in root_test_graph["root:test"]["deps"]} == { - "pathfinder:test", - "bindings:test", - "core:test", - } - assert root_test_graph["root:test"]["options"]["runDepsInParallel"] is True - - benchmark_graph = task_graph("bindings:benchmark") - assert set(benchmark_graph) == { - "pathfinder:install", - "bindings:install", - "bindings:benchmark", - } - assert "pip install" not in benchmark_graph["bindings:benchmark"]["script"] - - def test_local_core_wheel_builds_current_dependency_chain(self) -> None: - assert set(task_graph("core:wheel")) == { - "pathfinder:wheel", - "bindings:wheel", - "core:wheel", - } - - def test_ci_routes_have_only_hidden_direct_executors(self) -> None: - expected = { - "pathfinder:ci-test-linux": {"pathfinder:test-installed"}, - "pathfinder:ci-test-windows": {"pathfinder:test-installed"}, - "bindings:ci-test-linux": {"bindings:test-installed", "bindings:benchmark-smoke"}, - "bindings:ci-test-windows": {"bindings:test-installed"}, - "core:ci-test-linux": {"core:test-installed"}, - "core:ci-test-windows": {"core:test-installed"}, - "metapackage:ci-test-linux": {"metapackage:test-installed"}, - "metapackage:ci-test-windows": {"metapackage:test-installed"}, - } - fallback = task_graph("root:ci-fallback") - for route, direct_targets in expected.items(): - actual = {dep["target"] for dep in fallback[route]["deps"]} - assert actual == direct_targets - assert all(fallback[target]["options"]["internal"] for target in actual) - assert all(fallback[target].get("deps") for target in actual) - for workflow in (".github/workflows/test-wheel-linux.yml", ".github/workflows/test-wheel-windows.yml"): - assert 'moon run "${target_args[@]}" --upstream direct --downstream none' in read(workflow) - - def test_build_traversal_stages_dependencies_and_runs_exact_targets(self) -> None: - workflow = read(".github/workflows/build-wheel.yml") - for phase in ( - "WHEEL_FOUNDATION_TARGETS", - "WHEEL_BINDINGS_TARGETS", - "WHEEL_CONSUMER_TARGETS", - "WHEEL_MULTI_CTK_TARGETS", - "WHEEL_FINALIZE_TARGETS", - ): - assert phase in workflow - assert workflow.count("--upstream none --downstream none") >= 5 - assert workflow.count("--upstream direct --downstream none") >= 2 - assert "Download reusable cuda.pathfinder wheel" in workflow - assert "Download reusable cuda.bindings wheel" in workflow - assert workflow.count("python -m pip install cibuildwheel twine wheel") == 2 - - def test_native_assets_follow_the_selected_os(self, tmp_path: Path) -> None: - assert BASH is not None - script = workflow_step_script(".github/workflows/build-wheel.yml", "Resolve Moon phase targets") - common = { - "WHEEL_FOUNDATION_TARGETS": "[]", - "WHEEL_BINDINGS_TARGETS": "[]", - "WHEEL_CONSUMER_TARGETS": "[]", - "WHEEL_MULTI_CTK_TARGETS": "[]", - "WHEEL_FINALIZE_TARGETS": "[]", - "TEST_ASSETS_CURRENT_TARGETS": '["bindings:ci-test-assets","core:ci-test-assets"]', - "TEST_ASSETS_PREVIOUS_TARGETS": '["core:ci-test-binaries"]', - } - cases = { - "linux-selected": { - "TEST_LINUX_TARGETS": '["core:ci-test-linux"]', - "TEST_WINDOWS_TARGETS": "[]", - "linux-64": "true", - "win-64": "false", - }, - "windows-selected": { - "TEST_LINUX_TARGETS": "[]", - "TEST_WINDOWS_TARGETS": '["core:ci-test-windows"]', - "linux-64": "false", - "win-64": "true", - }, - } - for case_name, case in cases.items(): - for platform in ("linux-64", "win-64"): - output = tmp_path / f"{case_name}-{platform}.env" - env = ( - os.environ - | common - | { - "HOST_PLATFORM": platform, - "GITHUB_ENV": str(output), - "TEST_LINUX_TARGETS": case["TEST_LINUX_TARGETS"], - "TEST_WINDOWS_TARGETS": case["TEST_WINDOWS_TARGETS"], - } - ) - result = subprocess.run( # noqa: S603 - controlled repository script. - [BASH, "-c", script], - cwd=ROOT, - env=env, - text=True, - check=False, - capture_output=True, - ) - assert result.returncode == 0, (case_name, platform, result.stderr) - values = dict(line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines()) - assert values["TEST_BINDINGS"] == case[platform] - assert values["TEST_CORE_CURRENT"] == case[platform] - assert values["TEST_CORE_PREVIOUS"] == case[platform] - - workflow = read(".github/workflows/ci.yml") - linux_arm = workflow.split(" build-linux-aarch64:", 1)[1].split(" build-windows:", 1)[0] - windows = workflow.split(" build-windows:", 1)[1].split(" test-sdist-linux:", 1)[0] - assert "ci-test-assets" not in linux_arm - assert "ci-test-assets" not in windows - - def test_core_uses_one_target_in_both_toolkits_then_merges(self) -> None: - graph = task_graph("root:ci-fallback") - assert set(graph["core:wheel"]["tags"]) == {"ci-wheel-consumers", "ci-wheel-multi-ctk"} - assert not graph["core:wheel-merge"].get("deps") - merger = graph["core:wheel-merge"]["script"] - assert "dist/cu12/*.whl dist/cu13/*.whl" in merger - assert "merge_cuda_core_wheels.py" in merger - - def test_only_merged_core_wheel_is_in_baseline_artifact(self) -> None: - workflow = read(".github/workflows/build-wheel.yml") - assert "name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}" in workflow - assert "path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl" in workflow - assert "path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu" not in workflow - for name in ("cuda-pathfinder-wheel", "cuda-python-wheel"): - assert f"name: {name}" in workflow - - def test_baseline_reuse_requires_one_exact_successful_complete_set(self) -> None: - workflow = read(".github/workflows/ci.yml") - for contract in ( - '--commit "${merge_base}"', - "--event push", - "--status success", - "if [[ $(jq 'length' <<< \"$runs\") -ne 1 ]]", - '"${run_sha}" != "${merge_base}"', - "length == 1 and .[0].expired == false", - "if (( ${#missing[@]} != 0 ))", - 'baseline_run_id=""', - 'baseline_sha=""', - ): - assert contract in workflow - assert "cuda-pathfinder-wheel cuda-python-wheel" in workflow - assert "CUDA_BINDINGS_ARTIFACT_BASENAME" in read(".github/workflows/build-wheel.yml") - assert "CUDA_CORE_ARTIFACT_BASENAME" in read(".github/workflows/build-wheel.yml") - assert "uvx --from pytest pytest -q tests/test_moon_ci.py" in workflow - - def test_baseline_reuse_behaviors(self, tmp_path: Path) -> None: - assert BASH is not None - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - gh = fake_bin / "gh" - gh.write_text( - """#!/usr/bin/env bash -printf '%s\\n' "$*" >> "$MOCK_GH_LOG" -if [[ "$1 $2" == "run list" ]]; then - printf '%s\\n' "$MOCK_RUNS" - exit "$MOCK_RUN_STATUS" -fi -if [[ "$1" == "api" ]]; then - printf '%s\\n' "$MOCK_ARTIFACTS" - exit "$MOCK_ARTIFACT_STATUS" -fi -exit 2 -""", - encoding="utf-8", - ) - yq = fake_bin / "yq" - yq.write_text( - """#!/usr/bin/env bash -if [[ "$1" == "-r" ]]; then - printf '%s\\n' 3.10 3.11 3.12 3.13 3.14 3.14t 3.15 3.15t -else - printf '%s\\n' 13.3.0 -fi -""", - encoding="utf-8", - ) - os.chmod(gh, 0o700) - os.chmod(yq, 0o700) - - merge_base = "exact-base" - complete = baseline_artifacts(merge_base=merge_base) - cases = { - "complete": { - "runs": [{"databaseId": 42, "headSha": merge_base}], - "artifacts": complete, - "accepted": True, - }, - "incomplete": { - "runs": [{"databaseId": 42, "headSha": merge_base}], - "artifacts": complete[:-1], - "accepted": False, - }, - "expired": { - "runs": [{"databaseId": 42, "headSha": merge_base}], - "artifacts": baseline_artifacts(merge_base=merge_base, expired="cuda-pathfinder-wheel"), - "accepted": False, - }, - "failed-run": {"runs": [], "artifacts": complete, "accepted": False}, - "wrong-sha": { - "runs": [{"databaseId": 42, "headSha": "another-sha"}], - "artifacts": complete, - "accepted": False, - }, - "duplicate": { - "runs": [{"databaseId": 42, "headSha": merge_base}], - "artifacts": [*complete, complete[0]], - "accepted": False, - }, - "lookup-failure": { - "runs": [{"databaseId": 42, "headSha": merge_base}], - "artifacts": complete, - "accepted": False, - "run_status": 1, - }, - } - script = workflow_step_script(".github/workflows/ci.yml", "Resolve reusable base artifacts") - for name, case in cases.items(): - output = tmp_path / f"{name}.output" - summary = tmp_path / f"{name}.summary" - log = tmp_path / f"{name}.gh.log" - output.touch() - summary.touch() - env = os.environ | { - "PATH": f"{fake_bin}:{os.environ['PATH']}", - "BASE_REF": "main", - "MERGE_BASE": merge_base, - "GITHUB_OUTPUT": str(output), - "GITHUB_STEP_SUMMARY": str(summary), - "MOCK_GH_LOG": str(log), - "MOCK_RUNS": json.dumps(case["runs"]), - "MOCK_ARTIFACTS": "\n".join(json.dumps(item) for item in case["artifacts"]), - "MOCK_RUN_STATUS": str(case.get("run_status", 0)), - "MOCK_ARTIFACT_STATUS": "0", - } - result = subprocess.run( # noqa: S603 - controlled script and fake tools. - [BASH, "-c", script], - cwd=ROOT, - env=env, - text=True, - check=False, - capture_output=True, - ) - assert result.returncode == 0, (name, result.stderr) - accepted = "run_id=42" in output.read_text(encoding="utf-8") - assert accepted is case["accepted"], name - if not case["accepted"]: - assert "No complete reusable artifact set" in summary.read_text(encoding="utf-8") - - complete_log = (tmp_path / "complete.gh.log").read_text(encoding="utf-8") - for argument in ("--commit exact-base", "--event push", "--status success"): - assert argument in complete_log - - def test_docs_select_component_or_parallel_aggregate_layout(self) -> None: - workflow = read(".github/workflows/build-docs.yml") - assert "all) targets='[\"root:docs\"]'" in workflow - for project in ("pathfinder", "bindings", "core", "metapackage"): - assert f'"{project}:docs"' in workflow - assert "DOCS_BUILD_ARGS" in workflow - assert "--upstream deep --downstream none" in workflow - assert "DOCS_USE_MOON" in workflow - assert "./build_all_docs.sh latest-only" in workflow - assert "./build_docs.sh latest-only" in workflow - root_docs = task_graph("root:docs") - assert {dep["target"] for dep in root_docs["root:docs"]["deps"]} == { - "pathfinder:docs", - "bindings:docs", - "core:docs", - "metapackage:docs", - } - script = root_docs["root:docs"]["script"] - for destination in ("cuda-bindings", "cuda-core", "cuda-pathfinder"): - assert f"cuda_python/docs/build/html/{destination}" in script - for project in ("pathfinder", "bindings", "core", "metapackage"): - assert "${DOCS_BUILD_ARGS:-}" in root_docs[f"{project}:docs"]["script"] diff --git a/toolshed/check_spdx.py b/toolshed/check_spdx.py index ce422aef997..d4c9430673c 100644 --- a/toolshed/check_spdx.py +++ b/toolshed/check_spdx.py @@ -23,7 +23,6 @@ TOP_LEVEL_DIRS_LICENSE_IDENTIFIERS = { ".agents": "Apache-2.0", ".github": "Apache-2.0", - ".moon": "Apache-2.0", "benchmarks": "Apache-2.0", "ci": "Apache-2.0", "cuda_bindings": "Apache-2.0", @@ -33,7 +32,6 @@ "cuda_python_test_helpers": "Apache-2.0", "qa": "LicenseRef-NVIDIA-SOFTWARE-LICENSE", "scripts": "Apache-2.0", - "tests": "Apache-2.0", "toolshed": "Apache-2.0", }