diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 7bb70809556..8f3c58ed67a 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -18,6 +18,11 @@ on: # - cuda-python # - cuda-pathfinder # - all + targets: + description: "JSON array of exact Moon docs targets. Empty derives targets from component." + required: false + default: "" + type: string git-tag: description: "Target git tag to build docs for" required: false @@ -60,6 +65,39 @@ jobs: fetch-depth: 1 ref: ${{ inputs.git-tag }} + - name: Resolve Moon docs targets + env: + REQUESTED_TARGETS: ${{ inputs.targets }} + COMPONENT: ${{ inputs.component }} + run: | + if [[ -n "$REQUESTED_TARGETS" ]]; then + targets="$REQUESTED_TARGETS" + else + case "$COMPONENT" in + all) targets='["root:docs"]' ;; + cuda-pathfinder) targets='["pathfinder:docs"]' ;; + cuda-bindings) targets='["bindings:docs"]' ;; + cuda-core) targets='["core:docs"]' ;; + cuda-python) targets='["metapackage:docs"]' ;; + *) echo "error: unsupported docs component: $COMPONENT" >&2; exit 1 ;; + esac + fi + jq -e ' + type == "array" and length > 0 and + all(.[]; + . == "root:docs" or + . == "pathfinder:docs" or + . == "bindings:docs" or + . == "core:docs" or + . == "metapackage:docs") + ' <<< "$targets" >/dev/null + echo "DOCS_TARGETS=$(jq -c . <<< "$targets")" >> "$GITHUB_ENV" + if [[ -f .moon/workspace.yml && -f moon.yml ]]; then + echo "DOCS_USE_MOON=true" >> "$GITHUB_ENV" + else + echo "DOCS_USE_MOON=false" >> "$GITHUB_ENV" + fi + - name: Read build CTK version run: | if [[ -f ci/versions.yml ]]; then @@ -94,6 +132,12 @@ jobs: conda config --show-sources conda config --show + - name: Install Moon + if: ${{ env.DOCS_USE_MOON == 'true' }} + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + # WAR: Building the doc currently requires CTK installed (NVIDIA/cuda-python#326,327) - name: Set up mini CTK uses: ./.github/actions/fetch_ctk @@ -132,7 +176,7 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel - path: . + path: ./cuda_python/dist run-id: ${{ inputs.run-id }} github-token: ${{ github.token }} @@ -145,7 +189,7 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel - path: ./cuda_pathfinder + path: ./cuda_pathfinder/dist run-id: ${{ inputs.run-id }} github-token: ${{ github.token }} @@ -200,7 +244,7 @@ jobs: - name: Install all packages run: | - pushd cuda_pathfinder + pushd cuda_pathfinder/dist pip install *.whl popd @@ -214,7 +258,7 @@ jobs: # Subpackages are already installed from CI artifacts above. # --no-deps avoids re-resolving cuda-core from PyPI during tag releases. - pip install --no-deps cuda_python*.whl + pip install --no-deps cuda_python/dist/*.whl # This step sets the PR_NUMBER/BUILD_LATEST/BUILD_PREVIEW env vars. - name: Get PR number @@ -227,42 +271,79 @@ jobs: # create an empty folder for removal use mkdir -p artifacts/empty_docs - - name: Build all docs - if: ${{ inputs.component == 'all' }} + - name: Build selected docs + if: ${{ env.DOCS_USE_MOON == 'true' }} + env: + DOCS_BUILD_ARGS: ${{ !inputs.is-release && 'latest-only' || '' }} run: | - pushd cuda_python/docs/ - if [[ "${{ inputs.is-release }}" == "false" ]]; then - ./build_all_docs.sh latest-only - else - ./build_all_docs.sh - # At release time, we don't want to update the latest docs - rm -rf build/html/latest - fi - ls -l build - popd - mv cuda_python/docs/build/html/* artifacts/docs/ + mapfile -t targets < <(jq -r '.[]' <<< "$DOCS_TARGETS") + moon run "${targets[@]}" --upstream deep --downstream none - - name: Build component docs - if: ${{ inputs.component != 'all' }} + # Release workflows may check out tags created before Moon was added. + - name: Build selected docs from a legacy tag + if: ${{ env.DOCS_USE_MOON != 'true' }} run: | - COMPONENT=$(echo "${{ inputs.component }}" | tr '-' '_') - pushd ${COMPONENT}/docs/ - if [[ "${{ inputs.is-release }}" == "false" ]]; then - ./build_docs.sh latest-only + if [[ "${{ inputs.component }}" == "all" ]]; then + pushd cuda_python/docs + if [[ "${{ inputs.is-release }}" == "false" ]]; then + ./build_all_docs.sh latest-only + else + ./build_all_docs.sh + rm -rf build/html/latest + fi + popd else - ./build_docs.sh - # At release time, we don't want to update the latest docs - rm -rf build/html/latest + component="${{ inputs.component }}" + component=${component//-/_} + pushd "$component/docs" + if [[ "${{ inputs.is-release }}" == "false" ]]; then + ./build_docs.sh latest-only + else + ./build_docs.sh + rm -rf build/html/latest + fi + popd fi - ls -l build - popd - if [[ "${{ inputs.component }}" != "cuda-python" ]]; then - TARGET="${{ inputs.component }}" - mkdir -p artifacts/docs/${TARGET} - else - TARGET="" + + - name: Assemble selected docs + run: | + if jq -e 'index("root:docs") != null' <<< "$DOCS_TARGETS" >/dev/null; then + if [[ "${{ inputs.is-release }}" == "true" ]]; then + rm -rf cuda_python/docs/build/html/latest + fi + ls -l cuda_python/docs/build + mv cuda_python/docs/build/html/* artifacts/docs/ + exit 0 fi - mv ${COMPONENT}/docs/build/html/* artifacts/docs/${TARGET} + + while IFS= read -r target; do + case "$target" in + pathfinder:docs) + component=cuda_pathfinder + destination=cuda-pathfinder + ;; + bindings:docs) + component=cuda_bindings + destination=cuda-bindings + ;; + core:docs) + component=cuda_core + destination=cuda-core + ;; + metapackage:docs) + component=cuda_python + destination= + ;; + esac + if [[ "${{ inputs.is-release }}" == "true" ]]; then + rm -rf "$component/docs/build/html/latest" + fi + ls -l "$component/docs/build" + if [[ -n "$destination" ]]; then + mkdir -p "artifacts/docs/$destination" + fi + mv "$component"/docs/build/html/* "artifacts/docs/$destination" + done < <(jq -r '.[]' <<< "$DOCS_TARGETS") - name: Write rendered docs file list if: ${{ !inputs.is-release && github.ref_name != 'main' && !startsWith(github.ref_name, 'release/') }} diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 7fde2419e58..2285f8e985d 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -14,35 +14,8 @@ on: prev-cuda-version: required: true type: string - build-pathfinder: - required: false - type: boolean - default: true - build-bindings: - required: false - type: boolean - default: true - build-core: - required: false - type: boolean - default: true - build-python: - required: false - type: boolean - default: true - test-bindings: - required: false - type: boolean - default: true - test-core: - required: false - type: boolean - default: true - baseline-run-id: - required: false - type: string - default: "" - baseline-sha: + workplan: + description: JSON workplan. An empty value builds and tests everything. required: false type: string default: "" @@ -57,6 +30,19 @@ permissions: jobs: build: + env: + WHEEL_FOUNDATION_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-foundation']) || '["pathfinder:wheel"]' }} + WHEEL_BINDINGS_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-bindings']) || '["bindings:wheel"]' }} + WHEEL_CONSUMER_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-consumers']) || '["core:wheel","metapackage:wheel"]' }} + WHEEL_MULTI_CTK_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-multi-ctk']) || '["core:wheel"]' }} + WHEEL_FINALIZE_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-wheel-finalize']) || '["core:wheel-merge"]' }} + TEST_ASSETS_CURRENT_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-test-assets-current']) || '["bindings:ci-test-assets","core:ci-test-assets"]' }} + TEST_ASSETS_PREVIOUS_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-test-assets-previous']) || '["core:ci-test-binaries"]' }} + TEST_LINUX_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-test-linux']) || '["pathfinder:ci-test-linux"]' }} + TEST_WINDOWS_TARGETS: ${{ inputs.workplan != '' && toJSON(fromJSON(inputs.workplan).targets['ci-test-windows']) || '["pathfinder:ci-test-windows"]' }} + HOST_PLATFORM: ${{ inputs.host-platform }} + BASELINE_RUN_ID: ${{ inputs.workplan != '' && fromJSON(inputs.workplan).baseline.run_id || '' }} + BASELINE_SHA: ${{ inputs.workplan != '' && fromJSON(inputs.workplan).baseline.sha || '' }} strategy: fail-fast: false matrix: @@ -82,8 +68,65 @@ jobs: fetch-depth: 0 filter: blob:none + - name: Install target resolver dependencies + if: ${{ startsWith(inputs.host-platform, 'linux') }} + uses: ./.github/actions/install_unix_deps + with: + dependencies: "jq" + dependent_exes: "jq" + + - name: Resolve Moon phase targets + run: | + for value in \ + "$WHEEL_FOUNDATION_TARGETS" \ + "$WHEEL_BINDINGS_TARGETS" \ + "$WHEEL_CONSUMER_TARGETS" \ + "$WHEEL_MULTI_CTK_TARGETS" \ + "$WHEEL_FINALIZE_TARGETS" \ + "$TEST_ASSETS_CURRENT_TARGETS" \ + "$TEST_ASSETS_PREVIOUS_TARGETS" \ + "$TEST_LINUX_TARGETS" \ + "$TEST_WINDOWS_TARGETS"; do + jq -e 'type == "array" and all(.[]; type == "string")' <<< "$value" >/dev/null + done + + has_target() { + jq -e --arg target "$2" 'index($target) != null' <<< "$1" >/dev/null + } + if [[ "$HOST_PLATFORM" == "win-64" ]]; then + platform_test_targets="$TEST_WINDOWS_TARGETS" + else + platform_test_targets="$TEST_LINUX_TARGETS" + fi + run_test_assets=$(jq -r 'length > 0' <<< "$platform_test_targets") + { + echo "BUILD_PATHFINDER=$(has_target "$WHEEL_FOUNDATION_TARGETS" pathfinder:wheel && echo true || echo false)" + echo "BUILD_BINDINGS=$(has_target "$WHEEL_BINDINGS_TARGETS" bindings:wheel && echo true || echo false)" + echo "BUILD_CORE_CURRENT=$(has_target "$WHEEL_CONSUMER_TARGETS" core:wheel && echo true || echo false)" + echo "BUILD_CORE_PREVIOUS=$(has_target "$WHEEL_MULTI_CTK_TARGETS" core:wheel && echo true || echo false)" + echo "FINALIZE_CORE=$(has_target "$WHEEL_FINALIZE_TARGETS" core:wheel-merge && echo true || echo false)" + if has_target "$WHEEL_CONSUMER_TARGETS" core:wheel || \ + has_target "$WHEEL_MULTI_CTK_TARGETS" core:wheel || \ + has_target "$WHEEL_FINALIZE_TARGETS" core:wheel-merge; then + echo "BUILD_CORE=true" + else + echo "BUILD_CORE=false" + fi + echo "BUILD_PYTHON=$(has_target "$WHEEL_CONSUMER_TARGETS" metapackage:wheel && echo true || echo false)" + echo "TEST_BINDINGS=$(if [[ "$run_test_assets" == "true" ]]; then has_target "$TEST_ASSETS_CURRENT_TARGETS" bindings:ci-test-assets && echo true || echo false; else echo false; fi)" + echo "TEST_CORE_CURRENT=$(if [[ "$run_test_assets" == "true" ]]; then has_target "$TEST_ASSETS_CURRENT_TARGETS" core:ci-test-assets && echo true || echo false; else echo false; fi)" + echo "TEST_CORE_PREVIOUS=$(if [[ "$run_test_assets" == "true" ]]; then has_target "$TEST_ASSETS_PREVIOUS_TARGETS" core:ci-test-binaries && echo true || echo false; else echo false; fi)" + if [[ "$run_test_assets" == "true" ]] && \ + (has_target "$TEST_ASSETS_CURRENT_TARGETS" core:ci-test-assets || \ + has_target "$TEST_ASSETS_PREVIOUS_TARGETS" core:ci-test-binaries); then + echo "TEST_CORE=true" + else + echo "TEST_CORE=false" + fi + } >> "$GITHUB_ENV" + - name: Install latest rapidsai/sccache - if: ${{ startsWith(inputs.host-platform, 'linux') && (inputs.build-bindings || inputs.build-core) }} + if: ${{ startsWith(inputs.host-platform, 'linux') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} run: | curl -fsSL "https://github.com/rapidsai/sccache/releases/latest/download/sccache-$(uname -m)-unknown-linux-musl.tar.gz" \ | sudo tar -C /usr/local/bin -xvzf - --wildcards --strip-components=1 -x '*/sccache' @@ -91,7 +134,7 @@ jobs: # xref: https://github.com/orgs/community/discussions/42856#discussioncomment-7678867 - name: Adding addtional GHA cache-related env vars - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: actions/github-script@v9 with: script: | @@ -117,14 +160,20 @@ jobs: # see https://github.com/actions/setup-python/issues/871 python-version: "3.12" + - name: Install Moon and build tools + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + python -m pip install cibuildwheel twine wheel + - name: Set up MSVC - if: ${{ startsWith(inputs.host-platform, 'win') && (inputs.build-bindings || inputs.build-core || inputs.test-bindings || inputs.test-core) }} + if: ${{ startsWith(inputs.host-platform, 'win') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') }} uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 - name: Set up yq # GitHub made an unprofessional decision to not provide it in their Windows VMs, # see https://github.com/actions/runner-images/issues/7443. - if: ${{ startsWith(inputs.host-platform, 'win') && inputs.build-core }} + if: ${{ startsWith(inputs.host-platform, 'win') && env.BUILD_CORE == 'true' }} env: YQ_VERSION: v4.52.5 YQ_SHA256: 47594981f3848a4b4447494adeca9555f908f7cf0a89c4da3fd0243a4631da1c @@ -156,26 +205,21 @@ jobs: run: | env - - name: Install twine - run: | - pip install twine - # To keep the build workflow simple, all matrix jobs will build a wheel for later use within this workflow. - name: Build and check cuda.pathfinder wheel - if: ${{ inputs.build-pathfinder }} + if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | - pushd cuda_pathfinder - pip wheel -v --no-deps . - popd + mapfile -t targets < <(jq -r '.[]' <<< "$WHEEL_FOUNDATION_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Download reusable cuda.pathfinder wheel - if: ${{ !inputs.build-pathfinder }} + if: ${{ env.BUILD_PATHFINDER != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel - path: cuda_pathfinder + path: cuda_pathfinder/dist github-token: ${{ github.token }} - run-id: ${{ inputs.baseline-run-id }} + run-id: ${{ env.BASELINE_RUN_ID }} - name: List the cuda.pathfinder artifacts directory run: | @@ -184,20 +228,20 @@ jobs: else export CHOWN="sudo chown" fi - $CHOWN -R $(whoami) cuda_pathfinder/*.whl - ls -lahR cuda_pathfinder + $CHOWN -R $(whoami) cuda_pathfinder/dist/*.whl + ls -lahR cuda_pathfinder/dist # We only need/want a single pure python wheel, pick linux-64 index 0. # This is what we will use for testing & releasing. - name: Check cuda.pathfinder wheel - if: ${{ inputs.build-pathfinder && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} + if: ${{ env.BUILD_PATHFINDER == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} run: | - twine check --strict cuda_pathfinder/*.whl + twine check --strict cuda_pathfinder/dist/*.whl - name: Constrain builds to the local cuda.pathfinder wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | - pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) + pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test -f "${pathfinder_wheels[0]}" mkdir -p wheel-constraints @@ -213,11 +257,11 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: cuda-pathfinder-wheel - path: cuda_pathfinder/*.whl + path: cuda_pathfinder/dist/*.whl if-no-files-found: error - name: Set up mini CTK - if: ${{ inputs.build-bindings || inputs.build-core || inputs.test-bindings || inputs.test-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -225,11 +269,10 @@ jobs: cuda-version: ${{ inputs.cuda-version }} - name: Build cuda.bindings wheel - if: ${{ inputs.build-bindings }} - uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 - with: - package-dir: ./cuda_bindings/ - output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} + if: ${{ env.BUILD_BINDINGS == 'true' }} + run: | + mapfile -t targets < <(jq -r '.[]' <<< "$WHEEL_BINDINGS_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none env: CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"' @@ -271,7 +314,7 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.bindings) - if: ${{ inputs.build-bindings && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_BINDINGS == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_bindings.json @@ -279,13 +322,13 @@ jobs: build-step: "Build cuda.bindings wheel" - name: Download reusable cuda.bindings wheel - if: ${{ !inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ env.CUDA_BINDINGS_ARTIFACT_BASENAME }}-${{ inputs.baseline-sha }} + name: ${{ env.CUDA_BINDINGS_ARTIFACT_BASENAME }}-${{ env.BASELINE_SHA }} path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} github-token: ${{ github.token }} - run-id: ${{ inputs.baseline-run-id }} + run-id: ${{ env.BASELINE_RUN_ID }} - name: List the cuda.bindings artifacts directory run: | @@ -298,14 +341,14 @@ jobs: ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} - name: Check cuda.bindings wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | twine check --strict ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl - name: Constrain cuda.core to the local cuda.bindings wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE_CURRENT == 'true' }} run: | - pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) + pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) bindings_wheels=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-"${BUILD_CUDA_MAJOR}".*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test "${#bindings_wheels[@]}" -eq 1 @@ -331,13 +374,19 @@ jobs: path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl if-no-files-found: error - - name: Build cuda.core wheel - if: ${{ inputs.build-core }} - uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 - with: - package-dir: ./cuda_core/ - output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + - name: Build current-context consumer wheels + if: ${{ env.BUILD_CORE_CURRENT == 'true' || (env.BUILD_PYTHON == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64') }} + run: | + targets="$WHEEL_CONSUMER_TARGETS" + if [[ "${{ inputs.host-platform }}" != "linux-64" || "${{ strategy.job-index }}" != "0" ]]; then + targets=$(jq -c 'map(select(. != "metapackage:wheel"))' <<< "$targets") + fi + mapfile -t target_args < <(jq -r '.[]' <<< "$targets") + if (( ${#target_args[@]} != 0 )); then + moon run "${target_args[@]}" --upstream none --downstream none + fi env: + CUDA_CORE_BUILD_MAJOR: ${{ env.BUILD_CUDA_MAJOR }} CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"' CIBW_BEFORE_BUILD_WINDOWS: 'python -m pip install --upgrade "pip>=25.3" delvewheel' @@ -380,15 +429,15 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.core) - if: ${{ inputs.build-core && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_CORE_CURRENT == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_core.json label: "cuda.core" build-step: "Build cuda.core wheel" - - name: List the cuda.core artifacts directory and rename - if: ${{ inputs.build-core }} + - name: List the cuda.core artifacts directory + if: ${{ env.BUILD_CORE_CURRENT == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == win* ]]; then export CHOWN=chown @@ -396,46 +445,25 @@ jobs: export CHOWN="sudo chown" fi $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - - # Rename wheel to include CUDA version suffix - mkdir -p "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}" - for wheel in ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl; do - if [[ -f "${wheel}" ]]; then - base_name=$(basename "${wheel}" .whl) - new_name="${base_name}.cu${BUILD_CUDA_MAJOR}.whl" - mv "${wheel}" "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}/${new_name}" - echo "Renamed wheel to: ${new_name}" - fi - done - ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - name: Download reusable cuda.core wheel - if: ${{ !inputs.build-core }} + if: ${{ env.BUILD_CORE != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ env.CUDA_CORE_ARTIFACT_BASENAME }}-${{ inputs.baseline-sha }} + name: ${{ env.CUDA_CORE_ARTIFACT_BASENAME }}-${{ env.BASELINE_SHA }} path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} github-token: ${{ github.token }} - run-id: ${{ inputs.baseline-run-id }} - - # We only need/want a single pure python wheel, pick linux-64 index 0. - - name: Build and check cuda-python wheel - if: ${{ inputs.build-python && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} - run: | - pushd cuda_python - pip wheel -v --no-deps . - twine check --strict *.whl - popd + run-id: ${{ env.BASELINE_RUN_ID }} - name: Download reusable cuda-python wheel - if: ${{ !inputs.build-python && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} + if: ${{ env.BUILD_PYTHON != 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel - path: cuda_python + path: cuda_python/dist github-token: ${{ github.token }} - run-id: ${{ inputs.baseline-run-id }} + run-id: ${{ env.BASELINE_RUN_ID }} - name: List the cuda-python artifacts directory if: ${{ strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} @@ -445,38 +473,42 @@ jobs: else export CHOWN="sudo chown" fi - $CHOWN -R $(whoami) cuda_python/*.whl - ls -lahR cuda_python + $CHOWN -R $(whoami) cuda_python/dist/*.whl + ls -lahR cuda_python/dist - name: Upload cuda-python build artifacts if: ${{ strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: cuda-python-wheel - path: cuda_python/*.whl + path: cuda_python/dist/*.whl if-no-files-found: error - name: Set up Python id: setup-python2 - if: ${{ inputs.test-bindings || inputs.test-core }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} # TODO: remove allow-prereleases once 3.15 is officially supported allow-prereleases: ${{ startsWith(matrix.python-version, '3.15') }} + - name: Reinstall build tools for the selected Python + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} + run: python -m pip install cibuildwheel twine wheel + - name: Enable Scientific Python Nightly Wheels for Python 3.15 - if: ${{ (inputs.test-bindings || inputs.test-core) && startsWith(matrix.python-version, '3.15') }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true') && startsWith(matrix.python-version, '3.15') }} run: | echo "PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" >> "$GITHUB_ENV" echo "PIP_ONLY_BINARY=numpy" >> "$GITHUB_ENV" - name: verify free-threaded build - if: ${{ (inputs.test-bindings || inputs.test-core) && endsWith(matrix.python-version, 't') }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true') && endsWith(matrix.python-version, 't') }} run: python -c 'import sys; assert not sys._is_gil_enabled()' - name: Set up Python include paths - if: ${{ inputs.test-bindings || inputs.test-core }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == linux* ]]; then echo "CPLUS_INCLUDE_PATH=${Python3_ROOT_DIR}/include/python${{ matrix.python-version }}" >> $GITHUB_ENV @@ -487,53 +519,52 @@ jobs: echo "PY_EXT_SUFFIX=$(python -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")" >> $GITHUB_ENV - name: Install cuda.pathfinder (required for next step) - if: ${{ inputs.test-bindings || inputs.test-core }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} run: | - pip install cuda_pathfinder/*.whl + pip install cuda_pathfinder/dist/*.whl - name: Hide GNU link.exe so Meson finds MSVC link.exe - if: ${{ startsWith(inputs.host-platform, 'win') && (inputs.test-bindings || inputs.test-core) }} + if: ${{ startsWith(inputs.host-platform, 'win') && (env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true') }} run: | if [ -f "/c/Program Files/Git/usr/bin/link.exe" ]; then mv "/c/Program Files/Git/usr/bin/link.exe" "/c/Program Files/Git/usr/bin/link.exe.bak" fi - - name: Build cuda.bindings Cython tests - if: ${{ inputs.test-bindings }} + - name: Install wheels for current-context native test assets + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} run: | - pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl --group ./cuda_bindings/pyproject.toml:test - pushd ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }} - bash build_tests.sh - popd + if [[ "$TEST_BINDINGS" == "true" ]]; then + pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl --group ./cuda_bindings/pyproject.toml:test + fi + if [[ "$TEST_CORE_CURRENT" == "true" ]]; then + pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl + if [[ "$BUILD_CORE_CURRENT" == "true" ]]; then + core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}" -maxdepth 1 -type f -name '*.whl' -print -quit) + else + core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" -maxdepth 1 -type f -name '*.whl' -print -quit) + fi + test -n "$core_wheel" + pip install "$core_wheel" --group ./cuda_core/pyproject.toml:test + fi + + - name: Build current-context native test assets + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE_CURRENT == 'true' }} + env: + CUDA_CORE_BUILD_MAJOR: ${{ env.BUILD_CUDA_MAJOR }} + run: | + mapfile -t targets < <(jq -r '.[]' <<< "$TEST_ASSETS_CURRENT_TARGETS") + moon run "${targets[@]}" --upstream direct --downstream none - name: Upload cuda.bindings Cython tests - if: ${{ inputs.test-bindings }} + if: ${{ env.TEST_BINDINGS == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }}/test_*${{ env.PY_EXT_SUFFIX }} if-no-files-found: error - - name: Build cuda.core Cython tests - if: ${{ inputs.test-core }} - run: | - pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl - if ${{ inputs.build-core }}; then - core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}" -maxdepth 1 -type f -name '*.whl' -print -quit) - else - core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" -maxdepth 1 -type f -name '*.whl' -print -quit) - fi - if [[ -z "${core_wheel}" ]]; then - echo "No cuda.core wheel found" >&2 - exit 1 - fi - pip install "${core_wheel}" --group ./cuda_core/pyproject.toml:test - pushd ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }} - bash build_tests.sh - popd - - name: Upload cuda.core Cython tests - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE_CURRENT == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests @@ -542,7 +573,7 @@ jobs: # Note: This overwrites CUDA_PATH etc - name: Set up mini CTK - if: ${{ inputs.build-core || inputs.test-core }} + if: ${{ env.BUILD_CORE_PREVIOUS == 'true' || env.TEST_CORE_PREVIOUS == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -550,14 +581,14 @@ jobs: cuda-version: ${{ inputs.prev-cuda-version }} cuda-path: "./cuda_toolkit_prev" - - name: Build cuda.core test binaries - if: ${{ inputs.test-core }} + - name: Build previous-context native test assets + if: ${{ env.TEST_CORE_PREVIOUS == 'true' }} run: | - nvcc --version - python "${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/build_test_binaries.py" + mapfile -t targets < <(jq -r '.[]' <<< "$TEST_ASSETS_PREVIOUS_TARGETS") + moon run "${targets[@]}" --upstream direct --downstream none - name: Upload cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE_PREVIOUS == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries @@ -568,7 +599,7 @@ jobs: if-no-files-found: error - name: Download cuda.bindings build artifacts from the prior branch - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE_PREVIOUS == 'true' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -585,25 +616,28 @@ jobs: fi OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml) - OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" - OLD_WHEEL_ARTIFACT_PATTERN="${OLD_BASENAME}[0-9a-f]" + OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]" LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id \ --branch "${OLD_BRANCH}" \ - --artifact "${OLD_WHEEL_ARTIFACT_PATTERN}" \ + --artifact "${OLD_ARTIFACT_PATTERN}" \ NVIDIA/cuda-python "CI") PREV_BINDINGS_DIR="cuda_bindings/dist-prev" - gh run download "${LATEST_PRIOR_RUN_ID}" -p "${OLD_BASENAME}" -R NVIDIA/cuda-python - rm -rf ${OLD_BASENAME}-tests # exclude cython test artifacts - ls -al $OLD_BASENAME + gh run download \ + "${LATEST_PRIOR_RUN_ID}" \ + -p "${OLD_ARTIFACT_PATTERN}" \ + -R NVIDIA/cuda-python + OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}") + test -d "${OLD_ARTIFACT_DIR}" + ls -al "${OLD_ARTIFACT_DIR}" mkdir -p "${PREV_BINDINGS_DIR}" - mv $OLD_BASENAME/*.whl "${PREV_BINDINGS_DIR}" - rmdir $OLD_BASENAME + mv "${OLD_ARTIFACT_DIR}"/*.whl "${PREV_BINDINGS_DIR}" + rmdir "${OLD_ARTIFACT_DIR}" - name: Constrain previous cuda.core to the downloaded cuda.bindings wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE_PREVIOUS == 'true' }} run: | - pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) + pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) bindings_wheels=(cuda_bindings/dist-prev/cuda_bindings-"${BUILD_PREV_CUDA_MAJOR}".*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test "${#bindings_wheels[@]}" -eq 1 @@ -622,13 +656,13 @@ jobs: printf 'cuda-bindings @ %s\n' "${bindings_uri}" } | tee wheel-constraints/cuda-core-prev.txt - - name: Build cuda.core wheel - if: ${{ inputs.build-core }} - uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 - with: - package-dir: ./cuda_core/ - output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + - name: Build previous-context cuda.core wheel + if: ${{ env.BUILD_CORE_PREVIOUS == 'true' }} + run: | + mapfile -t targets < <(jq -r '.[]' <<< "$WHEEL_MULTI_CTK_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none env: + CUDA_CORE_BUILD_MAJOR: ${{ env.BUILD_PREV_CUDA_MAJOR }} CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"' CIBW_BEFORE_BUILD_WINDOWS: 'python -m pip install --upgrade "pip>=25.3" delvewheel' @@ -671,15 +705,15 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.core prev) - if: ${{ inputs.build-core && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_CORE_PREVIOUS == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_core_prev.json label: "cuda.core (prev CTK)" build-step: "Build cuda.core wheel" - - name: List the cuda.core artifacts directory and rename - if: ${{ inputs.build-core }} + - name: List the previous-context cuda.core artifacts + if: ${{ env.BUILD_CORE_PREVIOUS == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == win* ]]; then export CHOWN=chown @@ -689,30 +723,14 @@ jobs: $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }} ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - # Rename wheel to include CUDA version suffix - mkdir -p "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_PREV_CUDA_MAJOR}" - for wheel in ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl; do - if [[ -f "${wheel}" ]]; then - base_name=$(basename "${wheel}" .whl) - new_name="${base_name}.cu${BUILD_PREV_CUDA_MAJOR}.whl" - mv "${wheel}" "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_PREV_CUDA_MAJOR}/${new_name}" - echo "Renamed wheel to: ${new_name}" - fi - done - - ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - - name: Merge cuda.core wheels - if: ${{ inputs.build-core }} + if: ${{ env.FINALIZE_CORE == 'true' }} run: | - pip install wheel - python ci/tools/merge_cuda_core_wheels.py \ - "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cu"${BUILD_CUDA_MAJOR}"/cuda_core*.whl \ - "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cu"${BUILD_PREV_CUDA_MAJOR}"/cuda_core*.whl \ - --output-dir "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" + mapfile -t targets < <(jq -r '.[]' <<< "$WHEEL_FINALIZE_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Check cuda.core wheel - if: ${{ inputs.build-core }} + if: ${{ env.FINALIZE_CORE == 'true' }} run: | twine check --strict ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b17a3fe19d9..01ff63753f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,9 +50,13 @@ jobs: should-skip: if: ${{ github.repository_owner == 'nvidia' }} runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read outputs: skip: ${{ steps.get-should-skip.outputs.skip }} doc-only: ${{ steps.get-should-skip.outputs.doc_only }} + base-ref: ${{ steps.get-should-skip.outputs.base_ref }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -64,180 +68,266 @@ jobs: set -euxo pipefail if ${{ startsWith(github.ref_name, 'pull-request/') }}; then pr_number="$(grep -Po '(\d+)$' <<< '${{ github.ref_name }}')" - pr_title="$(gh pr view "${pr_number}" --json title --jq '.title')" + pr="$(gh pr view "${pr_number}" --json baseRefName,title)" + pr_title="$(jq -r '.title' <<< "${pr}")" + base_ref="$(jq -r '.baseRefName' <<< "${pr}")" skip="$(echo "${pr_title}" | grep -q '\[no-ci\]' && echo true || echo false)" doc_only="$(echo "${pr_title}" | grep -q '\[doc-only\]' && echo true || echo false)" else skip=false doc_only=false + base_ref="" fi echo "skip=${skip}" >> "$GITHUB_OUTPUT" echo "doc_only=${doc_only}" >> "$GITHUB_OUTPUT" + echo "base_ref=${base_ref}" >> "$GITHUB_OUTPUT" - # Detect which top-level modules were touched by the PR so downstream build - # and test jobs can avoid rebuilding/retesting modules unaffected by the - # change. See issue #299. - # - # Dependency graph (verified in pyproject.toml files): - # cuda_pathfinder -> (no internal deps) - # cuda_bindings -> cuda_pathfinder - # cuda_core -> cuda_pathfinder, cuda_bindings - # cuda_python -> cuda_bindings (meta package) - # - # A change to cuda_pathfinder (or shared infra) forces a rebuild of every - # downstream module. A change to cuda_bindings forces rebuild of cuda_core. - # A change to cuda_core alone skips rebuilding/retesting cuda_bindings. - # On push to main, tag refs, schedule, or workflow_dispatch events we - # unconditionally run everything because there is no meaningful "changed - # paths" baseline for those events. + # Moon owns file ownership, package impact, and the task graph. This job only + # establishes whether trusted artifacts may be reused and groups Moon's + # directly affected tasks by semantic CI phase for the heterogeneous runners. detect-changes: if: ${{ github.repository_owner == 'nvidia' }} runs-on: ubuntu-latest + needs: should-skip + permissions: + actions: read + contents: read outputs: - bindings: ${{ steps.compose.outputs.bindings }} - core: ${{ steps.compose.outputs.core }} - pathfinder: ${{ steps.compose.outputs.pathfinder }} - python_meta: ${{ steps.compose.outputs.python_meta }} - test_helpers: ${{ steps.compose.outputs.test_helpers }} - shared: ${{ steps.compose.outputs.shared }} - build_bindings: ${{ steps.compose.outputs.build_bindings }} - build_core: ${{ steps.compose.outputs.build_core }} - build_pathfinder: ${{ steps.compose.outputs.build_pathfinder }} - test_bindings: ${{ steps.compose.outputs.test_bindings }} - test_core: ${{ steps.compose.outputs.test_core }} - test_pathfinder: ${{ steps.compose.outputs.test_pathfinder }} - pr_merge_base: ${{ steps.filter.outputs.merge_base }} + workplan: ${{ steps.workplan.outputs.workplan }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - # Treeless clone: commit graph is needed for `git merge-base` and - # `git diff --name-only` below, but historical blobs aren't. + # Treeless clone: the commit graph is needed to resolve the PR merge + # base and classify its changed paths, but historical blobs aren't. fetch-depth: 0 filter: blob:none - # copy-pr-bot pushes every PR (whether it targets main or a backport - # branch such as 12.9.x) to pull-request/, so the base branch - # cannot be inferred from github.ref_name. Look it up via the - # upstream PR metadata so the diff below is rooted at the right place. - - name: Resolve PR base branch - id: pr-info - if: ${{ startsWith(github.ref_name, 'pull-request/') }} - uses: nv-gha-runners/get-pr-info@main + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false - - name: Detect changed paths - id: filter + - name: Resolve PR merge base + id: merge-base if: ${{ startsWith(github.ref_name, 'pull-request/') }} env: - # GitHub Actions evaluates step-level `env:` expressions eagerly — - # the step's `if:` gate does NOT short-circuit them. On non-PR - # events (push/tag/schedule), `pr-info` is skipped and its outputs - # are empty strings, so `fromJSON('')` would raise a template error - # and fail the step despite `if:` being false. Guard the - # `fromJSON` call with a short-circuit so the expression resolves - # to an empty string on non-PR events; the step is still gated - # off by `if:`, so `BASE_REF` is never consumed there. - BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} + BASE_REF: ${{ needs.should-skip.outputs.base-ref }} run: | - # Diff against the merge base with the PR's actual target branch. - # Uses merge-base so diverged branches only show files changed on - # the PR side, not upstream commits. + set -euo pipefail if [[ -z "${BASE_REF}" ]]; then - echo "Could not resolve PR base branch from get-pr-info output" >&2 + echo "Could not resolve PR base branch" >&2 exit 1 fi + base=$(git merge-base HEAD "origin/${BASE_REF}") - changed=$(git diff --name-only "$base"...HEAD) + echo "sha=${base}" >> "$GITHUB_OUTPUT" - has_match() { - grep -qE "$1" <<< "$changed" && echo true || echo false + - name: Resolve reusable base artifacts + id: baseline + if: ${{ startsWith(github.ref_name, 'pull-request/') }} + env: + BASE_REF: ${{ needs.should-skip.outputs.base-ref }} + MERGE_BASE: ${{ steps.merge-base.outputs.sha }} + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + + unavailable() { + echo "No complete reusable artifact set was found; this run will build and test everything." >> "$GITHUB_STEP_SUMMARY" + exit 0 } + if [[ -z "${BASE_REF}" ]]; then + unavailable + fi + + merge_base="${MERGE_BASE}" + if [[ -z "${merge_base}" ]]; then + unavailable + fi + if ! runs=$(gh run list \ + --repo "${{ github.repository }}" \ + --branch "${BASE_REF}" \ + --commit "${merge_base}" \ + --event push \ + --workflow ci.yml \ + --status success \ + --limit 100 \ + --json databaseId,headSha); then + unavailable + fi + + # Reuse only artifacts produced from the exact commit used as the + # PR diff base. Using the latest base-branch run is unsafe for a PR + # that was opened before newer changes landed on that branch. + if [[ $(jq 'length' <<< "$runs") -ne 1 ]]; then + unavailable + fi + run_id=$(jq -r '.[0].databaseId // empty' <<< "$runs") + run_sha=$(jq -r '.[0].headSha // empty' <<< "$runs") + if [[ -z "${run_id}" || "${run_sha}" != "${merge_base}" ]]; then + unavailable + fi + + if ! artifacts=$(gh api \ + "repos/${{ github.repository }}/actions/runs/${run_id}/artifacts?per_page=100" \ + --paginate \ + --jq '.artifacts[] | {name, expired}'); then + unavailable + fi + + has_artifact() { + jq -se --arg name "$1" \ + '[.[] | select(.name == $name)] | length == 1 and .[0].expired == false' \ + <<< "$artifacts" >/dev/null + } + + missing=() + for name in cuda-pathfinder-wheel cuda-python-wheel; do + has_artifact "$name" || missing+=("$name") + done + + cuda_version=$(yq '.cuda.build.version' ci/versions.yml) + if ! python_versions=$(yq -r '.jobs.build.strategy.matrix."python-version"[]' .github/workflows/build-wheel.yml); then + unavailable + fi + if [[ -z "${python_versions}" ]]; then + unavailable + fi + while IFS= read -r python_version; do + python=${python_version//./} + for platform in linux-64 linux-aarch64 win-64; do + binding="cuda-bindings-python${python}-cuda${cuda_version}-${platform}-${merge_base}" + core="cuda-core-python${python}-${platform}-${merge_base}" + has_artifact "$binding" || missing+=("$binding") + has_artifact "$core" || missing+=("$core") + done + done <<< "${python_versions}" + + if (( ${#missing[@]} != 0 )); then + printf 'Missing reusable artifact: %s\n' "${missing[@]}" >&2 + unavailable + fi + + echo "run_id=${run_id}" >> "$GITHUB_OUTPUT" { - echo "bindings=$(has_match '^cuda_bindings/')" - echo "core=$(has_match '^cuda_core/')" - echo "pathfinder=$(has_match '^cuda_pathfinder/')" - echo "python_meta=$(has_match '^cuda_python/')" - echo "test_helpers=$(has_match '^cuda_python_test_helpers/')" - echo "shared=$(has_match '^(\.github/|ci/|scripts/|toolshed/|conftest\.py$|pyproject\.toml$|pixi\.(toml|lock)$|pytest\.ini$|ruff\.toml$)')" - echo "merge_base=${base}" - } >> "$GITHUB_OUTPUT" - - - name: Compose gating outputs - id: compose + echo + echo "Reusable artifacts: run \`${run_id}\` at \`${merge_base}\` on \`${BASE_REF}\`." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Compute Moon CI workplan + id: workplan env: - IS_PR: ${{ startsWith(github.ref_name, 'pull-request/') }} - BINDINGS: ${{ steps.filter.outputs.bindings || 'false' }} - CORE: ${{ steps.filter.outputs.core || 'false' }} - PATHFINDER: ${{ steps.filter.outputs.pathfinder || 'false' }} - PYTHON_META: ${{ steps.filter.outputs.python_meta || 'false' }} - TEST_HELPERS: ${{ steps.filter.outputs.test_helpers || 'false' }} - SHARED: ${{ steps.filter.outputs.shared || 'false' }} + MERGE_BASE: ${{ steps.merge-base.outputs.sha }} + BASELINE_RUN_ID: ${{ steps.baseline.outputs.run_id }} + DOC_ONLY: ${{ needs.should-skip.outputs.doc-only }} run: | - set -euxo pipefail - # Non-PR events (push to main, tag push, schedule, workflow_dispatch) - # always exercise the full pipeline because there is no baseline for - # a meaningful diff. - if [[ "${IS_PR}" != "true" ]]; then - bindings=true - core=true - pathfinder=true - python_meta=true - test_helpers=true - shared=true - else - bindings="${BINDINGS}" - core="${CORE}" - pathfinder="${PATHFINDER}" - python_meta="${PYTHON_META}" - test_helpers="${TEST_HELPERS}" - shared="${SHARED}" - fi + set -euo pipefail + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + export PATH="$HOME/.moon/bin:$PATH" + uvx --from pytest pytest -q tests/test_moon_ci.py + + semantic_tags='[ + "ci-wheel-foundation", + "ci-wheel-bindings", + "ci-wheel-consumers", + "ci-wheel-multi-ctk", + "ci-wheel-finalize", + "ci-sdist-foundation", + "ci-sdist-bindings", + "ci-sdist-consumers", + "ci-test-assets-current", + "ci-test-assets-previous", + "ci-test-linux", + "ci-test-windows", + "ci-docs", + "ci-api", + "ci-ignore", + "ci-force-all" + ]' + + visible_tasks() { + jq -c '[ + .tasks + | to_entries[] as $project + | $project.value + | to_entries[] + | select(.value.options.internal != true) + | { + target: "\($project.key):\(.key)", + tags: (.value.tags // []) + } + ]' + } - or_flag() { - for v in "$@"; do - if [[ "${v}" == "true" ]]; then - echo "true" - return + force_all=false + selected='[]' + if [[ -z "${MERGE_BASE}" || -z "${BASELINE_RUN_ID}" ]]; then + force_all=true + else + git diff --no-renames --name-only -z "${MERGE_BASE}"...HEAD > changed-paths + while IFS= read -r -d '' path; do + [[ -n "${path}" ]] || continue + result=$(printf '%s\n' "${path}" | moon query tasks --affected stdin --upstream none --downstream none) + owned=$(visible_tasks <<< "${result}") + if jq -e 'length == 0 or any(.[]; .target == "root:ci-fallback")' <<< "${owned}" >/dev/null; then + force_all=true + break fi - done - echo "false" - } + selected=$(jq -cn \ + --argjson current "${selected}" \ + --argjson next "${owned}" \ + '$current + $next | unique_by(.target)') + done < changed-paths + fi - # Build gating: pathfinder change forces rebuild of bindings and - # core; bindings change forces rebuild of core. shared changes force - # a full rebuild. - build_pathfinder="$(or_flag "${shared}" "${pathfinder}")" - build_bindings="$(or_flag "${shared}" "${pathfinder}" "${bindings}")" - build_core="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${core}")" + if [[ "${force_all}" == "true" ]]; then + selected=$(moon query tasks | visible_tasks) + baseline_run_id="" + baseline_sha="" + else + baseline_run_id="${BASELINE_RUN_ID}" + baseline_sha="${MERGE_BASE}" + fi - # Test gating: tests for a module must run whenever that module, any - # of its runtime dependencies, the shared test helper package, or - # shared infra changes. pathfinder tests are cheap and always run. - test_pathfinder=true - test_bindings="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${test_helpers}")" - test_core="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${core}" "${test_helpers}")" + # Preserve the established [doc-only] behavior: build the complete + # documentation site even when the changed paths do not own docs. + if [[ "${DOC_ONLY}" == "true" ]]; then + docs=$(moon query tasks | visible_tasks | jq -c '[.[] | select(.tags | index("ci-docs"))]') + selected=$(jq -cn \ + --argjson current "${selected}" \ + --argjson docs "${docs}" \ + '$current + $docs | unique_by(.target)') + fi + targets=$(jq -cn \ + --argjson tags "${semantic_tags}" \ + --argjson selected "${selected}" \ + 'reduce $tags[] as $tag ({}; + .[$tag] = ([$selected[] + | select(.tags | index($tag)) + | .target] | unique | sort))') + workplan=$(jq -cn \ + --argjson targets "${targets}" \ + --arg merge_base "${MERGE_BASE}" \ + --arg baseline_run_id "${baseline_run_id}" \ + --arg baseline_sha "${baseline_sha}" \ + '{targets: $targets, merge_base: $merge_base, baseline: {run_id: $baseline_run_id, sha: $baseline_sha}}') + echo "workplan=$workplan" >> "$GITHUB_OUTPUT" { - echo "bindings=${bindings}" - echo "core=${core}" - echo "pathfinder=${pathfinder}" - echo "python_meta=${python_meta}" - echo "test_helpers=${test_helpers}" - echo "shared=${shared}" - echo "build_bindings=${build_bindings}" - echo "build_core=${build_core}" - echo "build_pathfinder=${build_pathfinder}" - echo "test_bindings=${test_bindings}" - echo "test_core=${test_core}" - echo "test_pathfinder=${test_pathfinder}" - } >> "$GITHUB_OUTPUT" + echo + echo "### CI workplan" + echo '```json' + jq . <<< "$workplan" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" api-check-core-vs-release: name: API check (cuda_core vs. latest release) if: >- ${{ !fromJSON(needs.should-skip.outputs.skip) && - fromJSON(needs.detect-changes.outputs.core) }} + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-api'][0] }} runs-on: ubuntu-latest needs: - should-skip @@ -275,20 +365,28 @@ jobs: git fetch --depth=1 --filter=blob:none origin \ "refs/tags/${{ steps.latest-tag.outputs.tag }}:refs/tags/${{ steps.latest-tag.outputs.tag }}" - - name: Check cuda_core public API - id: griffe - uses: ./.github/actions/griffe-api-check + - name: Install Moon 2.5.1 + shell: bash --noprofile --norc -euo pipefail {0} + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: - package-name: cuda.core - package-dir: cuda_core - merge-base: ${{ steps.latest-tag.outputs.tag }} + enable-cache: false + + - name: Check cuda_core public API + env: + CUDA_CORE_API_REF: ${{ steps.latest-tag.outputs.tag }} + run: moon run core:api-check --upstream none --downstream none api-check-core-vs-base: name: API check (cuda_core vs. merge base) if: >- ${{ startsWith(github.ref_name, 'pull-request/') && !fromJSON(needs.should-skip.outputs.skip) && - fromJSON(needs.detect-changes.outputs.core) }} + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-api'][0] }} runs-on: ubuntu-latest needs: - should-skip @@ -306,77 +404,134 @@ jobs: shell: bash --noprofile --norc -euo pipefail {0} run: | git fetch --depth=1 --filter=blob:none origin \ - "${{ needs.detect-changes.outputs.pr_merge_base }}" + "${{ fromJSON(needs.detect-changes.outputs.workplan).merge_base }}" - - name: Check cuda_core public API - id: griffe - uses: ./.github/actions/griffe-api-check + - name: Install Moon 2.5.1 + shell: bash --noprofile --norc -euo pipefail {0} + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: - package-name: cuda.core - package-dir: cuda_core - merge-base: ${{ needs.detect-changes.outputs.pr_merge_base }} + enable-cache: false + + - name: Check cuda_core public API + env: + CUDA_CORE_API_REF: ${{ fromJSON(needs.detect-changes.outputs.workplan).merge_base }} + run: moon run core:api-check --upstream none --downstream none # NOTE: Build jobs are intentionally split by platform rather than using a single - # matrix. This allows each test job to depend only on its corresponding build, - # so faster platforms can proceed through build & test without waiting for slower - # ones. Keep these job definitions textually identical except for: + # matrix. This lets each test job consume its platform-specific artifacts as + # soon as they are ready. ARM64 and Windows tests also wait for linux-64, + # which produces the universal pathfinder and cuda-python wheels. Keep these + # job definitions textually identical except for: # - host-platform value # - if: condition (build-linux-64 omits doc-only check since it's needed for docs) build-linux-64: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: host-platform: - linux-64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) }} + if: >- + ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + (fromJSON(needs.should-skip.outputs.doc-only) || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-consumers'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-multi-ctk'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-finalize'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-consumers'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-windows'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-docs'][0]) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See build-linux-64 for why build jobs are split by platform. build-linux-aarch64: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: host-platform: - linux-aarch64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-consumers'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-multi-ctk'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-finalize'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux'][0]) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See build-linux-64 for why build jobs are split by platform. build-windows: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: host-platform: - win-64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-consumers'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-multi-ctk'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-wheel-finalize'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-consumers'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-windows'][0]) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # NOTE: test-sdist jobs are split by platform (mirroring build-* and test-wheel-*) # so platform-specific sources (e.g. cuda_bindings/*_windows.pyx selected by @@ -388,26 +543,49 @@ jobs: needs: - ci-vars - should-skip + - detect-changes + - build-linux-64 name: Test sdist linux-64 - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-consumers'][0]) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/test-sdist-linux.yml with: host-platform: linux-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets) }} # See test-sdist-linux for why sdist test jobs are split by platform. test-sdist-windows: needs: - ci-vars - should-skip + - detect-changes + - build-linux-64 + - build-windows name: Test sdist win-64 - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-foundation'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-bindings'][0] || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-sdist-consumers'][0]) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/test-sdist-windows.yml with: host-platform: win-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets) }} # NOTE: Test jobs are split by platform for the same reason as build jobs (see # build-linux-64). Keep these job definitions textually identical except for: @@ -421,8 +599,12 @@ jobs: host-platform: - linux-64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux'][0] }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars @@ -436,7 +618,7 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux']) }} # See test-linux-64 for why test jobs are split by platform. test-linux-aarch64: @@ -446,13 +628,18 @@ jobs: host-platform: - linux-aarch64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux'][0] }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars - should-skip - detect-changes + - build-linux-64 - build-linux-aarch64 secrets: inherit uses: ./.github/workflows/test-wheel-linux.yml @@ -461,7 +648,7 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-linux']) }} # See test-linux-64 for why test jobs are split by platform. test-windows: @@ -471,13 +658,18 @@ jobs: host-platform: - win-64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-windows'][0] }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars - should-skip - detect-changes + - build-linux-64 - build-windows secrets: inherit uses: ./.github/workflows/test-wheel-windows.yml @@ -486,11 +678,14 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets['ci-test-windows']) }} doc: name: Docs - if: ${{ github.repository_owner == 'nvidia' }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + (fromJSON(needs.should-skip.outputs.doc-only) || + fromJSON(needs.detect-changes.outputs.workplan).targets['ci-docs'][0]) }} # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: id-token: write @@ -498,11 +693,14 @@ jobs: pull-requests: write needs: - ci-vars + - should-skip + - detect-changes - build-linux-64 secrets: inherit uses: ./.github/workflows/build-docs.yml with: is-release: ${{ github.ref_type == 'tag' }} + targets: ${{ toJSON(fromJSON(needs.detect-changes.outputs.workplan).targets['ci-docs']) }} precommit-windows: name: Pre-commit on Windows @@ -541,17 +739,26 @@ jobs: if: ${{ always() && github.repository_owner == 'nvidia' }} runs-on: ubuntu-latest needs: + - ci-vars - should-skip - detect-changes + - build-linux-64 + - build-linux-aarch64 + - build-windows - test-sdist-linux - test-sdist-windows - test-linux-64 - test-linux-aarch64 - test-windows + - api-check-core-vs-release + - api-check-core-vs-base - doc - precommit-windows steps: - name: Exit + env: + NEEDS_JSON: ${{ toJSON(needs) }} + WORKPLAN: ${{ needs.detect-changes.outputs.workplan }} run: | # GitHub treats `result == 'skipped'` as success for required # status checks (see CCCL gate comment + cccl#605). The previous @@ -568,9 +775,27 @@ jobs: fi doc_only="${{ needs.should-skip.outputs.doc-only }}" + wheel_selected=$(jq -r '([ + .targets["ci-wheel-foundation"][], + .targets["ci-wheel-bindings"][], + .targets["ci-wheel-consumers"][], + .targets["ci-wheel-multi-ctk"][], + .targets["ci-wheel-finalize"][] + ] | length > 0)' <<< "$WORKPLAN") + sdist_selected=$(jq -r '([ + .targets["ci-sdist-foundation"][], + .targets["ci-sdist-bindings"][], + .targets["ci-sdist-consumers"][] + ] | length > 0)' <<< "$WORKPLAN") + linux_selected=$(jq -r '.targets["ci-test-linux"] | length > 0' <<< "$WORKPLAN") + windows_selected=$(jq -r '.targets["ci-test-windows"] | length > 0' <<< "$WORKPLAN") + docs_selected=$(jq -r '.targets["ci-docs"] | length > 0' <<< "$WORKPLAN") + run_core_api_check=$(jq -r '.targets["ci-api"] | length > 0' <<< "$WORKPLAN") + is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { - name=$1; expected=$2; result=$3 + local name=$1 expected=$2 result + result=$(jq -r --arg name "$name" '.[$name].result // "missing"' <<< "$NEEDS_JSON") echo "Checking $name: result='$result' (expected '$expected')" if [[ "$result" != "$expected" ]]; then echo "::error::$name did not match expected result" @@ -578,18 +803,71 @@ jobs: fi } - # always expected to succeed (even in [doc-only] mode) - check_result "should-skip" "success" "${{ needs.should-skip.result }}" - check_result "detect-changes" "success" "${{ needs.detect-changes.result }}" - check_result "doc" "success" "${{ needs.doc.result }}" - check_result "precommit-windows" "success" "${{ needs.precommit-windows.result }}" - - # [doc-only] flips these from 'success' to 'skipped' - if [[ "$doc_only" == "true" ]]; then expected="skipped"; else expected="success"; fi - check_result "test-sdist-linux" "$expected" "${{ needs.test-sdist-linux.result }}" - check_result "test-sdist-windows" "$expected" "${{ needs.test-sdist-windows.result }}" - check_result "test-linux-64" "$expected" "${{ needs.test-linux-64.result }}" - check_result "test-linux-aarch64" "$expected" "${{ needs.test-linux-aarch64.result }}" - check_result "test-windows" "$expected" "${{ needs.test-windows.result }}" + # Control jobs and Windows pre-commit always run. + check_result "ci-vars" "success" + check_result "should-skip" "success" + check_result "detect-changes" "success" + check_result "precommit-windows" "success" + + # Build jobs copy forward the complete trusted artifact set whenever + # downstream work needs it, even if no package wheel is rebuilt. + expected="skipped" + if [[ "$doc_only" == "true" || "$wheel_selected" == "true" || + "$sdist_selected" == "true" || "$linux_selected" == "true" || + "$windows_selected" == "true" || "$docs_selected" == "true" ]]; then + expected="success" + fi + check_result "build-linux-64" "$expected" + + expected="skipped" + if [[ "$doc_only" != "true" && + ( "$wheel_selected" == "true" || "$linux_selected" == "true" ) ]]; then + expected="success" + fi + check_result "build-linux-aarch64" "$expected" + + expected="skipped" + if [[ "$doc_only" != "true" && + ( "$wheel_selected" == "true" || "$sdist_selected" == "true" || + "$windows_selected" == "true" ) ]]; then + expected="success" + fi + check_result "build-windows" "$expected" + + expected="skipped" + if [[ "$doc_only" != "true" && "$sdist_selected" == "true" ]]; then + expected="success" + fi + check_result "test-sdist-linux" "$expected" + check_result "test-sdist-windows" "$expected" + + expected="skipped" + if [[ "$doc_only" != "true" && "$linux_selected" == "true" ]]; then + expected="success" + fi + check_result "test-linux-64" "$expected" + check_result "test-linux-aarch64" "$expected" + + expected="skipped" + if [[ "$doc_only" != "true" && "$windows_selected" == "true" ]]; then + expected="success" + fi + check_result "test-windows" "$expected" + + expected="skipped" + if [[ "$doc_only" == "true" || "$docs_selected" == "true" ]]; then + expected="success" + fi + check_result "doc" "$expected" + + # API compatibility checks run for cuda_core source changes and for + # conservative full runs when reusable base artifacts are unavailable. + expected="skipped" + if [[ "$run_core_api_check" == "true" ]]; then expected="success"; fi + check_result "api-check-core-vs-release" "$expected" + + expected="skipped" + if [[ "$is_pr" == "true" && "$run_core_api_check" == "true" ]]; then expected="success"; fi + check_result "api-check-core-vs-base" "$expected" [[ "$status" == "success" ]] diff --git a/.github/workflows/test-sdist-linux.yml b/.github/workflows/test-sdist-linux.yml index f0f64492f2d..00722002da8 100644 --- a/.github/workflows/test-sdist-linux.yml +++ b/.github/workflows/test-sdist-linux.yml @@ -11,22 +11,11 @@ on: cuda-version: required: true type: string - build-pathfinder: + targets: + description: JSON object keyed by semantic Moon tags. An empty value builds everything. required: false - default: true - type: boolean - build-bindings: - required: false - default: true - type: boolean - build-core: - required: false - default: true - type: boolean - build-python: - required: false - default: true - type: boolean + default: "" + type: string defaults: run: @@ -39,7 +28,8 @@ permissions: jobs: test-sdist: name: Test sdist builds - if: ${{ inputs.build-pathfinder || inputs.build-bindings || inputs.build-core || inputs.build-python }} + env: + SEMANTIC_TARGETS: ${{ inputs.targets != '' && inputs.targets || '{"ci-sdist-foundation":["pathfinder:sdist"],"ci-sdist-bindings":["bindings:sdist"],"ci-sdist-consumers":["core:sdist","metapackage:sdist"]}' }} timeout-minutes: 60 runs-on: linux-amd64-cpu8 steps: @@ -51,36 +41,69 @@ jobs: fetch-depth: 0 filter: blob:none + - name: Install target resolver dependencies + uses: ./.github/actions/install_unix_deps + with: + dependencies: "jq" + dependent_exes: "jq" + + - name: Resolve Moon sdist targets + run: | + jq -e ' + . as $root | + type == "object" and + all(["ci-sdist-foundation", "ci-sdist-bindings", "ci-sdist-consumers"][]; + . as $tag | + (($root[$tag] // []) | type == "array" and all(.[]; type == "string"))) + ' <<< "$SEMANTIC_TARGETS" >/dev/null + foundation=$(jq -c '.["ci-sdist-foundation"] // []' <<< "$SEMANTIC_TARGETS") + bindings=$(jq -c '.["ci-sdist-bindings"] // []' <<< "$SEMANTIC_TARGETS") + consumers=$(jq -c '.["ci-sdist-consumers"] // []' <<< "$SEMANTIC_TARGETS") + all_targets=$(jq -cn \ + --argjson foundation "$foundation" \ + --argjson bindings "$bindings" \ + --argjson consumers "$consumers" \ + '$foundation + $bindings + $consumers') + has_target() { + jq -e --arg target "$1" 'index($target) != null' <<< "$all_targets" >/dev/null + } + { + echo "SDIST_FOUNDATION_TARGETS=$foundation" + echo "SDIST_BINDINGS_TARGETS=$bindings" + echo "SDIST_CONSUMER_TARGETS=$consumers" + echo "BUILD_PATHFINDER=$(has_target pathfinder:sdist && echo true || echo false)" + echo "BUILD_BINDINGS=$(has_target bindings:sdist && echo true || echo false)" + echo "BUILD_CORE=$(has_target core:sdist && echo true || echo false)" + echo "BUILD_PYTHON=$(has_target metapackage:sdist && echo true || echo false)" + } >> "$GITHUB_ENV" + - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" - - name: Install build tools - run: python -m pip install "pip>=25.3" build - - # Pure Python packages -- no CTK needed. - - name: Build cuda.pathfinder sdist and wheel-from-sdist - if: ${{ inputs.build-pathfinder }} + - name: Install Moon and build tools run: | - python -m build --sdist cuda_pathfinder/ - pip wheel --no-deps --wheel-dir cuda_pathfinder/dist cuda_pathfinder/dist/*.tar.gz + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + python -m pip install "pip>=25.3" build - - name: Build cuda-python sdist and wheel-from-sdist - if: ${{ inputs.build-python }} + # Pure Python packages -- no CTK needed. + - name: Build foundation sdists and wheels-from-sdists + if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | - python -m build --sdist cuda_python/ - pip wheel --no-deps --wheel-dir cuda_python/dist cuda_python/dist/*.tar.gz + mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_FOUNDATION_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Download cuda.pathfinder wheel - if: ${{ !inputs.build-pathfinder && (inputs.build-bindings || inputs.build-core) }} + if: ${{ env.BUILD_PATHFINDER != 'true' && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel path: cuda_pathfinder/dist - name: Constrain builds to the local cuda.pathfinder wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 @@ -93,14 +116,14 @@ jobs: # The env vars ACTIONS_CACHE_SERVICE_V2, ACTIONS_RESULTS_URL, and ACTIONS_RUNTIME_TOKEN # are exposed by this action. - name: Enable sccache - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # 0.0.10 with: disable_annotations: 'true' # xref: https://github.com/orgs/community/discussions/42856#discussioncomment-7678867 - name: Adding additional GHA cache-related env vars - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: actions/github-script@v9 with: script: | @@ -108,14 +131,14 @@ jobs: core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL']) - name: Setup proxy cache - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: nv-gha-runners/setup-proxy-cache@main continue-on-error: true with: enable-apt: true - name: Set up mini CTK - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -124,26 +147,26 @@ jobs: # cuda_bindings/setup.py parses CUDA headers at import time, so CUDA_PATH # (set by fetch_ctk) must be available for both sdist and wheel builds. - - name: Build cuda.bindings sdist and wheel-from-sdist - if: ${{ inputs.build-bindings }} + - name: Build bindings sdists and wheels-from-sdists + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export CC="sccache cc" export CXX="sccache c++" export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-bindings.txt" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - python -m build --sdist cuda_bindings/ - pip wheel --no-deps --wheel-dir cuda_bindings/dist cuda_bindings/dist/*.tar.gz + mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_BINDINGS_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Download cuda.bindings wheel - if: ${{ !inputs.build-bindings && inputs.build-core }} + if: ${{ env.BUILD_BINDINGS != 'true' && env.BUILD_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-bindings-python312-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ github.sha }} path: cuda_bindings/dist - name: Constrain cuda.core to the local cuda.bindings wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | CUDA_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) @@ -163,18 +186,20 @@ jobs: # cuda_core sdist delegates to setuptools (no CTK needed), but # wheel-from-sdist needs CTK and cuda-bindings (dynamic build dep via # get_requires_for_build_wheel in build_hooks.py). - - name: Build cuda.core sdist and wheel-from-sdist - if: ${{ inputs.build-core }} + - name: Build consumer sdists and wheels-from-sdists + if: ${{ env.BUILD_CORE == 'true' || env.BUILD_PYTHON == 'true' }} run: | - export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) - export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" - export CC="sccache cc" - export CXX="sccache c++" - export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-core.txt" - export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - python -m build --sdist cuda_core/ - pip wheel --no-deps --wheel-dir cuda_core/dist cuda_core/dist/*.tar.gz + if [[ "$BUILD_CORE" == "true" ]]; then + export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) + export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" + export CC="sccache cc" + export CXX="sccache c++" + export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-core.txt" + export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" + fi + mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_CONSUMER_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Show sccache stats - if: ${{ always() && (inputs.build-bindings || inputs.build-core) }} + if: ${{ always() && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} run: sccache --show-stats diff --git a/.github/workflows/test-sdist-windows.yml b/.github/workflows/test-sdist-windows.yml index 5451d20429e..623ce85fec1 100644 --- a/.github/workflows/test-sdist-windows.yml +++ b/.github/workflows/test-sdist-windows.yml @@ -17,22 +17,11 @@ on: cuda-version: required: true type: string - build-pathfinder: + targets: + description: JSON object keyed by semantic Moon tags. An empty value builds everything. required: false - default: true - type: boolean - build-bindings: - required: false - default: true - type: boolean - build-core: - required: false - default: true - type: boolean - build-python: - required: false - default: true - type: boolean + default: "" + type: string defaults: run: @@ -45,7 +34,8 @@ permissions: jobs: test-sdist: name: Test sdist builds - if: ${{ inputs.build-pathfinder || inputs.build-bindings || inputs.build-core || inputs.build-python }} + env: + SEMANTIC_TARGETS: ${{ inputs.targets != '' && inputs.targets || '{"ci-sdist-foundation":["pathfinder:sdist"],"ci-sdist-bindings":["bindings:sdist"],"ci-sdist-consumers":["core:sdist","metapackage:sdist"]}' }} timeout-minutes: 60 runs-on: windows-2022 steps: @@ -57,40 +47,68 @@ jobs: fetch-depth: 0 filter: blob:none + - name: Resolve Moon sdist targets + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + jq -e ' + . as $root | + type == "object" and + all(["ci-sdist-foundation", "ci-sdist-bindings", "ci-sdist-consumers"][]; + . as $tag | + (($root[$tag] // []) | type == "array" and all(.[]; type == "string"))) + ' <<< "$SEMANTIC_TARGETS" >/dev/null + foundation=$(jq -c '.["ci-sdist-foundation"] // []' <<< "$SEMANTIC_TARGETS") + bindings=$(jq -c '.["ci-sdist-bindings"] // []' <<< "$SEMANTIC_TARGETS") + consumers=$(jq -c '.["ci-sdist-consumers"] // []' <<< "$SEMANTIC_TARGETS") + all_targets=$(jq -cn \ + --argjson foundation "$foundation" \ + --argjson bindings "$bindings" \ + --argjson consumers "$consumers" \ + '$foundation + $bindings + $consumers') + has_target() { + jq -e --arg target "$1" 'index($target) != null' <<< "$all_targets" >/dev/null + } + { + echo "SDIST_FOUNDATION_TARGETS=$foundation" + echo "SDIST_BINDINGS_TARGETS=$bindings" + echo "SDIST_CONSUMER_TARGETS=$consumers" + echo "BUILD_PATHFINDER=$(has_target pathfinder:sdist && echo true || echo false)" + echo "BUILD_BINDINGS=$(has_target bindings:sdist && echo true || echo false)" + echo "BUILD_CORE=$(has_target core:sdist && echo true || echo false)" + echo "BUILD_PYTHON=$(has_target metapackage:sdist && echo true || echo false)" + } >> "$GITHUB_ENV" + - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" - name: Set up MSVC - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 - - name: Install build tools - run: python -m pip install "pip>=25.3" build - - # Pure Python packages -- no CTK needed. - - name: Build cuda.pathfinder sdist and wheel-from-sdist - if: ${{ inputs.build-pathfinder }} + - name: Install Moon and build tools run: | - python -m build --sdist cuda_pathfinder/ - pip wheel --no-deps --wheel-dir cuda_pathfinder/dist cuda_pathfinder/dist/*.tar.gz + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + python -m pip install "pip>=25.3" build - - name: Build cuda-python sdist and wheel-from-sdist - if: ${{ inputs.build-python }} + # Pure Python packages -- no CTK needed. + - name: Build foundation sdists and wheels-from-sdists + if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | - python -m build --sdist cuda_python/ - pip wheel --no-deps --wheel-dir cuda_python/dist cuda_python/dist/*.tar.gz + mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_FOUNDATION_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Download cuda.pathfinder wheel - if: ${{ !inputs.build-pathfinder && (inputs.build-bindings || inputs.build-core) }} + if: ${{ env.BUILD_PATHFINDER != 'true' && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel path: cuda_pathfinder/dist - name: Constrain builds to the local cuda.pathfinder wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 @@ -103,7 +121,7 @@ jobs: # smoke test, not a production build; see build-wheel.yml which also # limits sccache to Linux). - name: Set up mini CTK - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -114,24 +132,24 @@ jobs: # (set by fetch_ctk) must be available for both sdist and wheel builds. # Constraint paths are passed as native Windows paths because the pip # subprocesses run outside Git Bash. - - name: Build cuda.bindings sdist and wheel-from-sdist - if: ${{ inputs.build-bindings }} + - name: Build bindings sdists and wheels-from-sdists + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-bindings.txt")" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - python -m build --sdist cuda_bindings/ - pip wheel --no-deps --wheel-dir cuda_bindings/dist cuda_bindings/dist/*.tar.gz + mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_BINDINGS_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none - name: Download cuda.bindings wheel - if: ${{ !inputs.build-bindings && inputs.build-core }} + if: ${{ env.BUILD_BINDINGS != 'true' && env.BUILD_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-bindings-python312-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ github.sha }} path: cuda_bindings/dist - name: Constrain cuda.core to the local cuda.bindings wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | CUDA_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) @@ -151,12 +169,14 @@ jobs: # cuda_core sdist delegates to setuptools (no CTK needed), but # wheel-from-sdist needs CTK and cuda-bindings (dynamic build dep via # get_requires_for_build_wheel in build_hooks.py). - - name: Build cuda.core sdist and wheel-from-sdist - if: ${{ inputs.build-core }} + - name: Build consumer sdists and wheels-from-sdists + if: ${{ env.BUILD_CORE == 'true' || env.BUILD_PYTHON == 'true' }} run: | - export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) - export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" - export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-core.txt")" - export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - python -m build --sdist cuda_core/ - pip wheel --no-deps --wheel-dir cuda_core/dist cuda_core/dist/*.tar.gz + if [[ "$BUILD_CORE" == "true" ]]; then + export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) + export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" + export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-core.txt")" + export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" + fi + mapfile -t targets < <(jq -r '.[]' <<< "$SDIST_CONSUMER_TARGETS") + moon run "${targets[@]}" --upstream none --downstream none diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 2fd918e2859..dd99ae55384 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -22,18 +22,10 @@ on: nruns: type: number default: 1 - test-pathfinder: - type: boolean - default: true - test-bindings: - type: boolean - default: true - test-core: - type: boolean - default: true - test-python: - type: boolean - default: true + targets: + description: JSON array of exact Moon Linux test route targets. An empty value tests everything. + type: string + default: "" run-id: description: > Workflow run ID to download artifacts from. @@ -106,6 +98,8 @@ jobs: echo "OLD_BRANCH=${OLD_BRANCH}" >> "$GITHUB_OUTPUT" test: + env: + MOON_TARGETS: ${{ inputs.targets != '' && inputs.targets || '["pathfinder:ci-test-linux","bindings:ci-test-linux","core:ci-test-linux","metapackage:ci-test-linux"]' }} name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }}${{ matrix.FLAVOR && format(', {0}', matrix.FLAVOR) || '' }}${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }} timeout-minutes: 60 needs: compute-matrix @@ -146,6 +140,19 @@ jobs: dependencies: "jq wget libgl1 libegl1 g++ util-linux" dependent_exes: "jq wget" + - name: Resolve Moon test targets + run: | + jq -e 'type == "array" and all(.[]; type == "string")' <<< "$MOON_TARGETS" >/dev/null + has_target() { + jq -e --arg target "$1" 'index($target) != null' <<< "$MOON_TARGETS" >/dev/null + } + { + echo "TEST_PATHFINDER=$(has_target pathfinder:ci-test-linux && echo true || echo false)" + echo "TEST_BINDINGS=$(has_target bindings:ci-test-linux && echo true || echo false)" + echo "TEST_CORE=$(has_target core:ci-test-linux && echo true || echo false)" + echo "TEST_PYTHON=$(has_target metapackage:ci-test-linux && echo true || echo false)" + } >> "$GITHUB_ENV" + - name: Install GPU driver if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} env: @@ -164,7 +171,7 @@ jobs: LOCAL_CTK: ${{ matrix.LOCAL_CTK }} PY_VER: ${{ matrix.PY_VER }} SHA: ${{ inputs.sha || github.sha }} - SKIP_BINDINGS_TEST_OVERRIDE: ${{ !inputs.test-bindings && '1' || '0' }} + SKIP_BINDINGS_TEST_OVERRIDE: ${{ env.TEST_BINDINGS != 'true' && '1' || '0' }} run: ./ci/tools/env-vars test - name: Apply extra matrix environment variables @@ -174,7 +181,7 @@ jobs: run: echo "$MATRIX_ENV" | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> "$GITHUB_ENV" - name: Download cuda-pathfinder build artifacts - if: ${{ inputs.test-pathfinder || inputs.test-bindings || inputs.test-core || inputs.test-python }} + if: ${{ env.TEST_PATHFINDER == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel @@ -183,7 +190,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python build artifacts - if: ${{ inputs.test-python && env.BINDINGS_SOURCE == 'main' }} + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel @@ -192,7 +199,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda.bindings build artifacts - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'main' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -202,7 +209,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python & cuda.bindings build artifacts from the prior branch - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'backport' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -218,47 +225,66 @@ jobs: && apt install gh -y OLD_BRANCH=${{ needs.compute-matrix.outputs.OLD_BRANCH }} - OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" - OLD_WHEEL_ARTIFACT_PATTERN="${OLD_BASENAME}[0-9a-f]" + OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]" LOOKUP_ARGS=( --branch "${OLD_BRANCH}" - --artifact "${OLD_WHEEL_ARTIFACT_PATTERN}" + --artifact "${OLD_ARTIFACT_PATTERN}" ) - if ${{ inputs.test-python }}; then + if ${{ env.TEST_PYTHON == 'true' }}; then LOOKUP_ARGS+=(--artifact cuda-python-wheel) fi LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id \ "${LOOKUP_ARGS[@]}" NVIDIA/cuda-python "CI") - gh run download "${LATEST_PRIOR_RUN_ID}" -p "${OLD_BASENAME}" -R NVIDIA/cuda-python - rm -rf ${OLD_BASENAME}-tests # exclude cython test artifacts - ls -al $OLD_BASENAME + gh run download \ + "${LATEST_PRIOR_RUN_ID}" \ + -p "${OLD_ARTIFACT_PATTERN}" \ + -R NVIDIA/cuda-python + OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}") + test -d "${OLD_ARTIFACT_DIR}" + ls -al "${OLD_ARTIFACT_DIR}" mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}" - mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ - rmdir $OLD_BASENAME - - if ${{ inputs.test-python }}; then - gh run download "${LATEST_PRIOR_RUN_ID}" -p cuda-python-wheel -R NVIDIA/cuda-python + mv "${OLD_ARTIFACT_DIR}"/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ + rmdir "${OLD_ARTIFACT_DIR}" + + if ${{ env.TEST_PYTHON == 'true' }}; then + gh run download \ + "${LATEST_PRIOR_RUN_ID}" \ + -p cuda-python-wheel \ + -R NVIDIA/cuda-python ls -al cuda-python-wheel mv cuda-python-wheel/*.whl . rmdir cuda-python-wheel fi + - name: Stage pure wheels for Moon + if: ${{ inputs.test-mode == 'standard' }} + run: | + mkdir -p cuda_pathfinder/dist cuda_python/dist + pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) + if (( ${#pathfinder_wheels[@]} != 0 )) && [[ -f "${pathfinder_wheels[0]}" ]]; then + cp "${pathfinder_wheels[@]}" cuda_pathfinder/dist/ + fi + python_wheels=(cuda_python-*.whl) + if (( ${#python_wheels[@]} != 0 )) && [[ -f "${python_wheels[0]}" ]]; then + cp "${python_wheels[@]}" cuda_python/dist/ + fi + - name: Display structure of downloaded cuda-python artifacts - if: ${{ inputs.test-python && env.BINDINGS_SOURCE != 'published' }} + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'published' }} run: | pwd - ls -lah cuda_python*.whl cuda_pathfinder/ + find cuda_pathfinder cuda_python -maxdepth 2 -type f -name '*.whl' -print - name: Display structure of downloaded cuda.bindings artifacts - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE != 'published' }} run: | pwd ls -lahR $CUDA_BINDINGS_ARTIFACTS_DIR - name: Download cuda.bindings Cython tests - if: ${{ inputs.test-bindings && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests @@ -267,13 +293,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.bindings Cython tests - if: ${{ inputs.test-bindings && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} run: | pwd ls -lahR $CUDA_BINDINGS_CYTHON_TESTS_DIR - name: Download cuda.core build artifacts - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} @@ -282,13 +308,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core build artifacts - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} run: | pwd ls -lahR $CUDA_CORE_ARTIFACTS_DIR - name: Download cuda.core Cython tests - if: ${{ inputs.test-core && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests @@ -297,13 +323,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core Cython tests - if: ${{ inputs.test-core && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} run: | pwd ls -lahR $CUDA_CORE_CYTHON_TESTS_DIR - name: Download cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries @@ -312,7 +338,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | pwd ls -lahR $CUDA_CORE_TEST_BINARIES_DIR @@ -327,8 +353,14 @@ jobs: # we use self-hosted runners on which setup-python behaves weirdly (Python include can't be found)... AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache" + - name: Install Moon + if: ${{ inputs.test-mode == 'standard' }} + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + - name: Enable Scientific Python Nightly Wheels for Python 3.15 - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && startsWith(matrix.PY_VER, '3.15') }} run: | echo "PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" >> "$GITHUB_ENV" @@ -343,7 +375,7 @@ jobs: cuda-version: ${{ matrix.CUDA_VER }} - name: Set up latest cuda_sanitizer_api - if: ${{ (inputs.test-bindings || inputs.test-core) && env.SETUP_SANITIZER == '1' }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && env.SETUP_SANITIZER == '1' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -352,80 +384,30 @@ jobs: cuda-components: "cuda_sanitizer_api" - name: Set up compute-sanitizer - if: ${{ inputs.test-bindings || inputs.test-core }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} run: setup-sanitizer - name: Set up test repetition on nightly runs run: echo "PYTEST_ADDOPTS=\"--count=${{ inputs.nruns }}\"" >> "$GITHUB_ENV" - # ── Standard test steps (skipped for nightly modes) ── - - name: Run cuda.pathfinder tests with see_what_works - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} - env: - CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works - CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works - CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: see_what_works - run: run-tests pathfinder - - - name: Run cuda.bindings tests - if: ${{ inputs.test-mode == 'standard' && inputs.test-bindings && env.SKIP_CUDA_BINDINGS_TEST == '0' }} - env: - CUDA_VER: ${{ matrix.CUDA_VER }} - LOCAL_CTK: ${{ matrix.LOCAL_CTK }} - run: run-tests bindings - - - name: Run cuda.bindings benchmarks (smoke test) - if: ${{ inputs.test-mode == 'standard' && inputs.test-bindings && env.SKIP_CUDA_BINDINGS_TEST == '0' }} - run: | - pip install pyperf - pushd benchmarks/cuda_bindings - python run_pyperf.py --debug-single-value - popd - - - name: Run cuda.core tests - if: ${{ inputs.test-mode == 'standard' && inputs.test-core }} + # ── Standard test route (skipped for nightly modes) ── + - name: Run selected installed-wheel tests + if: ${{ inputs.test-mode == 'standard' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} - run: run-tests core - - - name: Ensure cuda-python installable - if: ${{ inputs.test-mode == 'standard' && inputs.test-python && env.BINDINGS_SOURCE == 'main' }} run: | - # Package suites install their own dependencies. A metapackage-only - # run has no preceding suite, so install the exact local internal - # wheels in one transaction while resolving released dependencies - # such as cuda-core from the package index. - if ${{ inputs.test-bindings || inputs.test-core }}; then - dependency_args=(--no-deps) - else - dependency_args=( - ./cuda_pathfinder/cuda_pathfinder-*.whl - "${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-*.whl - ) + targets="$MOON_TARGETS" + if [[ "$SKIP_CUDA_BINDINGS_TEST" != "0" ]]; then + targets=$(jq -c 'map(select(. != "bindings:ci-test-linux"))' <<< "$targets") fi - python_requirements=(cuda_python*.whl) - if [[ "${{ matrix.LOCAL_CTK }}" != 1 ]]; then - python_requirements=("${python_requirements[@]/%/[all]}") + if [[ "$BINDINGS_SOURCE" != "main" ]]; then + targets=$(jq -c 'map(select(. != "metapackage:ci-test-linux"))' <<< "$targets") + fi + mapfile -t target_args < <(jq -r '.[]' <<< "$targets") + if (( ${#target_args[@]} != 0 )); then + moon run "${target_args[@]}" --upstream direct --downstream none fi - pip install --only-binary=:all: "${dependency_args[@]}" "${python_requirements[@]}" - - - name: Install cuda.pathfinder extra wheels for testing - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} - run: | - set -euo pipefail - pushd cuda_pathfinder - pip install --only-binary=:all: -v ./*.whl --group "test-cu${TEST_CUDA_MAJOR}" - pip list - popd - - - name: Run cuda.pathfinder tests with all_must_work - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} - env: - CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work - CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work - CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: all_must_work - run: run-tests pathfinder # ── Nightly: install wheels + optional dep together ── - name: Install cuda-python wheels + PyTorch diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 1af7c4b625b..1ee5600042d 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -22,18 +22,10 @@ on: nruns: type: number default: 1 - test-pathfinder: - type: boolean - default: true - test-bindings: - type: boolean - default: true - test-core: - type: boolean - default: true - test-python: - type: boolean - default: true + targets: + description: JSON array of exact Moon Windows test route targets. An empty value tests everything. + type: string + default: "" run-id: description: > Workflow run ID to download artifacts from. @@ -96,6 +88,8 @@ jobs: echo "MATRIX=${MATRIX}" | tee --append "${GITHUB_OUTPUT}" test: + env: + MOON_TARGETS: ${{ inputs.targets != '' && inputs.targets || '["pathfinder:ci-test-windows","bindings:ci-test-windows","core:ci-test-windows","metapackage:ci-test-windows"]' }} name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }} (${{ matrix.DRIVER_MODE }})${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }} timeout-minutes: 60 # The build stage could fail but we want the CI to keep moving. @@ -111,6 +105,20 @@ jobs: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Resolve Moon test targets + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + jq -e 'type == "array" and all(.[]; type == "string")' <<< "$MOON_TARGETS" >/dev/null + has_target() { + jq -e --arg target "$1" 'index($target) != null' <<< "$MOON_TARGETS" >/dev/null + } + { + echo "TEST_PATHFINDER=$(has_target pathfinder:ci-test-windows && echo true || echo false)" + echo "TEST_BINDINGS=$(has_target bindings:ci-test-windows && echo true || echo false)" + echo "TEST_CORE=$(has_target core:ci-test-windows && echo true || echo false)" + echo "TEST_PYTHON=$(has_target metapackage:ci-test-windows && echo true || echo false)" + } >> "$GITHUB_ENV" + - name: Setup proxy cache uses: nv-gha-runners/setup-proxy-cache@main continue-on-error: true @@ -151,7 +159,7 @@ jobs: LOCAL_CTK: ${{ matrix.LOCAL_CTK }} PY_VER: ${{ matrix.PY_VER }} SHA: ${{ inputs.sha || github.sha }} - SKIP_BINDINGS_TEST_OVERRIDE: ${{ !inputs.test-bindings && '1' || '0' }} + SKIP_BINDINGS_TEST_OVERRIDE: ${{ env.TEST_BINDINGS != 'true' && '1' || '0' }} shell: bash --noprofile --norc -xeuo pipefail {0} run: ./ci/tools/env-vars test @@ -163,7 +171,7 @@ jobs: run: echo "$MATRIX_ENV" | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> "$GITHUB_ENV" - name: Download cuda-pathfinder build artifacts - if: ${{ inputs.test-pathfinder || inputs.test-bindings || inputs.test-core || inputs.test-python }} + if: ${{ env.TEST_PATHFINDER == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel @@ -172,7 +180,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python build artifacts - if: ${{ inputs.test-python && env.BINDINGS_SOURCE == 'main' }} + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel @@ -181,7 +189,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda.bindings build artifacts - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'main' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -191,54 +199,74 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python & cuda.bindings build artifacts from the prior branch - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'backport' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash --noprofile --norc -xeuo pipefail {0} run: | OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml) - OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" - OLD_WHEEL_ARTIFACT_PATTERN="${OLD_BASENAME}[0-9a-f]" + OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]" LOOKUP_ARGS=( --branch "${OLD_BRANCH}" - --artifact "${OLD_WHEEL_ARTIFACT_PATTERN}" + --artifact "${OLD_ARTIFACT_PATTERN}" ) - if ${{ inputs.test-python }}; then + if ${{ env.TEST_PYTHON == 'true' }}; then LOOKUP_ARGS+=(--artifact cuda-python-wheel) fi LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id \ "${LOOKUP_ARGS[@]}" NVIDIA/cuda-python "CI") - gh run download "${LATEST_PRIOR_RUN_ID}" -p "${OLD_BASENAME}" -R NVIDIA/cuda-python - rm -rf ${OLD_BASENAME}-tests # exclude cython test artifacts - ls -al $OLD_BASENAME + gh run download \ + "${LATEST_PRIOR_RUN_ID}" \ + -p "${OLD_ARTIFACT_PATTERN}" \ + -R NVIDIA/cuda-python + OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}") + test -d "${OLD_ARTIFACT_DIR}" + ls -al "${OLD_ARTIFACT_DIR}" mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}" - mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ - rmdir $OLD_BASENAME - - if ${{ inputs.test-python }}; then - gh run download "${LATEST_PRIOR_RUN_ID}" -p cuda-python-wheel -R NVIDIA/cuda-python + mv "${OLD_ARTIFACT_DIR}"/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ + rmdir "${OLD_ARTIFACT_DIR}" + + if ${{ env.TEST_PYTHON == 'true' }}; then + gh run download \ + "${LATEST_PRIOR_RUN_ID}" \ + -p cuda-python-wheel \ + -R NVIDIA/cuda-python ls -al cuda-python-wheel mv cuda-python-wheel/*.whl . rmdir cuda-python-wheel fi + - name: Stage pure wheels for Moon + if: ${{ inputs.test-mode == 'standard' }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + mkdir -p cuda_pathfinder/dist cuda_python/dist + pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) + if (( ${#pathfinder_wheels[@]} != 0 )) && [[ -f "${pathfinder_wheels[0]}" ]]; then + cp "${pathfinder_wheels[@]}" cuda_pathfinder/dist/ + fi + python_wheels=(cuda_python-*.whl) + if (( ${#python_wheels[@]} != 0 )) && [[ -f "${python_wheels[0]}" ]]; then + cp "${python_wheels[@]}" cuda_python/dist/ + fi + - name: Display structure of downloaded cuda-python artifacts - if: ${{ inputs.test-python && env.BINDINGS_SOURCE != 'published' }} + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'published' }} run: | Get-Location - Get-ChildItem cuda_python*.whl | Select-Object Mode, LastWriteTime, Length, FullName + Get-ChildItem -Recurse cuda_pathfinder,cuda_python -Filter *.whl | Select-Object Mode, LastWriteTime, Length, FullName - name: Display structure of downloaded cuda.bindings artifacts - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE != 'published' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_BINDINGS_ARTIFACTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.bindings Cython tests - if: ${{ inputs.test-bindings && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests @@ -247,13 +275,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.bindings Cython tests - if: ${{ inputs.test-bindings && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_BINDINGS_CYTHON_TESTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.core build artifacts - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} @@ -262,13 +290,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core build artifacts - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_CORE_ARTIFACTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.core Cython tests - if: ${{ inputs.test-core && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests @@ -277,13 +305,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core Cython tests - if: ${{ inputs.test-core && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_CORE_CYTHON_TESTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries @@ -292,7 +320,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_CORE_TEST_BINARIES_DIR | Select-Object Mode, LastWriteTime, Length, FullName @@ -304,8 +332,15 @@ jobs: # TODO: remove allow-prereleases once 3.15 is officially supported allow-prereleases: ${{ startsWith(matrix.PY_VER, '3.15') }} + - name: Install Moon + if: ${{ inputs.test-mode == 'standard' }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + curl -fsSL "https://github.com/moonrepo/moon/releases/download/v2.5.1/moon_cli-installer.sh" | bash + echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + - name: Enable Scientific Python Nightly Wheels for Python 3.15 - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && startsWith(matrix.PY_VER, '3.15') }} shell: bash --noprofile --norc -xeuo pipefail {0} run: | @@ -333,70 +368,25 @@ jobs: shell: bash --noprofile --norc -xeuo pipefail {0} run: echo "PYTEST_ADDOPTS=\"--count=${{ inputs.nruns }}\"" >> "$GITHUB_ENV" - # ── Standard test steps (skipped for nightly modes) ── - - name: Run cuda.pathfinder tests with see_what_works - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} - env: - CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works - CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works - CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: see_what_works - shell: bash --noprofile --norc -xeuo pipefail {0} - run: run-tests pathfinder - - - name: Run cuda.bindings tests - if: ${{ inputs.test-mode == 'standard' && inputs.test-bindings && env.SKIP_CUDA_BINDINGS_TEST == '0' }} + # ── Standard test route (skipped for nightly modes) ── + - name: Run selected installed-wheel tests + if: ${{ inputs.test-mode == 'standard' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} shell: bash --noprofile --norc -xeuo pipefail {0} - run: run-tests bindings - - - name: Run cuda.core tests - if: ${{ inputs.test-mode == 'standard' && inputs.test-core }} - env: - CUDA_VER: ${{ matrix.CUDA_VER }} - LOCAL_CTK: ${{ matrix.LOCAL_CTK }} - shell: bash --noprofile --norc -xeuo pipefail {0} - run: run-tests core - - - name: Ensure cuda-python installable - if: ${{ inputs.test-mode == 'standard' && inputs.test-python && env.BINDINGS_SOURCE == 'main' }} - run: | - # Package suites install their own dependencies. A metapackage-only - # run has no preceding suite, so install the exact local internal - # wheels in one transaction while resolving released dependencies - # such as cuda-core from the package index. - if ('${{ inputs.test-bindings || inputs.test-core }}' -eq 'true') { - $dependencyArgs = @('--no-deps') - } else { - $dependencyArgs = @( - (Get-Item ./cuda_pathfinder/cuda_pathfinder-*.whl).FullName - (Get-Item "$env:CUDA_BINDINGS_ARTIFACTS_DIR/cuda_bindings-*.whl").FullName - ) - } - $pythonRequirements = @((Get-Item ./cuda_python*.whl).FullName) - if ('${{ matrix.LOCAL_CTK }}' -ne '1') { - $pythonRequirements = @($pythonRequirements | ForEach-Object { "$($_)[all]" }) - } - pip install --only-binary=:all: @dependencyArgs @pythonRequirements - - - name: Install cuda.pathfinder extra wheels for testing - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} - shell: bash --noprofile --norc -xeuo pipefail {0} run: | - pushd cuda_pathfinder - pip install --only-binary=:all: -v ./*.whl --group "test-cu${TEST_CUDA_MAJOR}" - pip list - popd - - - name: Run cuda.pathfinder tests with all_must_work - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} - env: - CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work - CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work - CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: all_must_work - shell: bash --noprofile --norc -xeuo pipefail {0} - run: run-tests pathfinder + targets="$MOON_TARGETS" + if [[ "$SKIP_CUDA_BINDINGS_TEST" != "0" ]]; then + targets=$(jq -c 'map(select(. != "bindings:ci-test-windows"))' <<< "$targets") + fi + if [[ "$BINDINGS_SOURCE" != "main" ]]; then + targets=$(jq -c 'map(select(. != "metapackage:ci-test-windows"))' <<< "$targets") + fi + mapfile -t target_args < <(jq -r '.[]' <<< "$targets") + if (( ${#target_args[@]} != 0 )); then + moon run "${target_args[@]}" --upstream direct --downstream none + fi # ── Nightly: install wheels + optional dep together ── - name: Install Visual C++ Redistributable (required by PyTorch on Windows) diff --git a/.gitignore b/.gitignore index 6b6a7dfc0b5..4824d472ec6 100644 --- a/.gitignore +++ b/.gitignore @@ -182,3 +182,4 @@ cython_debug/ # Cursor .cursorrules .claude/settings.local.json +.moon/cache/ diff --git a/.moon/workspace.yml b/.moon/workspace.yml new file mode 100644 index 00000000000..02cc41d1447 --- /dev/null +++ b/.moon/workspace.yml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +projects: + root: '.' + pathfinder: 'cuda_pathfinder' + bindings: 'cuda_bindings' + core: 'cuda_core' + metapackage: 'cuda_python' + +vcs: + defaultBranch: 'main' + +versionConstraint: '=2.5.1' diff --git a/cuda_bindings/moon.yml b/cuda_bindings/moon.yml new file mode 100644 index 00000000000..d21b05fa280 --- /dev/null +++ b/cuda_bindings/moon.yml @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +$schema: https://moonrepo.dev/schemas/v2/project.json + +dependsOn: ['pathfinder'] + +fileGroups: + package: + - 'cuda/**/*' + - '!cuda/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!cuda/**/*.{md,svg}' + - 'build_hooks.py' + - 'DESCRIPTION.rst' + - 'LICENSE' + - 'MANIFEST.in' + - 'pyproject.toml' + - 'setup.py' + - '.git_archival.txt' + - '/.git_archival.txt' + - '/cuda_pathfinder/.git_archival.txt' + tests: + - 'tests/**/*' + - 'examples/**/*' + - '!{tests,examples}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + docs: + - 'docs/**/*' + - '!docs/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + benchmarks: + - '/benchmarks/cuda_bindings/benchmarks/**/*' + - '/benchmarks/cuda_bindings/runner/**/*' + - '/benchmarks/cuda_bindings/tests/**/*' + - '/benchmarks/cuda_bindings/compare.py' + - '/benchmarks/cuda_bindings/run_cpp.py' + - '/benchmarks/cuda_bindings/run_pyperf.py' + - '/benchmarks/cuda_bindings/pixi.lock' + - '/benchmarks/cuda_bindings/pixi.toml' + sharedTestInfra: + - '/cuda_python_test_helpers/**/*' + - '/benchmarks/**/*' + - '!/{benchmarks,cuda_python_test_helpers}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!/benchmarks/**/*.{md,svg}' + - '/ci/test-matrix.yml' + - '/ci/tools/download-wheels' + - '/ci/tools/run-tests' + linuxTestInfra: + - '/.github/workflows/test-wheel-linux.yml' + - '/ci/tools/guess_latest.sh' + - '/ci/tools/install_gpu_driver.sh' + - '/ci/tools/setup-sanitizer' + windowsTestInfra: + - '/.github/workflows/test-wheel-windows.yml' + - '/ci/tools/configure_driver_mode.ps1' + - '/ci/tools/install_gpu_driver.ps1' + +tasks: + install: + script: 'python -m pip install -e . pyperf --group test' + deps: ['pathfinder:install'] + inputs: ['@group(package)'] + options: + cache: false + windowsShell: 'bash' + + test: + script: | + bash tests/cython/build_tests.sh + python -m pytest . --override-ini norecursedirs=examples + python -m pytest ../benchmarks/cuda_bindings/tests/ + deps: ['install'] + inputs: ['@group(package)', '@group(tests)', '@group(benchmarks)', '/cuda_python_test_helpers/**/*'] + options: + cache: false + windowsShell: 'bash' + + test-installed: + script: | + set -euo pipefail + cd .. + temporary_wheels=() + for wheel in cuda_pathfinder/dist/*.whl; do + staged="cuda_pathfinder/$(basename "$wheel")" + if [[ ! -e "$staged" ]]; then + cp "$wheel" "$staged" + temporary_wheels+=("$staged") + fi + done + trap 'rm -f "${temporary_wheels[@]}"' EXIT + ci/tools/run-tests bindings + deps: ['wheel', 'build-cython-tests'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + options: + cache: false + internal: true + mutex: 'ci-python-gpu' + windowsShell: 'bash' + + build-cython-tests: + script: 'python -m pip install ../cuda_pathfinder/dist/*.whl dist/*.whl --group ./pyproject.toml:test && bash tests/cython/build_tests.sh' + deps: ['wheel'] + inputs: ['@group(package)', 'tests/cython/**/*', '/cuda_python_test_helpers/**/*'] + options: + cache: false + internal: true + mutex: 'ci-python-build' + windowsShell: 'bash' + + benchmark-smoke: + script: 'python -m pip install ../cuda_pathfinder/dist/*.whl dist/*.whl pyperf --group ./pyproject.toml:test && cd ../benchmarks/cuda_bindings && python run_pyperf.py --debug-single-value' + deps: ['wheel'] + inputs: ['@group(package)', '@group(benchmarks)'] + options: + cache: false + internal: true + mutex: 'ci-python-gpu' + windowsShell: 'bash' + + wheel: + script: 'cd .. && mkdir -p cuda_bindings/dist && python -m cibuildwheel cuda_bindings --output-dir cuda_bindings/dist' + deps: ['pathfinder:wheel'] + inputs: + - '@group(package)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_python/DESCRIPTION.rst' + - '/cuda_python/LICENSE' + - '/cuda_python/pyproject.toml' + - '/cuda_python/setup.py' + - '/cuda_python/README.md' + - '/README.md' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-wheel-bindings'] + + sdist: + script: 'mkdir -p dist && export PIP_FIND_LINKS=../cuda_pathfinder/dist PIP_PRE=1 && python -m build --sdist --outdir dist . && python -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz' + deps: ['pathfinder:sdist'] + inputs: + - '@group(package)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_python/DESCRIPTION.rst' + - '/cuda_python/LICENSE' + - '/cuda_python/pyproject.toml' + - '/cuda_python/setup.py' + - '/cuda_python/README.md' + - '/README.md' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-sdist-bindings'] + + docs: + script: 'rm -rf docs/build docs/source/generated && cd docs && ./build_docs.sh ${DOCS_BUILD_ARGS:-}' + inputs: ['@group(docs)'] + options: + cache: false + tags: ['ci-docs'] + + benchmark: + script: 'cd ../benchmarks/cuda_bindings && python run_pyperf.py' + deps: ['install'] + inputs: ['@group(package)', '@group(benchmarks)'] + options: + cache: false + + ci-test-assets: + deps: ['build-cython-tests'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(linuxTestInfra)' + - '@group(windowsTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + options: + cache: false + tags: ['ci-test-assets-current'] + + ci-test-linux: + deps: ['test-installed', 'benchmark-smoke'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(linuxTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + options: + cache: false + tags: ['ci-test-linux'] + + ci-test-windows: + deps: ['test-installed'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(windowsTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + options: + cache: false + tags: ['ci-test-windows'] diff --git a/cuda_core/moon.yml b/cuda_core/moon.yml new file mode 100644 index 00000000000..c4b5b47f455 --- /dev/null +++ b/cuda_core/moon.yml @@ -0,0 +1,318 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +$schema: https://moonrepo.dev/schemas/v2/project.json + +dependsOn: ['bindings'] + +fileGroups: + package: + - 'cuda/**/*' + - '!cuda/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!cuda/**/*.{md,svg}' + - 'build_hooks.py' + - 'DESCRIPTION.rst' + - 'LICENSE' + - 'MANIFEST.in' + - 'NOTICE' + - 'pyproject.toml' + - 'setup.py' + - '.git_archival.txt' + - '/.git_archival.txt' + tests: + - 'tests/**/*' + - 'examples/**/*' + - '!{tests,examples}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - 'pytest.ini' + docs: + - 'docs/**/*' + - '!docs/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + sharedTestInfra: + - '/cuda_python_test_helpers/**/*' + - '/benchmarks/**/*' + - '!/{benchmarks,cuda_python_test_helpers}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!/benchmarks/**/*.{md,svg}' + - '/ci/test-matrix.yml' + - '/ci/tools/download-wheels' + - '/ci/tools/run-tests' + linuxTestInfra: + - '/.github/workflows/test-wheel-linux.yml' + - '/ci/tools/guess_latest.sh' + - '/ci/tools/install_gpu_driver.sh' + - '/ci/tools/setup-sanitizer' + windowsTestInfra: + - '/.github/workflows/test-wheel-windows.yml' + - '/ci/tools/configure_driver_mode.ps1' + - '/ci/tools/install_gpu_driver.ps1' + upstreamPackageVersions: + - '/cuda_pathfinder/.git_archival.txt' + - '/cuda_bindings/.git_archival.txt' + +tasks: + install: + script: 'python -m pip install -e . --group test' + deps: ['bindings:install'] + inputs: ['@group(package)'] + options: + cache: false + windowsShell: 'bash' + + test: + script: | + bash tests/cython/build_tests.sh + python -m pytest . --override-ini norecursedirs="" + deps: ['install'] + inputs: ['@group(package)', '@group(tests)', '/cuda_python_test_helpers/**/*'] + options: + cache: false + windowsShell: 'bash' + + test-installed: + script: | + set -euo pipefail + cd .. + temporary_wheels=() + for wheel in cuda_pathfinder/dist/*.whl; do + staged="cuda_pathfinder/$(basename "$wheel")" + if [[ ! -e "$staged" ]]; then + cp "$wheel" "$staged" + temporary_wheels+=("$staged") + fi + done + trap 'rm -f "${temporary_wheels[@]}"' EXIT + ci/tools/run-tests core + deps: ['wheel', 'build-cython-tests', 'build-test-binaries'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + options: + cache: false + internal: true + mutex: 'ci-python-gpu' + windowsShell: 'bash' + + build-cython-tests: + script: | + set -euo pipefail + core_wheels=(dist/cu${CUDA_CORE_BUILD_MAJOR:?}/*.whl) + if [[ ! -e "${core_wheels[0]}" ]]; then + core_wheels=(dist/*.whl) + fi + python -m pip install ../cuda_pathfinder/dist/*.whl ../cuda_bindings/dist/*.whl "${core_wheels[@]}" --group ./pyproject.toml:test + bash tests/cython/build_tests.sh + deps: ['wheel'] + inputs: ['@group(package)', 'tests/cython/**/*', '/cuda_bindings/cuda/**/*', '/cuda_python_test_helpers/**/*'] + options: + cache: false + internal: true + mutex: 'ci-python-build' + windowsShell: 'bash' + + build-test-binaries: + command: 'python' + args: ['tests/test_binaries/build_test_binaries.py'] + inputs: ['tests/test_binaries/build_test_binaries.py', 'tests/test_binaries/saxpy.cu'] + options: + cache: false + internal: true + mutex: 'ci-python-build' + + wheel: + script: | + cd .. + cuda_major=${CUDA_CORE_BUILD_MAJOR:?} + mkdir -p "cuda_core/dist/cu${cuda_major}" + python -m cibuildwheel cuda_core --output-dir "cuda_core/dist/cu${cuda_major}" + shopt -s nullglob + wheels=("cuda_core/dist/cu${cuda_major}"/*.whl) + test "${#wheels[@]}" -eq 1 + if [[ "${wheels[0]}" != *.cu${cuda_major}.whl ]]; then + mv "${wheels[0]}" "${wheels[0]%.whl}.cu${cuda_major}.whl" + fi + deps: ['bindings:wheel'] + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/ci/tools/merge_cuda_core_wheels.py' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-wheel-consumers', 'ci-wheel-multi-ctk'] + + wheel-merge: + script: 'python ../ci/tools/merge_cuda_core_wheels.py dist/cu12/*.whl dist/cu13/*.whl --output-dir dist' + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - 'dist/cu12/*.whl' + - 'dist/cu13/*.whl' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/ci/tools/merge_cuda_core_wheels.py' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-wheel-finalize'] + + sdist: + script: 'mkdir -p dist && export PIP_FIND_LINKS="../cuda_pathfinder/dist ../cuda_bindings/dist" PIP_PRE=1 && python -m build --sdist --outdir dist . && python -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz' + deps: ['bindings:sdist'] + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-sdist-consumers'] + + docs: + script: 'rm -rf docs/build docs/source/generated && cd docs && ./build_docs.sh ${DOCS_BUILD_ARGS:-}' + inputs: ['@group(docs)'] + options: + cache: false + tags: ['ci-docs'] + + api-check: + script: 'uvx griffe check cuda.core --search . --find-stubs-packages --against "${CUDA_CORE_API_REF:?}" --format github 2>&1' + inputs: ['@group(package)', '/.github/actions/griffe-api-check/action.yml'] + options: + cache: false + tags: ['ci-api'] + + ci-test-assets: + deps: ['build-cython-tests'] + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(linuxTestInfra)' + - '@group(windowsTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + options: + cache: false + tags: ['ci-test-assets-current'] + + ci-test-binaries: + deps: ['build-test-binaries'] + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(linuxTestInfra)' + - '@group(windowsTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + options: + cache: false + tags: ['ci-test-assets-previous'] + + ci-test-linux: + deps: ['test-installed'] + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(linuxTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + options: + cache: false + tags: ['ci-test-linux'] + + ci-test-windows: + deps: ['test-installed'] + inputs: + - '@group(package)' + - '@group(upstreamPackageVersions)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(windowsTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + options: + cache: false + tags: ['ci-test-windows'] diff --git a/cuda_pathfinder/moon.yml b/cuda_pathfinder/moon.yml new file mode 100644 index 00000000000..c164576c52c --- /dev/null +++ b/cuda_pathfinder/moon.yml @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +$schema: https://moonrepo.dev/schemas/v2/project.json + +fileGroups: + package: + - 'cuda/**/*' + - '!cuda/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!cuda/**/*.{md,svg}' + - 'DESCRIPTION.rst' + - 'LICENSE' + - 'pyproject.toml' + - '.git_archival.txt' + - '/.git_archival.txt' + tests: + - 'tests/**/*' + - 'examples/**/*' + - '!{tests,examples}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + docs: + - 'docs/**/*' + - '!docs/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + sharedTestInfra: + - '/cuda_python_test_helpers/**/*' + - '/benchmarks/**/*' + - '!/{benchmarks,cuda_python_test_helpers}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!/benchmarks/**/*.{md,svg}' + - '/ci/test-matrix.yml' + - '/ci/tools/download-wheels' + - '/ci/tools/run-tests' + linuxTestInfra: + - '/.github/workflows/test-wheel-linux.yml' + - '/ci/tools/guess_latest.sh' + - '/ci/tools/install_gpu_driver.sh' + - '/ci/tools/setup-sanitizer' + windowsTestInfra: + - '/.github/workflows/test-wheel-windows.yml' + - '/ci/tools/configure_driver_mode.ps1' + - '/ci/tools/install_gpu_driver.ps1' + +tasks: + install: + script: 'python -m pip install -e . --group test' + inputs: ['@group(package)'] + options: + cache: false + windowsShell: 'bash' + + test: + script: 'python -m pytest tests/' + deps: ['install'] + inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)'] + options: + cache: false + windowsShell: 'bash' + + test-installed: + script: | + set -euo pipefail + cd .. + temporary_wheels=() + for wheel in cuda_pathfinder/dist/*.whl; do + staged="cuda_pathfinder/$(basename "$wheel")" + if [[ ! -e "$staged" ]]; then + cp "$wheel" "$staged" + temporary_wheels+=("$staged") + fi + done + trap 'rm -f "${temporary_wheels[@]}"' EXIT + CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS=see_what_works CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS=see_what_works CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS=see_what_works ci/tools/run-tests pathfinder + python -m pip install --only-binary=:all: -v cuda_pathfinder/*.whl --group "./cuda_pathfinder/pyproject.toml:test-cu${TEST_CUDA_MAJOR:?}" + CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS=all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS=all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS=all_must_work ci/tools/run-tests pathfinder + deps: ['wheel'] + inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)'] + options: + cache: false + internal: true + mutex: 'ci-python-gpu' + windowsShell: 'bash' + + wheel: + script: 'mkdir -p dist && python -m pip wheel -v --no-deps --wheel-dir dist .' + inputs: ['@group(package)'] + options: + cache: false + windowsShell: 'bash' + tags: ['ci-wheel-foundation'] + + sdist: + script: 'mkdir -p dist && python -m build --sdist --outdir dist . && python -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz' + inputs: ['@group(package)'] + options: + cache: false + windowsShell: 'bash' + tags: ['ci-sdist-foundation'] + + docs: + script: 'rm -rf docs/build docs/source/generated && cd docs && ./build_docs.sh ${DOCS_BUILD_ARGS:-}' + inputs: ['@group(docs)'] + options: + cache: false + tags: ['ci-docs'] + + ci-test-linux: + deps: ['test-installed'] + inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)', '@group(linuxTestInfra)'] + options: + cache: false + tags: ['ci-test-linux'] + + ci-test-windows: + deps: ['test-installed'] + inputs: ['@group(package)', '@group(tests)', '@group(sharedTestInfra)', '@group(windowsTestInfra)'] + options: + cache: false + tags: ['ci-test-windows'] diff --git a/cuda_python/moon.yml b/cuda_python/moon.yml new file mode 100644 index 00000000000..52999bb9ca1 --- /dev/null +++ b/cuda_python/moon.yml @@ -0,0 +1,194 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +$schema: https://moonrepo.dev/schemas/v2/project.json + +dependsOn: ['bindings'] + +fileGroups: + package: + - 'DESCRIPTION.rst' + - 'LICENSE' + - 'pyproject.toml' + - 'setup.py' + - 'README.md' + - '/README.md' + - '/cuda_pathfinder/.git_archival.txt' + - '/cuda_bindings/.git_archival.txt' + - '/.git_archival.txt' + tests: + - '/tests/integration/**/*' + - '!/tests/integration/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + docs: + - 'docs/**/*' + - '!docs/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + sharedTestInfra: + - '/cuda_python_test_helpers/**/*' + - '/benchmarks/**/*' + - '!/{benchmarks,cuda_python_test_helpers}/**/{AGENTS.md,CLAUDE.md,pixi.lock,pixi.toml}' + - '!/benchmarks/**/*.{md,svg}' + - '/cuda_core/.git_archival.txt' + - '/ci/test-matrix.yml' + - '/ci/tools/download-wheels' + - '/ci/tools/run-tests' + linuxTestInfra: + - '/.github/workflows/test-wheel-linux.yml' + - '/ci/tools/guess_latest.sh' + - '/ci/tools/install_gpu_driver.sh' + - '/ci/tools/setup-sanitizer' + windowsTestInfra: + - '/.github/workflows/test-wheel-windows.yml' + - '/ci/tools/configure_driver_mode.ps1' + - '/ci/tools/install_gpu_driver.ps1' + +tasks: + test-installed: + script: | + set -euo pipefail + core_wheels=(../cuda_core/dist/*.whl) + if [[ ! -e "${core_wheels[0]}" ]]; then + core_wheels=(../cuda_core/dist/cu${CUDA_CORE_BUILD_MAJOR:?}/*.whl) + fi + python -m pip install --only-binary=:all: ../cuda_pathfinder/dist/*.whl ../cuda_bindings/dist/*.whl "${core_wheels[@]}" dist/*.whl + deps: ['wheel', 'core:wheel'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/cuda_core/cuda/**/*' + - '/cuda_core/build_hooks.py' + - '/cuda_core/DESCRIPTION.rst' + - '/cuda_core/LICENSE' + - '/cuda_core/MANIFEST.in' + - '/cuda_core/NOTICE' + - '/cuda_core/pyproject.toml' + - '/cuda_core/setup.py' + options: + cache: false + internal: true + mutex: 'ci-python-gpu' + windowsShell: 'bash' + + wheel: + script: 'mkdir -p dist && python -m pip wheel -v --no-deps --wheel-dir dist .' + deps: ['bindings:wheel'] + inputs: + - '@group(package)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/cuda_python/README.md' + - '/README.md' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-wheel-consumers'] + + sdist: + script: 'mkdir -p dist && python -m build --sdist --outdir dist . && python -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz' + inputs: + - '@group(package)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/cuda_python/README.md' + - '/README.md' + options: + cache: false + windowsShell: 'bash' + tags: ['ci-sdist-consumers'] + + docs: + script: 'rm -rf docs/build docs/source/generated && cd docs && ./build_docs.sh ${DOCS_BUILD_ARGS:-}' + inputs: ['@group(docs)'] + options: + cache: false + tags: ['ci-docs'] + + ci-test-linux: + deps: ['test-installed'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(linuxTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/cuda_core/cuda/**/*' + - '/cuda_core/build_hooks.py' + - '/cuda_core/DESCRIPTION.rst' + - '/cuda_core/LICENSE' + - '/cuda_core/MANIFEST.in' + - '/cuda_core/NOTICE' + - '/cuda_core/pyproject.toml' + - '/cuda_core/setup.py' + options: + cache: false + tags: ['ci-test-linux'] + + ci-test-windows: + deps: ['test-installed'] + inputs: + - '@group(package)' + - '@group(tests)' + - '@group(sharedTestInfra)' + - '@group(windowsTestInfra)' + - '/cuda_pathfinder/cuda/**/*' + - '/cuda_pathfinder/DESCRIPTION.rst' + - '/cuda_pathfinder/LICENSE' + - '/cuda_pathfinder/pyproject.toml' + - '/cuda_bindings/cuda/**/*' + - '/cuda_bindings/build_hooks.py' + - '/cuda_bindings/DESCRIPTION.rst' + - '/cuda_bindings/LICENSE' + - '/cuda_bindings/MANIFEST.in' + - '/cuda_bindings/pyproject.toml' + - '/cuda_bindings/setup.py' + - '/cuda_core/cuda/**/*' + - '/cuda_core/build_hooks.py' + - '/cuda_core/DESCRIPTION.rst' + - '/cuda_core/LICENSE' + - '/cuda_core/MANIFEST.in' + - '/cuda_core/NOTICE' + - '/cuda_core/pyproject.toml' + - '/cuda_core/setup.py' + options: + cache: false + tags: ['ci-test-windows'] diff --git a/moon.yml b/moon.yml new file mode 100644 index 00000000000..2670648d385 --- /dev/null +++ b/moon.yml @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +$schema: https://moonrepo.dev/schemas/v2/project.json + +tasks: + test: + deps: + - 'pathfinder:test' + - 'bindings:test' + - 'core:test' + inputs: [] + options: + cache: false + + docs: + script: | + rm -rf cuda_python/docs/build/html/cuda-bindings cuda_python/docs/build/html/cuda-core cuda_python/docs/build/html/cuda-pathfinder + mkdir -p cuda_python/docs/build/html/cuda-bindings cuda_python/docs/build/html/cuda-core cuda_python/docs/build/html/cuda-pathfinder + cp -R cuda_bindings/docs/build/html/. cuda_python/docs/build/html/cuda-bindings/ + cp -R cuda_core/docs/build/html/. cuda_python/docs/build/html/cuda-core/ + cp -R cuda_pathfinder/docs/build/html/. cuda_python/docs/build/html/cuda-pathfinder/ + deps: + - 'pathfinder:docs' + - 'bindings:docs' + - 'core:docs' + - 'metapackage:docs' + inputs: [] + options: + cache: false + tags: ['ci-docs'] + + ci-ignore: + inputs: + - '/.agents/**/*' + - '/.coveragerc' + - '/.gitattributes' + - '/.gitignore' + - '/.github/**/*' + - '!/.github/actions/**/*' + - '!/.github/workflows/**/*' + - '/.mailmap' + - '/.pre-commit-config.yaml' + - '/.spdx-ignore' + - '/**/AGENTS.md' + - '/**/CLAUDE.md' + - '/**/pixi.lock' + - '/**/pixi.toml' + - '/**/*.md' + - '/**/*.svg' + - '!/README.md' + - '!/cuda_python/README.md' + - '/context7.json' + - '/greptile.json' + - '/LICENSE' + - '/ruff.toml' + - '/toolshed/**/*' + options: + cache: false + tags: ['ci-ignore'] + + ci-fallback: + deps: + - 'pathfinder:wheel' + - 'bindings:wheel' + - 'core:wheel' + - 'metapackage:wheel' + - 'core:wheel-merge' + - 'pathfinder:sdist' + - 'bindings:sdist' + - 'core:sdist' + - 'metapackage:sdist' + - 'pathfinder:ci-test-linux' + - 'pathfinder:ci-test-windows' + - 'bindings:ci-test-linux' + - 'bindings:ci-test-windows' + - 'bindings:ci-test-assets' + - 'core:ci-test-linux' + - 'core:ci-test-windows' + - 'core:ci-test-assets' + - 'core:ci-test-binaries' + - 'metapackage:ci-test-linux' + - 'metapackage:ci-test-windows' + - 'core:api-check' + - 'docs' + inputs: + - '/.github/actions/**/*' + - '/.github/workflows/build-docs.yml' + - '/.github/workflows/build-wheel.yml' + - '/.github/workflows/ci-nightly.yml' + - '/.github/workflows/ci-pixi-source-test.yml' + - '/.github/workflows/ci.yml' + - '/.github/workflows/coverage.yml' + - '/.github/workflows/release*.yml' + - '/.github/workflows/test-sdist-linux.yml' + - '/.github/workflows/test-sdist-windows.yml' + - '/.moon/workspace.yml' + - '/moon.yml' + - '/cuda_pathfinder/moon.yml' + - '/cuda_bindings/moon.yml' + - '/cuda_core/moon.yml' + - '/cuda_python/moon.yml' + - '/ci/tools/env-vars' + - '/ci/versions.yml' + - '/pyproject.toml' + - '/pytest.ini' + - '/tests/test_moon_ci.py' + options: + cache: false + tags: ['ci-force-all'] diff --git a/tests/test_moon_ci.py b/tests/test_moon_ci.py new file mode 100644 index 00000000000..03b26c675c0 --- /dev/null +++ b/tests/test_moon_ci.py @@ -0,0 +1,654 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Behavior checks for the Moon-owned selective CI graph.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import textwrap +from pathlib import Path +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +MOON = shutil.which("moon") +BASH = shutil.which("bash") + +VISIBLE_TASKS = { + "root": {"test", "docs", "ci-ignore", "ci-fallback"}, + "pathfinder": {"install", "test", "docs", "wheel", "sdist", "ci-test-linux", "ci-test-windows"}, + "bindings": { + "install", + "test", + "docs", + "wheel", + "sdist", + "benchmark", + "ci-test-linux", + "ci-test-windows", + "ci-test-assets", + }, + "core": { + "install", + "test", + "docs", + "wheel", + "wheel-merge", + "sdist", + "api-check", + "ci-test-linux", + "ci-test-windows", + "ci-test-assets", + "ci-test-binaries", + }, + "metapackage": {"docs", "wheel", "sdist", "ci-test-linux", "ci-test-windows"}, +} +INTERNAL_TASKS = { + "pathfinder:test-installed", + "bindings:test-installed", + "bindings:build-cython-tests", + "bindings:benchmark-smoke", + "core:test-installed", + "core:build-cython-tests", + "core:build-test-binaries", + "metapackage:test-installed", +} +ALL_ROUTES = { + f"{project}:ci-test-{os_name}" + for project in ("pathfinder", "bindings", "core", "metapackage") + for os_name in ("linux", "windows") +} +LINUX_ROUTES = {target for target in ALL_ROUTES if target.endswith("-linux")} +WINDOWS_ROUTES = {target for target in ALL_ROUTES if target.endswith("-windows")} +ASSET_ROUTES = {"bindings:ci-test-assets", "core:ci-test-assets", "core:ci-test-binaries"} +PRODUCERS = { + f"{project}:{kind}" for project in ("pathfinder", "bindings", "core", "metapackage") for kind in ("wheel", "sdist") +} | {"core:wheel-merge"} +CI_TASKS = ( + ALL_ROUTES + | ASSET_ROUTES + | PRODUCERS + | {f"{project}:docs" for project in ("root", "pathfinder", "bindings", "core", "metapackage")} + | {"core:api-check", "root:ci-ignore", "root:ci-fallback"} +) + + +def run_moon(*args: str, stdin: str | None = None) -> subprocess.CompletedProcess[str]: + assert MOON is not None + return subprocess.run( # noqa: S603 - MOON resolves to the pinned executable. + [MOON, *args], + cwd=ROOT, + input=stdin, + text=True, + check=False, + capture_output=True, + ) + + +def moon_json(*args: str, stdin: str | None = None) -> dict[str, Any]: + result = run_moon(*args, stdin=stdin) + result.check_returncode() + return json.loads(result.stdout) + + +def targets(payload: dict[str, Any]) -> set[str]: + return {f"{project}:{task}" for project, project_tasks in payload["tasks"].items() for task in project_tasks} + + +def affected(*paths: str) -> set[str]: + payload = moon_json( + "query", + "tasks", + "--affected", + "stdin", + "--upstream", + "none", + "--downstream", + "none", + stdin="".join(f"{path}\n" for path in paths), + ) + return targets(payload) & CI_TASKS + + +def task_graph(target: str) -> dict[str, dict[str, Any]]: + payload = moon_json("task-graph", target, "--json") + return {task["target"]: task for task in payload["data"].values()} + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def workflow_step_script(path: str, name: str) -> str: + lines = read(path).splitlines() + start = lines.index(f" - name: {name}") + run = next(index for index in range(start, len(lines)) if lines[index].strip() == "run: |") + end = next( + (index for index in range(run + 1, len(lines)) if lines[index].startswith(" - name:")), + len(lines), + ) + return textwrap.dedent("\n".join(lines[run + 1 : end])).replace("${{ github.repository }}", "NVIDIA/cuda-python") + + +def baseline_artifacts(*, merge_base: str, expired: str | None = None) -> list[dict[str, object]]: + names = ["cuda-pathfinder-wheel", "cuda-python-wheel"] + for version in ("3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15", "3.15t"): + python = version.replace(".", "") + for platform in ("linux-64", "linux-aarch64", "win-64"): + names.append(f"cuda-bindings-python{python}-cuda13.3.0-{platform}-{merge_base}") + names.append(f"cuda-core-python{python}-{platform}-{merge_base}") + return [{"name": name, "expired": name == expired} for name in names] + + +@pytest.mark.skipif(MOON is None, reason="Moon 2.5.1 is required") +@pytest.mark.agent_authored(model="gpt-5") +class TestMoonCi: + def test_workspace_is_pinned_and_visible_inventory_is_exact(self) -> None: + assert run_moon("--version").stdout.strip() == "moon 2.5.1" + payload = moon_json("query", "tasks") + assert {project: set(project_tasks) for project, project_tasks in payload["tasks"].items()} == VISIBLE_TASKS + assert not (targets(payload) & INTERNAL_TASKS) + + def test_internal_inventory_is_hidden_and_rejects_direct_execution(self) -> None: + graph = task_graph("root:ci-fallback") + internal = {target for target, task in graph.items() if task["options"]["internal"]} + assert internal == INTERNAL_TASKS + for target in sorted(INTERNAL_TASKS): + result = run_moon("run", target, "--upstream", "none", "--downstream", "none") + assert result.returncode != 0 + assert "Unknown task" in result.stderr + + def test_all_tasks_disable_caching_and_routes_are_commandless(self) -> None: + visible = moon_json("query", "tasks")["tasks"] + graph = task_graph("root:ci-fallback") + assert all(task["options"]["cache"] is False for tasks in visible.values() for task in tasks.values()) + assert all(task["options"]["cache"] is False for task in graph.values()) + for target in ( + ALL_ROUTES + | ASSET_ROUTES + | { + "root:ci-ignore", + "root:ci-fallback", + "root:test", + } + ): + project, task = target.split(":") + assert visible[project][task]["command"] == "noop" + + def test_semantic_tag_inventory_is_exact(self) -> None: + payload = moon_json("query", "tasks") + actual = { + target: set(payload["tasks"][target.split(":")[0]][target.split(":")[1]].get("tags", [])) + for target in targets(payload) + if payload["tasks"][target.split(":")[0]][target.split(":")[1]].get("tags") + } + expected = { + "pathfinder:wheel": {"ci-wheel-foundation"}, + "bindings:wheel": {"ci-wheel-bindings"}, + "core:wheel": {"ci-wheel-consumers", "ci-wheel-multi-ctk"}, + "metapackage:wheel": {"ci-wheel-consumers"}, + "core:wheel-merge": {"ci-wheel-finalize"}, + "pathfinder:sdist": {"ci-sdist-foundation"}, + "bindings:sdist": {"ci-sdist-bindings"}, + "core:sdist": {"ci-sdist-consumers"}, + "metapackage:sdist": {"ci-sdist-consumers"}, + "bindings:ci-test-assets": {"ci-test-assets-current"}, + "core:ci-test-assets": {"ci-test-assets-current"}, + "core:ci-test-binaries": {"ci-test-assets-previous"}, + "core:api-check": {"ci-api"}, + "root:ci-ignore": {"ci-ignore"}, + "root:ci-fallback": {"ci-force-all"}, + } + expected.update({target: {"ci-test-linux"} for target in LINUX_ROUTES}) + expected.update({target: {"ci-test-windows"} for target in WINDOWS_ROUTES}) + expected.update({f"{project}:docs": {"ci-docs"} for project in VISIBLE_TASKS}) + assert actual == expected + + def test_package_source_impact_routes(self) -> None: + cases = { + "cuda_pathfinder/cuda/pathfinder/__init__.py": PRODUCERS | ALL_ROUTES | ASSET_ROUTES, + "cuda_bindings/cuda/bindings/__init__.py": { + "bindings:wheel", + "bindings:sdist", + "core:wheel", + "core:wheel-merge", + "core:sdist", + "metapackage:wheel", + "metapackage:sdist", + "bindings:ci-test-linux", + "bindings:ci-test-windows", + "core:ci-test-linux", + "core:ci-test-windows", + "metapackage:ci-test-linux", + "metapackage:ci-test-windows", + } + | ASSET_ROUTES, + "cuda_core/cuda/core/__init__.py": { + "core:wheel", + "core:wheel-merge", + "core:sdist", + "core:api-check", + "core:ci-test-linux", + "core:ci-test-windows", + "metapackage:ci-test-linux", + "metapackage:ci-test-windows", + "core:ci-test-assets", + "core:ci-test-binaries", + }, + "cuda_python/pyproject.toml": { + "bindings:wheel", + "bindings:sdist", + "metapackage:wheel", + "metapackage:sdist", + "metapackage:ci-test-linux", + "metapackage:ci-test-windows", + }, + } + cases["cuda_pathfinder/.git_archival.txt"] = cases["cuda_pathfinder/cuda/pathfinder/__init__.py"] + cases["cuda_bindings/.git_archival.txt"] = cases["cuda_bindings/cuda/bindings/__init__.py"] + cases["cuda_core/.git_archival.txt"] = cases["cuda_core/cuda/core/__init__.py"] + for path, expected in cases.items(): + assert affected(path) == expected, path + + def test_tests_helpers_benchmarks_and_os_infrastructure_impact(self) -> None: + cases = { + "cuda_pathfinder/tests/test_pathfinder.py": { + "pathfinder:ci-test-linux", + "pathfinder:ci-test-windows", + }, + "cuda_bindings/examples/0_Introduction/vectorAddDrv.py": { + "bindings:ci-test-linux", + "bindings:ci-test-windows", + "bindings:ci-test-assets", + }, + "cuda_core/tests/test_device.py": { + "core:ci-test-linux", + "core:ci-test-windows", + "core:ci-test-assets", + "core:ci-test-binaries", + }, + "cuda_python_test_helpers/pyproject.toml": ALL_ROUTES | ASSET_ROUTES, + "benchmarks/cuda_bindings/run_pyperf.py": ALL_ROUTES | ASSET_ROUTES, + "benchmarks/cuda_bindings/compare.py": ALL_ROUTES | ASSET_ROUTES, + "benchmarks/cuda_core/runtime.py": ALL_ROUTES | ASSET_ROUTES, + ".github/workflows/test-wheel-linux.yml": LINUX_ROUTES | ASSET_ROUTES, + ".github/workflows/test-wheel-windows.yml": WINDOWS_ROUTES | ASSET_ROUTES, + "ci/tools/guess_latest.sh": LINUX_ROUTES | ASSET_ROUTES, + } + for path, expected in cases.items(): + assert affected(path) == expected, path + + def test_docs_ignored_unknown_and_fallback_ownership(self) -> None: + assert affected("cuda_core/docs/source/index.rst") == {"core:docs"} + for path in ( + ".coveragerc", + ".github/ISSUE_TEMPLATE/bug_report.yml", + ".github/labeler.yml", + ".pre-commit-config.yaml", + "CONTRIBUTING.md", + "context7.json", + "cuda_core/pixi.toml", + "cuda_core/tests/AGENTS.md", + "diagram.svg", + "greptile.json", + "new-area/pixi.lock", + "ruff.toml", + "toolshed/README.md", + ): + assert affected(path) == {"root:ci-ignore"} + assert affected(".github/workflows/ci.yml") == {"root:ci-fallback"} + assert affected("an-entirely-new-path.txt") == set() + gate = read(".github/workflows/ci.yml") + assert 'length == 0 or any(.[]; .target == "root:ci-fallback")' in gate + + fallback = task_graph("root:ci-fallback")["root:ci-fallback"] + assert {dep["target"] for dep in fallback["deps"]} == ( + PRODUCERS | ALL_ROUTES | ASSET_ROUTES | {"core:api-check", "root:docs"} + ) + for path in (".moon/workspace.yml", "moon.yml", "cuda_core/moon.yml", "ci/versions.yml"): + assert "root:ci-fallback" in affected(path) + + def test_mixed_changes_and_symlink_consumers(self) -> None: + assert affected("cuda_core/docs/source/index.rst", "cuda_bindings/tests/test_api.py") == { + "core:docs", + "bindings:ci-test-linux", + "bindings:ci-test-windows", + "bindings:ci-test-assets", + } + expected_readme = { + "bindings:wheel", + "bindings:sdist", + "metapackage:wheel", + "metapackage:sdist", + "metapackage:ci-test-linux", + "metapackage:ci-test-windows", + } + assert affected("README.md") == expected_readme + assert affected("cuda_python/README.md") == expected_readme + assert affected(".git_archival.txt") == PRODUCERS | ALL_ROUTES | ASSET_ROUTES | {"core:api-check"} + + def test_editable_installs_are_first_class_dependencies(self) -> None: + expected_install_deps = { + "pathfinder:install": set(), + "bindings:install": {"pathfinder:install"}, + "core:install": {"bindings:install"}, + } + graph = task_graph("core:install") + assert set(graph) == set(expected_install_deps) + for target, expected in expected_install_deps.items(): + assert {dep["target"] for dep in graph[target].get("deps", [])} == expected + script = graph[target]["script"] + assert "pip install -e ." in script + assert "../cuda_" not in script + + expected_test_graphs = { + "pathfinder:test": {"pathfinder:install", "pathfinder:test"}, + "bindings:test": {"pathfinder:install", "bindings:install", "bindings:test"}, + "core:test": { + "pathfinder:install", + "bindings:install", + "core:install", + "core:test", + }, + } + for target, expected in expected_test_graphs.items(): + graph = task_graph(target) + assert set(graph) == expected + test_script = graph[target]["script"] + assert "pip install" not in test_script + assert all("wheel" not in graph_target for graph_target in graph) + assert all("cibuildwheel" not in task["script"] for task in graph.values()) + + root_test_graph = task_graph("root:test") + assert {dep["target"] for dep in root_test_graph["root:test"]["deps"]} == { + "pathfinder:test", + "bindings:test", + "core:test", + } + assert root_test_graph["root:test"]["options"]["runDepsInParallel"] is True + + benchmark_graph = task_graph("bindings:benchmark") + assert set(benchmark_graph) == { + "pathfinder:install", + "bindings:install", + "bindings:benchmark", + } + assert "pip install" not in benchmark_graph["bindings:benchmark"]["script"] + + def test_local_core_wheel_builds_current_dependency_chain(self) -> None: + assert set(task_graph("core:wheel")) == { + "pathfinder:wheel", + "bindings:wheel", + "core:wheel", + } + + def test_ci_routes_have_only_hidden_direct_executors(self) -> None: + expected = { + "pathfinder:ci-test-linux": {"pathfinder:test-installed"}, + "pathfinder:ci-test-windows": {"pathfinder:test-installed"}, + "bindings:ci-test-linux": {"bindings:test-installed", "bindings:benchmark-smoke"}, + "bindings:ci-test-windows": {"bindings:test-installed"}, + "core:ci-test-linux": {"core:test-installed"}, + "core:ci-test-windows": {"core:test-installed"}, + "metapackage:ci-test-linux": {"metapackage:test-installed"}, + "metapackage:ci-test-windows": {"metapackage:test-installed"}, + } + fallback = task_graph("root:ci-fallback") + for route, direct_targets in expected.items(): + actual = {dep["target"] for dep in fallback[route]["deps"]} + assert actual == direct_targets + assert all(fallback[target]["options"]["internal"] for target in actual) + assert all(fallback[target].get("deps") for target in actual) + for workflow in (".github/workflows/test-wheel-linux.yml", ".github/workflows/test-wheel-windows.yml"): + assert 'moon run "${target_args[@]}" --upstream direct --downstream none' in read(workflow) + + def test_build_traversal_stages_dependencies_and_runs_exact_targets(self) -> None: + workflow = read(".github/workflows/build-wheel.yml") + for phase in ( + "WHEEL_FOUNDATION_TARGETS", + "WHEEL_BINDINGS_TARGETS", + "WHEEL_CONSUMER_TARGETS", + "WHEEL_MULTI_CTK_TARGETS", + "WHEEL_FINALIZE_TARGETS", + ): + assert phase in workflow + assert workflow.count("--upstream none --downstream none") >= 5 + assert workflow.count("--upstream direct --downstream none") >= 2 + assert "Download reusable cuda.pathfinder wheel" in workflow + assert "Download reusable cuda.bindings wheel" in workflow + assert workflow.count("python -m pip install cibuildwheel twine wheel") == 2 + + def test_native_assets_follow_the_selected_os(self, tmp_path: Path) -> None: + assert BASH is not None + script = workflow_step_script(".github/workflows/build-wheel.yml", "Resolve Moon phase targets") + common = { + "WHEEL_FOUNDATION_TARGETS": "[]", + "WHEEL_BINDINGS_TARGETS": "[]", + "WHEEL_CONSUMER_TARGETS": "[]", + "WHEEL_MULTI_CTK_TARGETS": "[]", + "WHEEL_FINALIZE_TARGETS": "[]", + "TEST_ASSETS_CURRENT_TARGETS": '["bindings:ci-test-assets","core:ci-test-assets"]', + "TEST_ASSETS_PREVIOUS_TARGETS": '["core:ci-test-binaries"]', + } + cases = { + "linux-selected": { + "TEST_LINUX_TARGETS": '["core:ci-test-linux"]', + "TEST_WINDOWS_TARGETS": "[]", + "linux-64": "true", + "win-64": "false", + }, + "windows-selected": { + "TEST_LINUX_TARGETS": "[]", + "TEST_WINDOWS_TARGETS": '["core:ci-test-windows"]', + "linux-64": "false", + "win-64": "true", + }, + } + for case_name, case in cases.items(): + for platform in ("linux-64", "win-64"): + output = tmp_path / f"{case_name}-{platform}.env" + env = ( + os.environ + | common + | { + "HOST_PLATFORM": platform, + "GITHUB_ENV": str(output), + "TEST_LINUX_TARGETS": case["TEST_LINUX_TARGETS"], + "TEST_WINDOWS_TARGETS": case["TEST_WINDOWS_TARGETS"], + } + ) + result = subprocess.run( # noqa: S603 - controlled repository script. + [BASH, "-c", script], + cwd=ROOT, + env=env, + text=True, + check=False, + capture_output=True, + ) + assert result.returncode == 0, (case_name, platform, result.stderr) + values = dict(line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines()) + assert values["TEST_BINDINGS"] == case[platform] + assert values["TEST_CORE_CURRENT"] == case[platform] + assert values["TEST_CORE_PREVIOUS"] == case[platform] + + workflow = read(".github/workflows/ci.yml") + linux_arm = workflow.split(" build-linux-aarch64:", 1)[1].split(" build-windows:", 1)[0] + windows = workflow.split(" build-windows:", 1)[1].split(" test-sdist-linux:", 1)[0] + assert "ci-test-assets" not in linux_arm + assert "ci-test-assets" not in windows + + def test_core_uses_one_target_in_both_toolkits_then_merges(self) -> None: + graph = task_graph("root:ci-fallback") + assert set(graph["core:wheel"]["tags"]) == {"ci-wheel-consumers", "ci-wheel-multi-ctk"} + assert not graph["core:wheel-merge"].get("deps") + merger = graph["core:wheel-merge"]["script"] + assert "dist/cu12/*.whl dist/cu13/*.whl" in merger + assert "merge_cuda_core_wheels.py" in merger + + def test_only_merged_core_wheel_is_in_baseline_artifact(self) -> None: + workflow = read(".github/workflows/build-wheel.yml") + assert "name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}" in workflow + assert "path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl" in workflow + assert "path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu" not in workflow + for name in ("cuda-pathfinder-wheel", "cuda-python-wheel"): + assert f"name: {name}" in workflow + + def test_baseline_reuse_requires_one_exact_successful_complete_set(self) -> None: + workflow = read(".github/workflows/ci.yml") + for contract in ( + '--commit "${merge_base}"', + "--event push", + "--status success", + "if [[ $(jq 'length' <<< \"$runs\") -ne 1 ]]", + '"${run_sha}" != "${merge_base}"', + "length == 1 and .[0].expired == false", + "if (( ${#missing[@]} != 0 ))", + 'baseline_run_id=""', + 'baseline_sha=""', + ): + assert contract in workflow + assert "cuda-pathfinder-wheel cuda-python-wheel" in workflow + assert "CUDA_BINDINGS_ARTIFACT_BASENAME" in read(".github/workflows/build-wheel.yml") + assert "CUDA_CORE_ARTIFACT_BASENAME" in read(".github/workflows/build-wheel.yml") + assert "uvx --from pytest pytest -q tests/test_moon_ci.py" in workflow + + def test_public_automation_roots_are_nvidia_only(self) -> None: + workflow = read(".github/workflows/ci.yml") + assert workflow.count(" if: ${{ github.repository_owner == 'nvidia' }}") == 3 + assert " if: ${{ always() && github.repository_owner == 'nvidia' }}" in workflow + + def test_baseline_reuse_behaviors(self, tmp_path: Path) -> None: + assert BASH is not None + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + gh = fake_bin / "gh" + gh.write_text( + """#!/usr/bin/env bash +printf '%s\\n' "$*" >> "$MOCK_GH_LOG" +if [[ "$1 $2" == "run list" ]]; then + printf '%s\\n' "$MOCK_RUNS" + exit "$MOCK_RUN_STATUS" +fi +if [[ "$1" == "api" ]]; then + printf '%s\\n' "$MOCK_ARTIFACTS" + exit "$MOCK_ARTIFACT_STATUS" +fi +exit 2 +""", + encoding="utf-8", + ) + yq = fake_bin / "yq" + yq.write_text( + """#!/usr/bin/env bash +if [[ "$1" == "-r" ]]; then + printf '%s\\n' 3.10 3.11 3.12 3.13 3.14 3.14t 3.15 3.15t +else + printf '%s\\n' 13.3.0 +fi +""", + encoding="utf-8", + ) + os.chmod(gh, 0o700) + os.chmod(yq, 0o700) + + merge_base = "exact-base" + complete = baseline_artifacts(merge_base=merge_base) + cases = { + "complete": { + "runs": [{"databaseId": 42, "headSha": merge_base}], + "artifacts": complete, + "accepted": True, + }, + "incomplete": { + "runs": [{"databaseId": 42, "headSha": merge_base}], + "artifacts": complete[:-1], + "accepted": False, + }, + "expired": { + "runs": [{"databaseId": 42, "headSha": merge_base}], + "artifacts": baseline_artifacts(merge_base=merge_base, expired="cuda-pathfinder-wheel"), + "accepted": False, + }, + "failed-run": {"runs": [], "artifacts": complete, "accepted": False}, + "wrong-sha": { + "runs": [{"databaseId": 42, "headSha": "another-sha"}], + "artifacts": complete, + "accepted": False, + }, + "duplicate": { + "runs": [{"databaseId": 42, "headSha": merge_base}], + "artifacts": [*complete, complete[0]], + "accepted": False, + }, + "lookup-failure": { + "runs": [{"databaseId": 42, "headSha": merge_base}], + "artifacts": complete, + "accepted": False, + "run_status": 1, + }, + } + script = workflow_step_script(".github/workflows/ci.yml", "Resolve reusable base artifacts") + for name, case in cases.items(): + output = tmp_path / f"{name}.output" + summary = tmp_path / f"{name}.summary" + log = tmp_path / f"{name}.gh.log" + output.touch() + summary.touch() + env = os.environ | { + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "BASE_REF": "main", + "MERGE_BASE": merge_base, + "GITHUB_OUTPUT": str(output), + "GITHUB_STEP_SUMMARY": str(summary), + "MOCK_GH_LOG": str(log), + "MOCK_RUNS": json.dumps(case["runs"]), + "MOCK_ARTIFACTS": "\n".join(json.dumps(item) for item in case["artifacts"]), + "MOCK_RUN_STATUS": str(case.get("run_status", 0)), + "MOCK_ARTIFACT_STATUS": "0", + } + result = subprocess.run( # noqa: S603 - controlled script and fake tools. + [BASH, "-c", script], + cwd=ROOT, + env=env, + text=True, + check=False, + capture_output=True, + ) + assert result.returncode == 0, (name, result.stderr) + accepted = "run_id=42" in output.read_text(encoding="utf-8") + assert accepted is case["accepted"], name + if not case["accepted"]: + assert "No complete reusable artifact set" in summary.read_text(encoding="utf-8") + + complete_log = (tmp_path / "complete.gh.log").read_text(encoding="utf-8") + for argument in ("--commit exact-base", "--event push", "--status success"): + assert argument in complete_log + + def test_docs_select_component_or_parallel_aggregate_layout(self) -> None: + workflow = read(".github/workflows/build-docs.yml") + assert "all) targets='[\"root:docs\"]'" in workflow + for project in ("pathfinder", "bindings", "core", "metapackage"): + assert f'"{project}:docs"' in workflow + assert "DOCS_BUILD_ARGS" in workflow + assert "--upstream deep --downstream none" in workflow + assert "DOCS_USE_MOON" in workflow + assert "./build_all_docs.sh latest-only" in workflow + assert "./build_docs.sh latest-only" in workflow + root_docs = task_graph("root:docs") + assert {dep["target"] for dep in root_docs["root:docs"]["deps"]} == { + "pathfinder:docs", + "bindings:docs", + "core:docs", + "metapackage:docs", + } + script = root_docs["root:docs"]["script"] + for destination in ("cuda-bindings", "cuda-core", "cuda-pathfinder"): + assert f"cuda_python/docs/build/html/{destination}" in script + for project in ("pathfinder", "bindings", "core", "metapackage"): + assert "${DOCS_BUILD_ARGS:-}" in root_docs[f"{project}:docs"]["script"] diff --git a/toolshed/check_spdx.py b/toolshed/check_spdx.py index d4c9430673c..ce422aef997 100644 --- a/toolshed/check_spdx.py +++ b/toolshed/check_spdx.py @@ -23,6 +23,7 @@ TOP_LEVEL_DIRS_LICENSE_IDENTIFIERS = { ".agents": "Apache-2.0", ".github": "Apache-2.0", + ".moon": "Apache-2.0", "benchmarks": "Apache-2.0", "ci": "Apache-2.0", "cuda_bindings": "Apache-2.0", @@ -32,6 +33,7 @@ "cuda_python_test_helpers": "Apache-2.0", "qa": "LicenseRef-NVIDIA-SOFTWARE-LICENSE", "scripts": "Apache-2.0", + "tests": "Apache-2.0", "toolshed": "Apache-2.0", }