diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 632ae001dcb2..a2314098f843 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -30,6 +30,17 @@ # It is being rolled out incrementally across all workflows. # Track progress at https://github.com/apache/datafusion/issues/24487. # +# # Pushes to `main` +# +# Commits reach upstream `main` through the merge queue, which runs this +# workflow at the same SHA. A push to upstream `main` therefore runs only the +# jobs that save caches, publish coverage, or run the non-required FFI check. +# The other jobs carry an `if:` that skips them for that event alone. PRs, the +# merge queue, `workflow_dispatch`, and every other branch run all jobs. A +# direct push that bypasses the queue is not re-checked; dispatch the workflow +# on `main` for a full run. `ci/scripts/check_asf_yaml_status_checks.py` keeps +# the skipped set explicit. +# name: Rust concurrency: @@ -80,6 +91,7 @@ jobs: run: cargo xtask ci step check workspace - name: Archive Cargo check artifacts id: archive-check + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} continue-on-error: true shell: bash run: | @@ -91,7 +103,7 @@ jobs: --exclude='registry/src/*/*/false' \ -cf "$RUNNER_TEMP/cargo-check/check-registry.tar" -C "$CARGO_HOME" registry - name: Upload Cargo check artifacts - if: steps.archive-check.outcome == 'success' + if: ${{ steps.archive-check.outcome == 'success' && !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -109,6 +121,7 @@ jobs: linux-datafusion-common-features: name: cargo check datafusion-common features needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust @@ -139,6 +152,7 @@ jobs: linux-datafusion-substrait-features: name: cargo check datafusion-substrait features needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} env: CARGO_INCREMENTAL: "0" runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} @@ -185,6 +199,7 @@ jobs: linux-datafusion-proto-features: name: cargo check datafusion-proto features needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -247,6 +262,7 @@ jobs: linux-cargo-check-datafusion: name: cargo check datafusion features needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} env: CARGO_INCREMENTAL: "0" runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} @@ -367,6 +383,7 @@ jobs: linux-cargo-check-datafusion-spark: name: cargo check datafusion-spark features needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust @@ -448,6 +465,7 @@ jobs: linux-test-datafusion-cli: name: cargo test datafusion-cli (amd64) needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} steps: - uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1 @@ -525,6 +543,7 @@ jobs: linux-test-doc: name: cargo test doc (amd64) needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -547,6 +566,7 @@ jobs: linux-rustdoc: name: cargo doc needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -562,6 +582,7 @@ jobs: linux-wasm-pack: name: build and run with wasm-pack + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -589,6 +610,7 @@ jobs: verify-benchmark-results: name: verify benchmark results (amd64) needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -621,6 +643,7 @@ jobs: sqllogictest-postgres: name: "Run sqllogictest with Postgres runner" needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -658,6 +681,7 @@ jobs: sqllogictest-substrait: name: "Run sqllogictest in Substrait round-trip mode" needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -718,6 +742,7 @@ jobs: vendor: name: Verify Vendored Code + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust @@ -735,6 +760,7 @@ jobs: check-fmt: name: Check cargo fmt + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust @@ -750,6 +776,7 @@ jobs: check-workflow-tool-installs: name: Check GitHub Actions install tooling + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -787,6 +814,7 @@ jobs: cargo-toml-formatting-checks: name: check Cargo.toml formatting needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust @@ -810,6 +838,7 @@ jobs: config-docs-check: name: check configs.md and ***_functions.md is up-to-date needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -845,6 +874,7 @@ jobs: examples-docs-check: name: check example README is up-to-date needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust @@ -877,6 +907,7 @@ jobs: # - datafusion-cli msrv: name: Verify MSRV (Min Supported Rust Version) + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust diff --git a/ci/scripts/check_asf_yaml_status_checks.py b/ci/scripts/check_asf_yaml_status_checks.py index 135654159051..1bd2852f4e0d 100755 --- a/ci/scripts/check_asf_yaml_status_checks.py +++ b/ci/scripts/check_asf_yaml_status_checks.py @@ -24,6 +24,11 @@ A typo or stale entry in required_status_checks will block all merges for the project, so this check catches that early. + +Also validate the post-merge conditions in rust.yml: exactly the jobs +in POST_MERGE_SKIPPED_JOBS carry the condition that skips them on +pushes to upstream `main`, and the Cargo check artifact steps are +guarded the same way. """ import glob @@ -32,6 +37,57 @@ import yaml +# Commits reach upstream `main` through the merge queue, which already ran +# rust.yml at the same SHA. On a push to upstream `main`, rust.yml skips the +# jobs below and keeps only the jobs that save caches, publish coverage, or run +# the non-required FFI check. A job is skipped only when it is listed here and +# carries exactly POST_MERGE_SKIP_CONDITION, so a new job runs by default and +# changing the set is a deliberate edit to both places. +RUST_WORKFLOW = "rust.yml" +POST_MERGE_PUSH = ( + "github.event_name == 'push'" + " && github.ref == 'refs/heads/main'" + " && github.repository == 'apache/datafusion'" +) +POST_MERGE_SKIP_CONDITION = "${{ !(" + POST_MERGE_PUSH + ") }}" +POST_MERGE_SKIPPED_JOBS = frozenset( + { + "cargo-toml-formatting-checks", + "check-fmt", + "check-workflow-tool-installs", + "config-docs-check", + "examples-docs-check", + "linux-cargo-check-datafusion", + "linux-cargo-check-datafusion-spark", + "linux-datafusion-common-features", + "linux-datafusion-proto-features", + "linux-datafusion-substrait-features", + "linux-rustdoc", + "linux-test-datafusion-cli", + "linux-test-doc", + "linux-wasm-pack", + "msrv", + "sqllogictest-postgres", + "sqllogictest-substrait", + "vendor", + "verify-benchmark-results", + } +) +# `linux-build-lib` uploads its Cargo check artifacts for two skipped jobs, so +# the archive and upload steps skip on the same pushes. +ARTIFACT_JOB = "linux-build-lib" +ARTIFACT_ARCHIVE_STEP = "archive-check" +ARTIFACT_NAME = "cargo-check" +ARTIFACT_UPLOAD_CONDITION = ( + "${{ steps." + ARTIFACT_ARCHIVE_STEP + ".outcome == 'success'" + " && !(" + POST_MERGE_PUSH + ") }}" +) + + +def load_workflow(path): + with open(path) as f: + return yaml.safe_load(f) + def get_required_checks(asf_yaml_path): """Extract all required_status_checks contexts from .asf.yaml.""" @@ -58,8 +114,7 @@ def get_workflow_jobs(workflows_dir): """ jobs = {} # identifier -> [(workflow_file, has_path_filters)] for workflow_file in sorted(glob.glob(os.path.join(workflows_dir, "*.yml"))): - with open(workflow_file) as f: - workflow = yaml.safe_load(f) + workflow = load_workflow(workflow_file) if not workflow or "jobs" not in workflow: continue @@ -85,6 +140,90 @@ def get_workflow_jobs(workflows_dir): return jobs +def iter_steps(job_config): + """Yield the steps of a job, flattening `parallel:` groups.""" + for step in job_config.get("steps") or []: + if not isinstance(step, dict): + continue + if "parallel" in step: + yield from (s for s in step["parallel"] or [] if isinstance(s, dict)) + else: + yield step + + +def uses_action(step, action): + return str(step.get("uses", "")).startswith(action) + + +def check_post_merge_conditions(workflow): + """Return the problems with rust.yml's post-merge job and artifact conditions.""" + errors = [] + jobs = { + key: config + for key, config in (workflow.get("jobs") or {}).items() + if isinstance(config, dict) + } + + for job in sorted(POST_MERGE_SKIPPED_JOBS - set(jobs)): + errors.append( + f" - job `{job}` is listed in POST_MERGE_SKIPPED_JOBS " + f"but does not exist in {RUST_WORKFLOW}" + ) + for job, config in sorted(jobs.items()): + condition = config.get("if") + if job in POST_MERGE_SKIPPED_JOBS: + if condition != POST_MERGE_SKIP_CONDITION: + errors.append( + f" - job `{job}` must carry exactly " + f"`if: {POST_MERGE_SKIP_CONDITION}`, found {condition!r}" + ) + elif condition is not None and any( + token in str(condition) + for token in ("github.event_name", "github.ref", "refs/heads/") + ): + errors.append( + f" - job `{job}` is not listed in POST_MERGE_SKIPPED_JOBS " + f"but carries an event condition: {condition!r}" + ) + + build = jobs.get(ARTIFACT_JOB) + if build is None: + errors.append(f" - job `{ARTIFACT_JOB}` does not exist in {RUST_WORKFLOW}") + else: + steps = list(iter_steps(build)) + archive = [s for s in steps if s.get("id") == ARTIFACT_ARCHIVE_STEP] + uploads = [s for s in steps if uses_action(s, "actions/upload-artifact")] + if len(archive) != 1 or archive[0].get("if") != POST_MERGE_SKIP_CONDITION: + found = [s.get("if") for s in archive] + errors.append( + f" - the step `id: {ARTIFACT_ARCHIVE_STEP}` in `{ARTIFACT_JOB}` " + f"must exist once and carry exactly " + f"`if: {POST_MERGE_SKIP_CONDITION}`, found {found!r}" + ) + if len(uploads) != 1 or uploads[0].get("if") != ARTIFACT_UPLOAD_CONDITION: + found = [s.get("if") for s in uploads] + errors.append( + f" - the upload-artifact step in `{ARTIFACT_JOB}` must exist " + f"once and carry exactly `if: {ARTIFACT_UPLOAD_CONDITION}`, " + f"found {found!r}" + ) + + for job, config in sorted(jobs.items()): + downloads = [ + s + for s in iter_steps(config) + if uses_action(s, "actions/download-artifact") + and (s.get("with") or {}).get("name") == ARTIFACT_NAME + ] + if downloads and job not in POST_MERGE_SKIPPED_JOBS: + errors.append( + f" - job `{job}` downloads the `{ARTIFACT_NAME}` artifact, which " + f"is not uploaded on pushes to main, so it must be listed in " + f"POST_MERGE_SKIPPED_JOBS" + ) + return errors + + def main(): repo_root = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -95,7 +234,6 @@ def main(): required_checks = get_required_checks(asf_yaml) if not required_checks: print("No required_status_checks found in .asf.yaml — nothing to validate.") - return jobs = get_workflow_jobs(workflows_dir) errors = [] @@ -126,6 +264,10 @@ def main(): f"and will block merging" ) + post_merge_errors = check_post_merge_conditions( + load_workflow(os.path.join(workflows_dir, RUST_WORKFLOW)) + ) + if errors: print("ERROR: Problems found with required_status_checks in .asf.yaml:\n") print("\n".join(errors)) @@ -133,11 +275,23 @@ def main(): print("Available job names across all workflows:") for name in sorted(jobs): print(f" - {name}") + if post_merge_errors: + print( + f"ERROR: Problems found with post-merge conditions in {RUST_WORKFLOW}:\n" + ) + print("\n".join(post_merge_errors)) + print() + if errors or post_merge_errors: sys.exit(1) + if required_checks: + print( + f"OK: All {len(required_checks)} required_status_checks " + "match existing GitHub Actions jobs." + ) print( - f"OK: All {len(required_checks)} required_status_checks " - "match existing GitHub Actions jobs." + f"OK: {RUST_WORKFLOW} skips exactly {len(POST_MERGE_SKIPPED_JOBS)} " + "jobs on pushes to main." ) diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md index dc4430ebf4a1..d519aa518d13 100644 --- a/docs/source/contributor-guide/testing.md +++ b/docs/source/contributor-guide/testing.md @@ -223,8 +223,9 @@ bash ci/scripts/rust_docs.sh ## ASF Status Check Validation `ci/scripts/check_asf_yaml_status_checks.py` checks that every required status -check in `.asf.yaml` matches a job in `.github/workflows`. `./dev/rust_lint.sh` -runs it and needs `python3` with [PyYAML]. The [uv] workspace provides both: +check in `.asf.yaml` matches a job in `.github/workflows`, and that `rust.yml` +skips only its listed jobs on pushes to `main`. `./dev/rust_lint.sh` runs it +and needs `python3` with [PyYAML]. The [uv] workspace provides both: ```shell uv run ./dev/rust_lint.sh