From 1a3b57054efc69326d8dba5f359bca7c1273c444 Mon Sep 17 00:00:00 2001 From: Kumar Ujjawal Date: Wed, 16 Sep 2026 13:54:07 +0530 Subject: [PATCH] ci: skip queue-verified Rust jobs on pushes to main Commits reach upstream main through the merge queue, which runs the Rust workflow at the same SHA. The push to main then ran the same 26 jobs again. Only the jobs that save caches, publish coverage, or run the non-required FFI check give new output after the merge. Add an `if:` condition to the other 19 jobs and to the Cargo check artifact steps, so they skip on pushes to upstream main. PRs, the merge queue, workflow_dispatch, and other branches run all jobs. Extend check_asf_yaml_status_checks.py to require that exactly the listed jobs carry this condition. Part of #25148 --- .github/workflows/rust.yml | 33 ++++- ci/scripts/check_asf_yaml_status_checks.py | 164 ++++++++++++++++++++- docs/source/contributor-guide/testing.md | 5 +- 3 files changed, 194 insertions(+), 8 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 632ae001dcb29..a2314098f8433 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -30,6 +30,17 @@ # It is being rolled out incrementally across all workflows. # Track progress at https://github.com/apache/datafusion/issues/24487. # +# # Pushes to `main` +# +# Commits reach upstream `main` through the merge queue, which runs this +# workflow at the same SHA. A push to upstream `main` therefore runs only the +# jobs that save caches, publish coverage, or run the non-required FFI check. +# The other jobs carry an `if:` that skips them for that event alone. PRs, the +# merge queue, `workflow_dispatch`, and every other branch run all jobs. A +# direct push that bypasses the queue is not re-checked; dispatch the workflow +# on `main` for a full run. `ci/scripts/check_asf_yaml_status_checks.py` keeps +# the skipped set explicit. +# name: Rust concurrency: @@ -80,6 +91,7 @@ jobs: run: cargo xtask ci step check workspace - name: Archive Cargo check artifacts id: archive-check + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} continue-on-error: true shell: bash run: | @@ -91,7 +103,7 @@ jobs: --exclude='registry/src/*/*/false' \ -cf "$RUNNER_TEMP/cargo-check/check-registry.tar" -C "$CARGO_HOME" registry - name: Upload Cargo check artifacts - if: steps.archive-check.outcome == 'success' + if: ${{ steps.archive-check.outcome == 'success' && !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -109,6 +121,7 @@ jobs: linux-datafusion-common-features: name: cargo check datafusion-common features needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust @@ -139,6 +152,7 @@ jobs: linux-datafusion-substrait-features: name: cargo check datafusion-substrait features needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} env: CARGO_INCREMENTAL: "0" runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} @@ -185,6 +199,7 @@ jobs: linux-datafusion-proto-features: name: cargo check datafusion-proto features needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -247,6 +262,7 @@ jobs: linux-cargo-check-datafusion: name: cargo check datafusion features needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} env: CARGO_INCREMENTAL: "0" runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} @@ -367,6 +383,7 @@ jobs: linux-cargo-check-datafusion-spark: name: cargo check datafusion-spark features needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust @@ -448,6 +465,7 @@ jobs: linux-test-datafusion-cli: name: cargo test datafusion-cli (amd64) needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} steps: - uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1 @@ -525,6 +543,7 @@ jobs: linux-test-doc: name: cargo test doc (amd64) needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -547,6 +566,7 @@ jobs: linux-rustdoc: name: cargo doc needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -562,6 +582,7 @@ jobs: linux-wasm-pack: name: build and run with wasm-pack + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -589,6 +610,7 @@ jobs: verify-benchmark-results: name: verify benchmark results (amd64) needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -621,6 +643,7 @@ jobs: sqllogictest-postgres: name: "Run sqllogictest with Postgres runner" needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -658,6 +681,7 @@ jobs: sqllogictest-substrait: name: "Run sqllogictest in Substrait round-trip mode" needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -718,6 +742,7 @@ jobs: vendor: name: Verify Vendored Code + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust @@ -735,6 +760,7 @@ jobs: check-fmt: name: Check cargo fmt + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust @@ -750,6 +776,7 @@ jobs: check-workflow-tool-installs: name: Check GitHub Actions install tooling + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -787,6 +814,7 @@ jobs: cargo-toml-formatting-checks: name: check Cargo.toml formatting needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust @@ -810,6 +838,7 @@ jobs: config-docs-check: name: check configs.md and ***_functions.md is up-to-date needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -845,6 +874,7 @@ jobs: examples-docs-check: name: check example README is up-to-date needs: linux-build-lib + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust @@ -877,6 +907,7 @@ jobs: # - datafusion-cli msrv: name: Verify MSRV (Min Supported Rust Version) + if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }} runs-on: ubuntu-latest container: image: amd64/rust diff --git a/ci/scripts/check_asf_yaml_status_checks.py b/ci/scripts/check_asf_yaml_status_checks.py index 135654159051c..1bd2852f4e0d8 100755 --- a/ci/scripts/check_asf_yaml_status_checks.py +++ b/ci/scripts/check_asf_yaml_status_checks.py @@ -24,6 +24,11 @@ A typo or stale entry in required_status_checks will block all merges for the project, so this check catches that early. + +Also validate the post-merge conditions in rust.yml: exactly the jobs +in POST_MERGE_SKIPPED_JOBS carry the condition that skips them on +pushes to upstream `main`, and the Cargo check artifact steps are +guarded the same way. """ import glob @@ -32,6 +37,57 @@ import yaml +# Commits reach upstream `main` through the merge queue, which already ran +# rust.yml at the same SHA. On a push to upstream `main`, rust.yml skips the +# jobs below and keeps only the jobs that save caches, publish coverage, or run +# the non-required FFI check. A job is skipped only when it is listed here and +# carries exactly POST_MERGE_SKIP_CONDITION, so a new job runs by default and +# changing the set is a deliberate edit to both places. +RUST_WORKFLOW = "rust.yml" +POST_MERGE_PUSH = ( + "github.event_name == 'push'" + " && github.ref == 'refs/heads/main'" + " && github.repository == 'apache/datafusion'" +) +POST_MERGE_SKIP_CONDITION = "${{ !(" + POST_MERGE_PUSH + ") }}" +POST_MERGE_SKIPPED_JOBS = frozenset( + { + "cargo-toml-formatting-checks", + "check-fmt", + "check-workflow-tool-installs", + "config-docs-check", + "examples-docs-check", + "linux-cargo-check-datafusion", + "linux-cargo-check-datafusion-spark", + "linux-datafusion-common-features", + "linux-datafusion-proto-features", + "linux-datafusion-substrait-features", + "linux-rustdoc", + "linux-test-datafusion-cli", + "linux-test-doc", + "linux-wasm-pack", + "msrv", + "sqllogictest-postgres", + "sqllogictest-substrait", + "vendor", + "verify-benchmark-results", + } +) +# `linux-build-lib` uploads its Cargo check artifacts for two skipped jobs, so +# the archive and upload steps skip on the same pushes. +ARTIFACT_JOB = "linux-build-lib" +ARTIFACT_ARCHIVE_STEP = "archive-check" +ARTIFACT_NAME = "cargo-check" +ARTIFACT_UPLOAD_CONDITION = ( + "${{ steps." + ARTIFACT_ARCHIVE_STEP + ".outcome == 'success'" + " && !(" + POST_MERGE_PUSH + ") }}" +) + + +def load_workflow(path): + with open(path) as f: + return yaml.safe_load(f) + def get_required_checks(asf_yaml_path): """Extract all required_status_checks contexts from .asf.yaml.""" @@ -58,8 +114,7 @@ def get_workflow_jobs(workflows_dir): """ jobs = {} # identifier -> [(workflow_file, has_path_filters)] for workflow_file in sorted(glob.glob(os.path.join(workflows_dir, "*.yml"))): - with open(workflow_file) as f: - workflow = yaml.safe_load(f) + workflow = load_workflow(workflow_file) if not workflow or "jobs" not in workflow: continue @@ -85,6 +140,90 @@ def get_workflow_jobs(workflows_dir): return jobs +def iter_steps(job_config): + """Yield the steps of a job, flattening `parallel:` groups.""" + for step in job_config.get("steps") or []: + if not isinstance(step, dict): + continue + if "parallel" in step: + yield from (s for s in step["parallel"] or [] if isinstance(s, dict)) + else: + yield step + + +def uses_action(step, action): + return str(step.get("uses", "")).startswith(action) + + +def check_post_merge_conditions(workflow): + """Return the problems with rust.yml's post-merge job and artifact conditions.""" + errors = [] + jobs = { + key: config + for key, config in (workflow.get("jobs") or {}).items() + if isinstance(config, dict) + } + + for job in sorted(POST_MERGE_SKIPPED_JOBS - set(jobs)): + errors.append( + f" - job `{job}` is listed in POST_MERGE_SKIPPED_JOBS " + f"but does not exist in {RUST_WORKFLOW}" + ) + for job, config in sorted(jobs.items()): + condition = config.get("if") + if job in POST_MERGE_SKIPPED_JOBS: + if condition != POST_MERGE_SKIP_CONDITION: + errors.append( + f" - job `{job}` must carry exactly " + f"`if: {POST_MERGE_SKIP_CONDITION}`, found {condition!r}" + ) + elif condition is not None and any( + token in str(condition) + for token in ("github.event_name", "github.ref", "refs/heads/") + ): + errors.append( + f" - job `{job}` is not listed in POST_MERGE_SKIPPED_JOBS " + f"but carries an event condition: {condition!r}" + ) + + build = jobs.get(ARTIFACT_JOB) + if build is None: + errors.append(f" - job `{ARTIFACT_JOB}` does not exist in {RUST_WORKFLOW}") + else: + steps = list(iter_steps(build)) + archive = [s for s in steps if s.get("id") == ARTIFACT_ARCHIVE_STEP] + uploads = [s for s in steps if uses_action(s, "actions/upload-artifact")] + if len(archive) != 1 or archive[0].get("if") != POST_MERGE_SKIP_CONDITION: + found = [s.get("if") for s in archive] + errors.append( + f" - the step `id: {ARTIFACT_ARCHIVE_STEP}` in `{ARTIFACT_JOB}` " + f"must exist once and carry exactly " + f"`if: {POST_MERGE_SKIP_CONDITION}`, found {found!r}" + ) + if len(uploads) != 1 or uploads[0].get("if") != ARTIFACT_UPLOAD_CONDITION: + found = [s.get("if") for s in uploads] + errors.append( + f" - the upload-artifact step in `{ARTIFACT_JOB}` must exist " + f"once and carry exactly `if: {ARTIFACT_UPLOAD_CONDITION}`, " + f"found {found!r}" + ) + + for job, config in sorted(jobs.items()): + downloads = [ + s + for s in iter_steps(config) + if uses_action(s, "actions/download-artifact") + and (s.get("with") or {}).get("name") == ARTIFACT_NAME + ] + if downloads and job not in POST_MERGE_SKIPPED_JOBS: + errors.append( + f" - job `{job}` downloads the `{ARTIFACT_NAME}` artifact, which " + f"is not uploaded on pushes to main, so it must be listed in " + f"POST_MERGE_SKIPPED_JOBS" + ) + return errors + + def main(): repo_root = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -95,7 +234,6 @@ def main(): required_checks = get_required_checks(asf_yaml) if not required_checks: print("No required_status_checks found in .asf.yaml — nothing to validate.") - return jobs = get_workflow_jobs(workflows_dir) errors = [] @@ -126,6 +264,10 @@ def main(): f"and will block merging" ) + post_merge_errors = check_post_merge_conditions( + load_workflow(os.path.join(workflows_dir, RUST_WORKFLOW)) + ) + if errors: print("ERROR: Problems found with required_status_checks in .asf.yaml:\n") print("\n".join(errors)) @@ -133,11 +275,23 @@ def main(): print("Available job names across all workflows:") for name in sorted(jobs): print(f" - {name}") + if post_merge_errors: + print( + f"ERROR: Problems found with post-merge conditions in {RUST_WORKFLOW}:\n" + ) + print("\n".join(post_merge_errors)) + print() + if errors or post_merge_errors: sys.exit(1) + if required_checks: + print( + f"OK: All {len(required_checks)} required_status_checks " + "match existing GitHub Actions jobs." + ) print( - f"OK: All {len(required_checks)} required_status_checks " - "match existing GitHub Actions jobs." + f"OK: {RUST_WORKFLOW} skips exactly {len(POST_MERGE_SKIPPED_JOBS)} " + "jobs on pushes to main." ) diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md index dc4430ebf4a1f..d519aa518d13b 100644 --- a/docs/source/contributor-guide/testing.md +++ b/docs/source/contributor-guide/testing.md @@ -223,8 +223,9 @@ bash ci/scripts/rust_docs.sh ## ASF Status Check Validation `ci/scripts/check_asf_yaml_status_checks.py` checks that every required status -check in `.asf.yaml` matches a job in `.github/workflows`. `./dev/rust_lint.sh` -runs it and needs `python3` with [PyYAML]. The [uv] workspace provides both: +check in `.asf.yaml` matches a job in `.github/workflows`, and that `rust.yml` +skips only its listed jobs on pushes to `main`. `./dev/rust_lint.sh` runs it +and needs `python3` with [PyYAML]. The [uv] workspace provides both: ```shell uv run ./dev/rust_lint.sh