ci: de-duplicate invariant gates across the test matrix #372
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -84,12 +84,6 @@ jobs: | |
| exit 1 | ||
| fi | ||
|
|
||
| - name: Validate changelog | ||
| run: python scripts/validate_changelog.py | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,55 +1,129 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Composable validation gates. The default command remains the authoritative | ||
| # local aggregate; CI selects only the gate groups appropriate to each job. | ||
| set -euo pipefail | ||
|
|
||
| gate="all" | ||
| node_contracts_missing=0 | ||
| if [[ $# -gt 0 ]]; then | ||
| if [[ $# -ne 2 || "$1" != "--gate" ]]; then | ||
| printf 'Usage: %s [--gate baseline|runtime|coverage|typing|style|contracts|benchmark|security]\n' "$0" >&2 | ||
| exit 2 | ||
| fi | ||
| gate="$2" | ||
| fi | ||
|
|
||
| require_commands() { | ||
| local command | ||
| for command in "$@"; do | ||
| command -v "$command" >/dev/null 2>&1 || { | ||
| printf 'Missing validation tool: %s. Install the required development extras first.\n' "$command" >&2 | ||
| exit 1 | ||
| } | ||
| done | ||
| } | ||
|
|
||
| run_baseline() { | ||
| bash ./tests/validate.sh | ||
| } | ||
|
|
||
| run_runtime() { | ||
| require_commands python | ||
| python -m pytest | ||
| } | ||
|
|
||
| run_coverage() { | ||
| require_commands python | ||
| python -m pytest --cov=base_cli --cov-report=term-missing --cov-report=json:coverage.json --cov-fail-under=80 | ||
| python scripts/validate_coverage.py coverage.json | ||
| } | ||
|
|
||
| run_typing() { | ||
| require_commands python mypy | ||
| python -m mypy --strict examples/typed_consumer.py | ||
| python -m mypy --strict lib/python/base_cli | ||
| } | ||
|
|
||
| run_style() { | ||
| require_commands ruff | ||
| ruff format --check lib/python/base_cli scripts examples tests | ||
| ruff check lib/python/base_cli scripts examples tests | ||
| } | ||
|
|
||
| run_contracts() { | ||
| require_commands python | ||
| python scripts/validate_docs.py | ||
| python scripts/validate_changelog.py | ||
| python scripts/validate_schemas.py | ||
| python scripts/validate_contract_fixtures.py | ||
| if command -v node >/dev/null 2>&1; then | ||
| node scripts/validate_contract_fixtures.mjs | ||
| elif [[ "$gate" == "all" ]]; then | ||
| node_contracts_missing=1 | ||
| printf 'Node.js is unavailable; the cross-language contract gate is incomplete.\n' >&2 | ||
| else | ||
| printf 'Node.js is unavailable; the cross-language contract gate is incomplete.\n' >&2 | ||
| return 2 | ||
| fi | ||
| python scripts/generate_compatibility_dashboard.py --check | ||
| python -m compileall -q examples | ||
| } | ||
|
|
||
| run_benchmark() { | ||
| require_commands python | ||
| python scripts/benchmark_runtime.py --check | ||
| } | ||
|
|
||
| run_security() { | ||
| require_commands bandit pip-audit python | ||
| bandit -q -r lib/python/base_cli scripts -lll -iii | ||
|
|
||
| # Audit third-party packages without asking pip-audit to resolve the | ||
| # unpublished editable checkout itself. | ||
| local audit_requirements | ||
| audit_requirements="$(mktemp)" | ||
| trap 'rm -f "$audit_requirements"' RETURN | ||
|
Contributor
Author
There was a problem hiding this comment. Correctness: cleanup trap no longer fires on the failure path it exists for
Confirmed empirically (bash 5.3): → the trap never fires and the temp file is leaked; the equivalent test with Failure scenario: |
||
| python -m pip freeze \ | ||
| | sed -E '/(^-e .*#egg=base[_-]cli|^base[_-]cli([[:space:]=@]|$))/Id' \ | ||
| > "$audit_requirements" | ||
| pip-audit --strict -r "$audit_requirements" | ||
| } | ||
|
|
||
| write_validation_result() { | ||
| local validation_result | ||
| validation_result="${BASE_CLI_VALIDATION_RESULT:-${TMPDIR:-/tmp}/base-cli-validation-result.json}" | ||
| if ((node_contracts_missing)); then | ||
| printf '%s\n' '{"status":"partial","skipped":["node contract validator"]}' > "$validation_result" | ||
| printf 'Validation result: partial; Node.js contract validation was skipped (%s).\n' "$validation_result" | ||
| return 2 | ||
|
Contributor
Author
There was a problem hiding this comment. Removed behavior: dropped user-facing guidance on the partial-result path The pre-refactor script's node-missing branch printed three lines, the last of which was:
|
||
| fi | ||
| printf '%s\n' '{"status":"full","skipped":[]}' > "$validation_result" | ||
| printf 'Validation result: full (%s)\n' "$validation_result" | ||
| } | ||
|
|
||
| case "$gate" in | ||
| baseline) run_baseline ;; | ||
| runtime) run_runtime ;; | ||
| coverage) run_coverage ;; | ||
| typing) run_typing ;; | ||
| style) run_style ;; | ||
| contracts) run_contracts ;; | ||
| benchmark) run_benchmark ;; | ||
| security) run_security ;; | ||
| all) | ||
| run_baseline | ||
| run_coverage | ||
| run_typing | ||
| run_style | ||
| run_contracts | ||
| run_benchmark | ||
| run_security | ||
| write_validation_result | ||
| printf 'Full base-cli validation passed.\n' | ||
| ;; | ||
| *) | ||
| printf 'Unknown validation gate: %s\n' "$gate" >&2 | ||
| exit 2 | ||
| ;; | ||
| esac | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Behavior gap: standalone
--gate contractsdoes less than the same gate run insideallWhen
$gate == "all"and Node is missing,run_contracts()setsnode_contracts_missing=1and falls through to still rungenerate_compatibility_dashboard.py --checkandcompileall(lines 69-70). But when--gate contractsis invoked standalone (e.g. from a future CI step or a developer's shell) and Node is missing, itreturn 2s immediately at line 67, skipping those same two checks entirely.So the 'contracts' gate silently validates a different, smaller set of things depending on whether it's invoked alone or as part of the aggregate — a maintainer relying on
./tests/full_validate.sh --gate contractsfor local iteration on a Node-less machine gets less coverage (no dashboard/compileall check) than they'd get from the full run, with no message indicating those two checks were skipped. Consider running the dashboard/compileall checks before deciding whether toreturn 2, so the standalone gate's coverage doesn't regress based on the node-missing branch taken.