diff --git a/.github/actions/prepare-release-benchmarks/action.yml b/.github/actions/prepare-release-benchmarks/action.yml index 3313def..aff6226 100644 --- a/.github/actions/prepare-release-benchmarks/action.yml +++ b/.github/actions/prepare-release-benchmarks/action.yml @@ -1,9 +1,25 @@ name: Prepare release benchmarks description: Install uncached tools, validate inputs, and inventory full release suites within the caller's shared timeout. +inputs: + validated-attempt: + description: Workflow attempt that checked the release draft + required: true + runs: using: composite steps: + - name: Require fresh release preflight + shell: bash + env: + VALIDATED_ATTEMPT: ${{ inputs.validated-attempt }} + run: | + set -euo pipefail + if [[ "$VALIDATED_ATTEMPT" != "$GITHUB_RUN_ATTEMPT" ]]; then + echo "::error::Rerun all jobs or dispatch again to revalidate the draft before benchmarking" + exit 1 + fi + - name: Install Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 with: diff --git a/.github/workflows/release-benchmarks.yml b/.github/workflows/release-benchmarks.yml index 5eff906..53d4af9 100644 --- a/.github/workflows/release-benchmarks.yml +++ b/.github/workflows/release-benchmarks.yml @@ -1,6 +1,6 @@ name: Release Benchmarks -# Archive full Criterion benchmark baselines for published releases. +# Attach full Criterion baselines to a draft before publishing the release. # Release jobs publish durable artifacts, so they intentionally do not restore # or save dependency caches. @@ -8,14 +8,15 @@ permissions: contents: read on: - release: - types: - - published - # Exercise the full producer on a selected ref without publishing a release. workflow_dispatch: + inputs: + tag: + description: Stable vX.Y.Z tag matching the workflow ref, with a mutable draft release + required: true + type: string concurrency: - group: release-benchmarks-${{ github.event.release.tag_name || github.ref }} + group: release-benchmarks-${{ inputs.tag }} cancel-in-progress: false env: @@ -23,21 +24,71 @@ env: RUST_BACKTRACE: 1 jobs: + validate-release: + # Draft visibility requires push access. This job executes no repository code. + permissions: + contents: write + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + release-id: ${{ steps.target.outputs.release-id }} + commit: ${{ steps.target.outputs.commit }} + validated-attempt: ${{ github.run_attempt }} + steps: + - name: Validate draft release target + id: target + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RELEASE_TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + + if [[ ! "$RELEASE_TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "::error::Expected an existing stable vX.Y.Z tag" + exit 1 + fi + if [[ "$GITHUB_REF" != "refs/tags/$RELEASE_TAG" ]]; then + echo "::error::Dispatch with --ref $RELEASE_TAG and the matching tag input" + exit 1 + fi + # The release-by-tag endpoint only guarantees published releases. + releases="$(gh api "repos/$GH_REPO/releases?per_page=100" --paginate --slurp)" + release_id="$(jq -er --arg tag "$RELEASE_TAG" ' + [.[][] | select(.tag_name == $tag)] + | if length == 1 and .[0].draft == true and .[0].prerelease == false + and .[0].immutable == false and .[0].name == $tag + then .[0].id else error("Expected exactly one mutable stable draft with the tag as its title") end + ' <<< "$releases")" + [[ "$release_id" =~ ^[1-9][0-9]*$ ]] || exit 1 + # Fully qualify the tag to avoid a same-named branch; peel annotated tags. + commit="$(gh api "repos/$GH_REPO/commits/refs/tags/$RELEASE_TAG" --jq .sha)" + [[ "$commit" =~ ^[0-9a-f]{40}$ ]] || exit 1 + if [[ "$commit" != "$GITHUB_SHA" ]]; then + echo "::error::Release tag no longer matches the workflow commit" + exit 1 + fi + echo "release-id=$release_id" >> "$GITHUB_OUTPUT" + echo "commit=$commit" >> "$GITHUB_OUTPUT" + release-baseline: + needs: validate-release runs-on: ubuntu-latest # 2 min checkout + 28 min shared setup + 150 min comparative + 90 min exact # + 15 min for the tail and runner overhead. timeout-minutes: 285 env: - RELEASE_TAG: ${{ github.event.release.tag_name || format('validation-{0}-{1}', github.run_id, github.run_attempt) }} + RELEASE_TAG: ${{ inputs.tag }} CRITERION_HOME: ${{ github.workspace }}/target/criterion outputs: release-asset: ${{ steps.package-baseline.outputs.asset }} + artifact-name: ${{ steps.package-baseline.outputs.artifact-name }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.release.tag_name || github.sha }} + # Use the workflow's own commit and tag cache scope, never a separate ref. + ref: ${{ github.sha }} persist-credentials: false timeout-minutes: 2 @@ -45,6 +96,8 @@ jobs: # One parent timeout bounds all nested installs, validation, and inventory. timeout-minutes: 28 uses: ./.github/actions/prepare-release-benchmarks # zizmor: ignore[self-repository] actionlint 1.7.12 does not accept $/... + with: + validated-attempt: ${{ needs.validate-release.outputs.validated-attempt }} - name: Save comparative Criterion baseline id: comparative @@ -78,11 +131,12 @@ jobs: cp target/release-benchmark-inventory.json target/criterion/release-benchmark-inventory.json tar -C target -czf "$asset" criterion echo "asset=$asset" >> "$GITHUB_OUTPUT" + echo "artifact-name=bench-baseline-${RELEASE_TAG}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" - name: Upload temporary baseline artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: bench-baseline-${{ env.RELEASE_TAG }} + name: ${{ steps.package-baseline.outputs.artifact-name }} path: ${{ steps.package-baseline.outputs.asset }} retention-days: 30 if-no-files-found: error @@ -121,8 +175,7 @@ jobs: } | tee -a "$GITHUB_STEP_SUMMARY" publish-baseline: - if: ${{ github.event_name == 'release' }} - needs: release-baseline + needs: [validate-release, release-baseline] permissions: contents: write runs-on: ubuntu-latest @@ -132,18 +185,69 @@ jobs: - name: Download release baseline uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: bench-baseline-${{ github.event.release.tag_name }} + name: ${{ needs.release-baseline.outputs.artifact-name }} - - name: Attach baseline to GitHub Release + - name: Attach baseline and publish draft env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} - RELEASE_TAG: ${{ github.event.release.tag_name }} + RELEASE_TAG: ${{ inputs.tag }} + RELEASE_ID: ${{ needs.validate-release.outputs.release-id }} + RELEASE_COMMIT: ${{ needs.validate-release.outputs.commit }} RELEASE_ASSET: ${{ needs.release-baseline.outputs.release-asset }} run: | set -euo pipefail - gh release upload "$RELEASE_TAG" "$RELEASE_ASSET" --clobber + [[ "$RELEASE_TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || exit 1 + [[ "$RELEASE_ID" =~ ^[1-9][0-9]*$ ]] || exit 1 + [[ "$RELEASE_COMMIT" =~ ^[0-9a-f]{40}$ ]] || exit 1 + [[ "$RELEASE_ASSET" == "la-stack-${RELEASE_TAG}-criterion-baseline.tar.gz" ]] || exit 1 + [[ -f "$RELEASE_ASSET" && ! -L "$RELEASE_ASSET" && -s "$RELEASE_ASSET" ]] || exit 1 + digest="$(sha256sum "$RELEASE_ASSET")" + digest="sha256:${digest%% *}" + size="$(wc -c < "$RELEASE_ASSET")" + release_path="repos/$GH_REPO/releases/$RELEASE_ID" + + check_draft() { + gh api "$release_path" | jq -e --arg tag "$RELEASE_TAG" --argjson id "$RELEASE_ID" ' + .id == $id and .tag_name == $tag and .name == $tag + and .draft == true and .prerelease == false and .immutable == false + ' > /dev/null + local current_commit + current_commit="$(gh api "repos/$GH_REPO/commits/refs/tags/$RELEASE_TAG" --jq .sha)" + [[ "$current_commit" == "$RELEASE_COMMIT" ]] + } + matching_assets() { + gh api "$release_path/assets?per_page=100" --paginate --slurp | + jq --arg name "$RELEASE_ASSET" '[.[][] | select(.name == $name)]' + } + verify_asset() { + jq -e --arg digest "$digest" --argjson size "$size" ' + length == 1 and .[0].state == "uploaded" + and .[0].digest == $digest and .[0].size == $size + ' > /dev/null + } + + check_draft + assets="$(matching_assets)" + if [[ "$(jq length <<< "$assets")" == 0 ]]; then + # Address the captured release ID, never a replacement draft by tag. + gh api "https://uploads.github.com/$release_path/assets?name=$RELEASE_ASSET" \ + --method POST --header 'Content-Type: application/gzip' \ + --input "$RELEASE_ASSET" > /dev/null + elif ! verify_asset <<< "$assets"; then + echo "::error::Conflicting draft asset; inspect it before removing it and retrying" + exit 1 + fi + + # An interrupted upload can be reused only when its bytes match exactly. + # Recheck mutable state and tag after upload, before the irreversible step. + check_draft + matching_assets | verify_asset + gh api "$release_path" --method PATCH -F draft=false | + jq -e --arg tag "$RELEASE_TAG" --argjson id "$RELEASE_ID" ' + .id == $id and .tag_name == $tag and .draft == false and .prerelease == false + ' > /dev/null - name: Release baseline summary env: @@ -154,5 +258,5 @@ jobs: { echo "### Release Benchmark Baseline" echo "" - echo "Uploaded release asset: \`$RELEASE_ASSET\`" + echo "Published release with verified asset: \`$RELEASE_ASSET\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index 243c776..4753da0 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -687,6 +687,17 @@ The durable published baseline is the GitHub Release artifact created by correctness gate before timing or packaging the artifact. The committed release comparison is `docs/performance.md`, created by `just performance-release`. +Follow [Releasing](RELEASING.md#5-create-the-draft-github-release): create the +tagged stable release as a draft, dispatch the workflow with +`--ref "$TAG" -f tag="$TAG"`, and let the workflow upload and verify the archive before it +publishes the draft. Dispatch requires a stable `vX.Y.Z` tag and exactly one +mutable draft with that tag as its title. The producer checks out the resolved +tag commit only when it matches the workflow's own commit. The dispatch ref +must be that same tag, which keeps execution in the tag's cache scope; the +publisher rechecks the commit and captured release ID. +Missing releases, prereleases, and published releases are rejected before +benchmarking. Publication makes the attached evidence immutable. + ### Hosted Release Runtime Budget The producer runs full `vs_linalg` and `exact` suites sequentially on one @@ -734,30 +745,39 @@ Each named baseline's four raw JSON files must match its `new` measurement. Missing diagnostics, failed Criterion writes, stale baselines, and malformed measurements all stop publication. Only successful validation permits packaging the single `criterion/` archive, including the inventory manifest, and uploading -the temporary Actions artifact. The release-only publisher attaches that archive -as `la-stack-$TAG-criterion-baseline.tar.gz`. +the temporary Actions artifact. The separate publisher attaches that archive +as `la-stack-$TAG-criterion-baseline.tar.gz` to the draft, verifies its uploaded +state, size, and SHA-256 digest, and only then publishes the release. ### Validate The Release Workflow -After pushing a branch containing the workflow change, dispatch the producer -against that ref with the GitHub CLI: +Run the applicable local gates for workflow changes: ```bash -gh workflow run release-benchmarks.yml --ref -gh run list --workflow release-benchmarks.yml --event workflow_dispatch -gh run watch --exit-status -gh run download --name bench-baseline-validation--1 +just lint-config +just python-ci +just markdown-ci +just doc-check ``` -This existing workflow is already registered by its release runs, so the CLI -can select a branch containing the manual trigger. The Actions page also offers -manual dispatch once the trigger is available on the default branch; see -[GitHub's dispatch documentation](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_dispatch). - -A manual run uses the selected commit and a `validation--` -baseline name. It performs full input validation, measurement, dataset checks, -packaging, and the 30-day temporary upload; its publisher is skipped. For a -rerun, substitute the actual attempt number in the artifact name. Record the -successful run URL and both elapsed suite times when validating a budget change. -The estimates above still require this representative hosted run; local tests -and archive fixtures do not establish GitHub-runner runtime or upload success. +The Python suite executes the workflow's shell with simulated GitHub API +responses to test draft rejection, commit changes, upload failures, asset +verification, and safe reruns. Archive fixtures check the complete dataset. + +Every hosted dispatch now requires a real release draft and authorizes its +publication; there is no producer-only manual mode. Use the +[release sequence](RELEASING.md#6-run-benchmarks-and-publish-the-draft) for a +planned release. Record the successful run URL and both elapsed suite times +when validating a budget change. The estimates above still require this +representative hosted run; local tests and archive fixtures do not establish +GitHub-runner runtime or upload success. + +The temporary artifact is named +`bench-baseline-$TAG--` and retained for 30 days. +Retry a failed producer by rerunning all jobs or dispatching again, so the +draft is checked in the same attempt before setup or measurement starts. +Rerunning only failed publisher jobs reuses the successful producer's artifact. +An existing draft asset is reused only when its bytes match; conflicting assets +require inspection and manual removal while the release is still a draft. +Published releases are always rejected, and `--clobber` is never used. See +[failed-run recovery](RELEASING.md#recovering-a-failed-run) before retrying. diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 5e2ccdd..ff611d3 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -1,7 +1,13 @@ # Releasing la-stack Prepare each `vX.Y.Z` release in a dedicated PR. After that PR is merged, -create the annotated tag, publish to crates.io, and create the GitHub release. +create the annotated tag, publish to crates.io, and create a draft GitHub release. +You then start the `Release Benchmarks` workflow, which attaches the durable +baseline and **automatically publishes that same draft as the public release**. + +The local benchmarks used to prepare the release PR do not require a draft. +The draft is needed later as the upload destination for the benchmark archive +produced on GitHub Actions. The changelog is generated for the target tag before the tag exists, so the release process does not require a temporary local tag. @@ -215,22 +221,73 @@ git push origin "$TAG" cargo publish --locked ``` -### 5. Create the GitHub release +### 5. Create the draft GitHub release + +Create an unpublished release where the workflow can attach its benchmark archive: ```bash -gh release create "$TAG" --title "$TAG" --notes-from-tag +gh release create "$TAG" --verify-tag --draft --title "$TAG" --notes-from-tag ``` Keep the release title identical to the tag, including its leading `v`. +Leave it as a stable draft: publishing now would make the release immutable +before the benchmark archive can be attached. GitHub recommends this +[draft-first sequence for immutable releases](https://docs.github.com/en/code-security/concepts/supply-chain-security/immutable-releases). + +### 6. Run benchmarks and publish the draft + +**Starting this workflow also authorizes automatic publication.** You do not +need a separate command to convert the draft into a published release. + +Start the workflow from the release tag, passing the same tag as its input: + +```bash +gh workflow run release-benchmarks.yml --ref "$TAG" -f tag="$TAG" +``` + +Once GitHub accepts the command, you can close the terminal and leave the +workflow running. It runs on GitHub's runners and needs no further input to +publish the release. Check its status later, when convenient, on the +[Release Benchmarks Actions page](https://github.com/acgetchell/la-stack/actions/workflows/release-benchmarks.yml). + +The workflow runs from the tagged version of its definition, so the tag must +include this release workflow. Using the tag as the workflow ref keeps execution +in that tag's cache scope. Dispatching from `main` or a different tag is rejected. + +The workflow automatically: -### 6. Verify the durable Criterion baseline +1. Runs the full benchmark suites for the tagged commit. +2. Uploads the archive to the draft and verifies its name, state, size, and + SHA-256 digest. +3. Removes the draft status, making the same release public with the archive + already attached. -After the `Release Benchmarks` workflow completes, verify that the release -contains its long-lived baseline archive: +A benchmark, upload, or verification failure leaves the release as a draft. +Follow [Recovering a failed run](#recovering-a-failed-run) to retry. + +Before benchmarking, the workflow requires +exactly one mutable, non-prerelease draft whose tag and title match the stable +`vX.Y.Z` input, and resolves the existing tag to a commit. It benchmarks that +commit only if it matches the workflow's own commit, then rechecks the captured +release ID and tag commit before attaching +the archive and again before publication. Do not edit the draft, move the tag, +or publish manually while the workflow is running. + +Runs for the same tag are serialized without cancelling an active run. + +### 7. Verify publication and the durable Criterion baseline + +After the workflow completes, open the repository's +[Releases page](https://github.com/acgetchell/la-stack/releases). Confirm that +`$TAG` is published and its assets include +`la-stack-$TAG-criterion-baseline.tar.gz`. This is a check of the completed +publication; the workflow does not wait for you to perform it. + +For an optional command-line check: ```bash -gh release view "$TAG" --json assets \ - --jq ".assets[] | select(.name == \"la-stack-$TAG-criterion-baseline.tar.gz\") | .name" | cat +gh release view "$TAG" --json isDraft,assets \ + --jq "select(.isDraft == false) | .assets[] | select(.name == \"la-stack-$TAG-criterion-baseline.tar.gz\") | .name" | cat ``` The command must print `la-stack-$TAG-criterion-baseline.tar.gz`. A short-lived @@ -239,19 +296,48 @@ Actions artifact is not a substitute for this release asset. The benchmark producer has read-only repository permissions and restores no dependency caches, including tool binaries. It disables Rust toolchain and `setup-just` caching and installs the pinned just and cargo-nextest versions -with `cargo install --locked`. Only the separate publisher job receives -`contents: write` to attach the packaged baseline to the release. +with `cargo install --locked`. The short preflight job receives `contents: write` +for draft visibility; the separate publisher receives it to attach the archive +and publish. Neither privileged job checks out or executes repository code. The producer budgets 150 minutes for `vs_linalg` and 90 minutes for `exact` within a 285-minute job. Both suites retain full release sampling. Inventory and raw-data validation must succeed before the single complete archive is packaged and uploaded; inspect the suite timing summary when a run fails. See the [hosted runtime budget](BENCHMARKING.md#hosted-release-runtime-budget) -for measured history, capacity estimates, and headroom. A manual -[workflow validation run](BENCHMARKING.md#validate-the-release-workflow) -exercises packaging and temporary upload without attaching a release asset. +for measured history, capacity estimates, and headroom, and +[workflow validation](BENCHMARKING.md#validate-the-release-workflow) for local +regression checks and hosted evidence requirements. + +### Recovering a failed run + +If preflight or the producer failed, rerun all jobs with `gh run rerun ` +or dispatch again. The producer requires a preflight from the current attempt; +rerunning failed benchmark jobs alone stops before tool setup or measurement +because the earlier draft check is stale. + +If the producer succeeded and only the publisher failed, preserve the existing +30-day Actions artifact and rerun failed jobs: + +```bash +gh run rerun --failed +``` -### 7. Remove the merged release branch +The publisher reuses an already uploaded draft asset only if its state, size, +and digest match the downloaded archive. It never overwrites or deletes assets. +A conflicting or incomplete upload stops the run; inspect the draft and remove +only that conflicting asset manually before retrying. An expired temporary +artifact requires a new full run. + +A full rerun or new dispatch creates a distinct Actions artifact named +`bench-baseline-$TAG--`. Fresh measurements may differ +from an earlier uploaded draft asset, so resolve any conflict before retrying +publication. If the release is already published, every rerun fails closed +without changing it, including when an earlier publication succeeded but its +response was lost. Verify the public release in that case. Existing immutable +releases with missing assets cannot be repaired by this workflow. + +### 8. Remove the merged release branch After publication and baseline verification succeed: diff --git a/scripts/tests/test_release_baseline.py b/scripts/tests/test_release_baseline.py index b840153..f7e21a4 100644 --- a/scripts/tests/test_release_baseline.py +++ b/scripts/tests/test_release_baseline.py @@ -1,5 +1,6 @@ """Exercise complete release archives and the workflow's publication barriers.""" +import hashlib import json import os import re @@ -43,13 +44,14 @@ def dataset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path, Path return criterion, manifest -def shell_step(name: str) -> str: +def shell_step(name: str, source: Path = WORKFLOW) -> str: """Extract a literal run block so tests exercise the workflow's actual shell.""" - text = WORKFLOW.read_text(encoding="utf-8") - step = text.split(f" - name: {name}\n", 1)[1].split("\n - ", 1)[0] + text = source.read_text(encoding="utf-8") + indent = " " if source == PREPARATION else " " + step = text.split(f"{indent}- name: {name}\n", 1)[1].split(f"\n{indent}- ", 1)[0] lines: list[str] = [] - for line in step.split(" run: |\n", 1)[1].splitlines(): - if line.strip() and not line.startswith(" "): + for line in step.split(f"{indent} run: |\n", 1)[1].splitlines(): + if line.strip() and not line.startswith(f"{indent} "): break lines.append(line) return textwrap.dedent("\n".join(lines)).rstrip() @@ -61,7 +63,7 @@ def run_shell(script: str, cwd: Path, extra_env: dict[str, str]) -> subprocess.C return subprocess.run( # noqa: S603 - run only checked-in workflow shell in an isolated fixture. [bash, "--noprofile", "--norc", "-c", script], cwd=cwd, - env={**os.environ, **extra_env}, + env={**{key: value for key, value in os.environ.items() if key not in {"BASH_ENV", "SHELLOPTS", "BASHOPTS"}}, **extra_env}, capture_output=True, encoding="utf-8", check=False, @@ -74,10 +76,14 @@ def test_complete_dataset_round_trips_through_workflow_archive(dataset: tuple[Pa assert release_baseline.validate(criterion, manifest, BASELINE) == 3 output = tmp_path / "step outputs" # Git Bash accepts forward-slash drive paths on Windows as well as POSIX paths. - result = run_shell(shell_step("Package release Criterion baseline"), tmp_path, {"RELEASE_TAG": BASELINE, "GITHUB_OUTPUT": output.as_posix()}) + result = run_shell( + shell_step("Package release Criterion baseline"), + tmp_path, + {"RELEASE_TAG": BASELINE, "GITHUB_OUTPUT": output.as_posix(), "GITHUB_RUN_ID": "123", "GITHUB_RUN_ATTEMPT": "2"}, + ) assert result.returncode == 0, result.stderr asset = f"la-stack-{BASELINE}-criterion-baseline.tar.gz" - assert output.read_text() == f"asset={asset}\n" + assert output.read_text(encoding="utf-8") == f"asset={asset}\nartifact-name=bench-baseline-{BASELINE}-123-2\n" with tarfile.open(tmp_path / asset) as archive: for ids in IDS.values(): for benchmark in ids: @@ -138,7 +144,7 @@ def test_invalid_measurement_blocks_publication(dataset: tuple[Path, Path], corr times = [2.0] * 99 if corruption == "short-samples" else [float("inf")] * 100 write_json(directory / "sample.json", {"iters": [1.0] * 100, "times": times}) else: - estimates = json.loads((directory / "estimates.json").read_text()) + estimates = json.loads((directory / "estimates.json").read_text(encoding="utf-8")) interval = estimates["median"]["confidence_interval"] interval["lower_bound" if corruption == "reversed-interval" else "confidence_level"] = 4.0 write_json(directory / "estimates.json", estimates) @@ -207,13 +213,14 @@ def test_discovery_refuses_stale_measurements(dataset: tuple[Path, Path], tmp_pa assert manifest.read_bytes() == before -def test_workflow_fails_closed_and_manual_runs_cannot_publish() -> None: +def test_workflow_requires_preflight_and_isolates_publication_permissions() -> None: workflow = WORKFLOW.read_text(encoding="utf-8") preparation = PREPARATION.read_text(encoding="utf-8") assert preparation.index("- name: Validate benchmark inputs") < preparation.index("- name: Inventory full release suites") assert "run: just test-bench-inputs" in preparation assert "run: just bench-release-inventory" in preparation assert "continue-on-error" not in preparation + assert preparation.index("- name: Require fresh release preflight") < preparation.index("- name: Install Rust toolchain") steps = [ "Prepare release benchmarks", "Save comparative Criterion baseline", @@ -227,11 +234,27 @@ def test_workflow_fails_closed_and_manual_runs_cannot_publish() -> None: assert "\n if:" not in workflow[positions[0] : positions[-1]] assert "continue-on-error" not in workflow assert "workflow_dispatch:" in workflow - publisher = workflow.split(" publish-baseline:\n", 1)[1] - assert "if: ${{ github.event_name == 'release' }}" in publisher - assert "needs: release-baseline" in publisher + assert "required: true" in workflow + assert "\n release:" not in workflow + assert "group: release-benchmarks-${{ inputs.tag }}" in workflow + assert "cancel-in-progress: false" in workflow + preflight, remainder = workflow.split(" release-baseline:\n", 1) + producer, publisher = remainder.split(" publish-baseline:\n", 1) + assert "needs: validate-release" in producer + assert "validated-attempt: ${{ github.run_attempt }}" in preflight + assert "validated-attempt: ${{ needs.validate-release.outputs.validated-attempt }}" in producer + assert "ref: ${{ github.sha }}" in producer + assert "ref: ${{ needs.validate-release.outputs.commit }}" not in producer + assert "contents: write" not in producer + assert "persist-credentials: false" in producer + assert "contents: write" in preflight + assert "actions/checkout@" not in preflight + publisher + assert "uses: ./" not in preflight + publisher + assert "--clobber" not in workflow + assert "needs: [validate-release, release-baseline]" in publisher assert "GH_REPO: ${{ github.repository }}" in publisher - assert "contents: write" not in workflow.split(" publish-baseline:\n", 1)[0] + assert "contents: write" in publisher + assert "name: ${{ needs.release-baseline.outputs.artifact-name }}" in publisher def test_setup_limits_preserve_benchmark_and_tail_budgets() -> None: @@ -279,5 +302,250 @@ def test_failed_suite_retains_timing_and_does_not_mark_completion(tmp_path: Path }, ) assert result.returncode == 0, result.stderr - assert "| vs_linalg | failure |" in summary.read_text() - assert "| exact | skipped |" in summary.read_text() + assert "| vs_linalg | failure |" in summary.read_text(encoding="utf-8") + assert "| exact | skipped |" in summary.read_text(encoding="utf-8") + + +# Only the GitHub boundary is substituted. jq predicates, checksums, shell +# failure handling, and publication ordering execute from the workflow itself. +GITHUB_STUB = r""" +gh() { + printf '%s\n' "$*" >> api-calls + [[ "$1" == api ]] || return 90 + case "$2" in + 'repos/owner/repo/releases?per_page=100') + [[ "$*" == *'--paginate --slurp' ]] || return 91 + cat releases.json + ;; + "repos/owner/repo/commits/refs/tags/$RELEASE_TAG") + [[ "${COMMIT_STATUS:-0}" == 0 ]] || return "$COMMIT_STATUS" + if [[ -f commit-read ]]; then + printf '%s\n' "${LATER_COMMIT:-$RELEASE_COMMIT}" + else + touch commit-read + printf '%s\n' "$RELEASE_COMMIT" + fi + ;; + repos/owner/repo/releases/42) + if [[ "$*" == *'--method PATCH -F draft=false' ]]; then + touch publish-called + [[ "${PUBLISH_STATUS:-0}" == 0 ]] || return "$PUBLISH_STATUS" + cat published.json + elif [[ -f draft-read ]]; then + cat later-draft.json + else + touch draft-read + cat draft.json + fi + ;; + 'repos/owner/repo/releases/42/assets?per_page=100') + [[ "$*" == *'--paginate --slurp' ]] || return 92 + if [[ -f upload-called ]]; then + cat uploaded-assets.json + else + cat assets.json + fi + ;; + "https://uploads.github.com/repos/owner/repo/releases/42/assets?name=$RELEASE_ASSET") + [[ "$*" == *'--method POST --header Content-Type: application/gzip --input '* ]] || return 93 + touch upload-called + return "${UPLOAD_STATUS:-0}" + ;; + *) return 94 ;; + esac +} +""" + + +def draft_release(**changes: object) -> dict[str, object]: + return {"id": 42, "tag_name": BASELINE, "name": BASELINE, "draft": True, "prerelease": False, "immutable": False, **changes} + + +@pytest.fixture +def github_release(tmp_path: Path) -> dict[str, str]: + asset = f"la-stack-{BASELINE}-criterion-baseline.tar.gz" + payload = b"test archive bytes\n" + (tmp_path / asset).write_bytes(payload) + metadata = {"name": asset, "state": "uploaded", "size": len(payload), "digest": f"sha256:{hashlib.sha256(payload).hexdigest()}"} + write_json(tmp_path / "releases.json", [[draft_release(tag_name="v0.1.0")], [draft_release()]]) + write_json(tmp_path / "draft.json", draft_release()) + write_json(tmp_path / "later-draft.json", draft_release()) + write_json(tmp_path / "published.json", draft_release(draft=False, immutable=True)) + write_json(tmp_path / "assets.json", [[]]) + write_json(tmp_path / "uploaded-assets.json", [[{"name": "unrelated.txt"}], [metadata]]) + return { + "GH_REPO": "owner/repo", + "RELEASE_TAG": BASELINE, + "RELEASE_ID": "42", + "RELEASE_COMMIT": "a" * 40, + "RELEASE_ASSET": asset, + "GITHUB_REF": f"refs/tags/{BASELINE}", + "GITHUB_SHA": "a" * 40, + "GITHUB_OUTPUT": (tmp_path / "outputs").as_posix(), + } + + +def test_preflight_captures_draft_identity_and_peeled_tag_commit(tmp_path: Path, github_release: dict[str, str]) -> None: + result = run_shell(GITHUB_STUB + shell_step("Validate draft release target"), tmp_path, github_release) + assert result.returncode == 0, result.stderr + assert (tmp_path / "outputs").read_text(encoding="utf-8") == f"release-id=42\ncommit={'a' * 40}\n" + calls = (tmp_path / "api-calls").read_text(encoding="utf-8") + assert "--paginate --slurp" in calls + assert f"commits/refs/tags/{BASELINE}" in calls + assert "--method" not in calls + + +@pytest.mark.parametrize("ref", ["refs/heads/main", f"refs/heads/{BASELINE}", "refs/tags/v9.9.9", ""]) +def test_preflight_rejects_dispatch_outside_release_tag(tmp_path: Path, github_release: dict[str, str], ref: str) -> None: + result = run_shell(GITHUB_STUB + shell_step("Validate draft release target"), tmp_path, {**github_release, "GITHUB_REF": ref}) + assert result.returncode == 1 + assert f"Dispatch with --ref {BASELINE}" in result.stdout + assert not (tmp_path / "api-calls").exists() + assert not (tmp_path / "outputs").exists() + + +def test_preflight_rejects_tag_moved_since_dispatch(tmp_path: Path, github_release: dict[str, str]) -> None: + result = run_shell(GITHUB_STUB + shell_step("Validate draft release target"), tmp_path, {**github_release, "RELEASE_COMMIT": "b" * 40}) + assert result.returncode == 1 + assert "Release tag no longer matches the workflow commit" in result.stdout + assert not (tmp_path / "outputs").exists() + + +@pytest.mark.parametrize("validated_attempt", ["", "1", "2"]) +def test_benchmark_retry_requires_current_preflight(tmp_path: Path, validated_attempt: str) -> None: + result = run_shell( + shell_step("Require fresh release preflight", PREPARATION), + tmp_path, + {"VALIDATED_ATTEMPT": validated_attempt, "GITHUB_RUN_ATTEMPT": "2"}, + ) + assert result.returncode == (0 if validated_attempt == "2" else 1) + if validated_attempt != "2": + assert "Rerun all jobs or dispatch again" in result.stdout + + +@pytest.mark.parametrize("tag", ["", "1.2.3", "v01.2.3", "v1.02.3", "v1.2.03", "v1.2.3-rc.1", "v1.2.3+meta", "main", "v1.2.3\nextra", "$(touch injected)"]) +def test_preflight_rejects_invalid_tag_before_api_access(tmp_path: Path, github_release: dict[str, str], tag: str) -> None: + result = run_shell(GITHUB_STUB + shell_step("Validate draft release target"), tmp_path, {**github_release, "RELEASE_TAG": tag}) + assert result.returncode == 1 + assert "Expected an existing stable vX.Y.Z tag" in result.stdout + assert not (tmp_path / "api-calls").exists() + assert not (tmp_path / "outputs").exists() + assert not (tmp_path / "injected").exists() + + +@pytest.mark.parametrize( + "releases", + [ + [], + [draft_release(), draft_release(id=43)], + [draft_release(draft=False)], + [draft_release(prerelease=True)], + [draft_release(immutable=True)], + [draft_release(immutable=None)], + [draft_release(tag_name="v9.9.9")], + [draft_release(name="Wrong title")], + [draft_release(id="42\ncommit=injected")], + ], +) +def test_preflight_rejects_unsuitable_release(tmp_path: Path, github_release: dict[str, str], releases: list[dict[str, object]]) -> None: + write_json(tmp_path / "releases.json", [releases]) + result = run_shell(GITHUB_STUB + shell_step("Validate draft release target"), tmp_path, github_release) + assert result.returncode != 0 + assert "commits/" not in (tmp_path / "api-calls").read_text(encoding="utf-8") + assert not (tmp_path / "outputs").exists() + + +@pytest.mark.parametrize("response", ["{invalid", "null"]) +def test_preflight_rejects_malformed_api_response(tmp_path: Path, github_release: dict[str, str], response: str) -> None: + (tmp_path / "releases.json").write_text(response, encoding="utf-8") + result = run_shell(GITHUB_STUB + shell_step("Validate draft release target"), tmp_path, github_release) + assert result.returncode != 0 + assert not (tmp_path / "outputs").exists() + + +@pytest.mark.parametrize("overrides", [{"COMMIT_STATUS": "4"}, {"RELEASE_COMMIT": ""}, {"RELEASE_COMMIT": "a" * 40 + "\ninjected=true"}]) +def test_missing_or_invalid_tag_commit_stops_preflight(tmp_path: Path, github_release: dict[str, str], overrides: dict[str, str]) -> None: + result = run_shell(GITHUB_STUB + shell_step("Validate draft release target"), tmp_path, {**github_release, **overrides}) + assert result.returncode != 0 + assert not (tmp_path / "outputs").exists() + + +@pytest.mark.parametrize("reuse", [False, True], ids=["fresh-upload", "interrupted-upload-rerun"]) +def test_publisher_verifies_durable_asset_before_publication(tmp_path: Path, github_release: dict[str, str], reuse: bool) -> None: + if reuse: + shutil.copyfile(tmp_path / "uploaded-assets.json", tmp_path / "assets.json") + result = run_shell(GITHUB_STUB + shell_step("Attach baseline and publish draft"), tmp_path, github_release) + assert result.returncode == 0, result.stderr + calls = (tmp_path / "api-calls").read_text(encoding="utf-8").splitlines() + assert calls[-1] == "api repos/owner/repo/releases/42 --method PATCH -F draft=false" + assert calls[-2] == "api repos/owner/repo/releases/42/assets?per_page=100 --paginate --slurp" + assert calls.count("api repos/owner/repo/releases/42") == 2 + assert (tmp_path / "upload-called").exists() is not reuse + assert (tmp_path / "publish-called").exists() + assert "--clobber" not in " ".join(calls) + + +@pytest.mark.parametrize("snapshot", ["draft.json", "later-draft.json"]) +@pytest.mark.parametrize( + "changes", + [{"draft": False}, {"prerelease": True}, {"immutable": True}, {"tag_name": "v9.9.9"}, {"name": "renamed"}, {"id": 43}], +) +def test_changed_release_stops_publication(tmp_path: Path, github_release: dict[str, str], snapshot: str, changes: dict[str, object]) -> None: + write_json(tmp_path / snapshot, draft_release(**changes)) + result = run_shell(GITHUB_STUB + shell_step("Attach baseline and publish draft"), tmp_path, github_release) + assert result.returncode != 0 + assert not (tmp_path / "publish-called").exists() + assert (tmp_path / "upload-called").exists() is (snapshot == "later-draft.json") + + +@pytest.mark.parametrize("after_upload", [False, True]) +def test_moved_tag_stops_publication(tmp_path: Path, github_release: dict[str, str], after_upload: bool) -> None: + overrides = {"LATER_COMMIT": "b" * 40} + if not after_upload: + (tmp_path / "commit-read").touch() + result = run_shell(GITHUB_STUB + shell_step("Attach baseline and publish draft"), tmp_path, {**github_release, **overrides}) + assert result.returncode != 0 + assert not (tmp_path / "publish-called").exists() + assert (tmp_path / "upload-called").exists() is after_upload + + +@pytest.mark.parametrize("existing", [False, True]) +@pytest.mark.parametrize("problem", ["missing", "duplicate", "wrong-name", "wrong-digest", "wrong-size", "starter", "no-digest"]) +def test_bad_durable_asset_stops_publication(tmp_path: Path, github_release: dict[str, str], existing: bool, problem: str) -> None: + metadata = json.loads((tmp_path / "uploaded-assets.json").read_text(encoding="utf-8"))[1][0] + assets = [metadata] + if problem == "missing": + assets = [] + elif problem == "duplicate": + assets.append(metadata) + else: + field, value = { + "wrong-name": ("name", "wrong.tar.gz"), + "wrong-digest": ("digest", "sha256:" + "0" * 64), + "wrong-size": ("size", 999), + "starter": ("state", "starter"), + "no-digest": ("digest", None), + }[problem] + metadata[field] = value + write_json(tmp_path / "uploaded-assets.json", [assets]) + if existing: + write_json(tmp_path / "assets.json", [assets]) + result = run_shell(GITHUB_STUB + shell_step("Attach baseline and publish draft"), tmp_path, github_release) + assert result.returncode != 0 + assert not (tmp_path / "publish-called").exists() + assert (tmp_path / "upload-called").exists() is (not existing or problem in {"missing", "wrong-name"}) + + +def test_upload_failure_leaves_draft_unpublished(tmp_path: Path, github_release: dict[str, str]) -> None: + result = run_shell(GITHUB_STUB + shell_step("Attach baseline and publish draft"), tmp_path, {**github_release, "UPLOAD_STATUS": "8"}) + assert result.returncode == 8 + assert (tmp_path / "upload-called").exists() + assert not (tmp_path / "publish-called").exists() + + +def test_publication_failure_propagates_without_deleting_uploaded_asset(tmp_path: Path, github_release: dict[str, str]) -> None: + result = run_shell(GITHUB_STUB + shell_step("Attach baseline and publish draft"), tmp_path, {**github_release, "PUBLISH_STATUS": "9"}) + assert result.returncode != 0 + assert (tmp_path / "upload-called").exists() + assert (tmp_path / "publish-called").exists() + assert "DELETE" not in (tmp_path / "api-calls").read_text(encoding="utf-8")