From 42a3e1fc2f4db0fee13e7ae2a72295ec7286934f Mon Sep 17 00:00:00 2001 From: Adam Getchell Date: Mon, 7 Sep 2026 11:48:00 -0700 Subject: [PATCH 1/2] fix: budget and validate release Criterion baselines - Allow 150 minutes for comparative benchmarks and 90 minutes for exact benchmarks within a 285-minute job, preserving full Criterion sampling. - Require complete, valid raw measurements and matching saved baselines before packaging the archive with its benchmark inventory. - Report suite runtimes and support manual runs without release publication. - Specify the repository explicitly when uploading release assets. - Document runtime estimates, headroom, and pre-release verification. Closes #224 --- .github/workflows/release-benchmarks.yml | 103 +++++++-- docs/BENCHMARKING.md | 73 +++++++ docs/RELEASING.md | 9 + docs/code_organization.md | 5 + justfile | 8 + scripts/README.md | 1 + scripts/release_baseline.py | 186 +++++++++++++++++ scripts/tests/test_release_baseline.py | 253 +++++++++++++++++++++++ 8 files changed, 622 insertions(+), 16 deletions(-) create mode 100644 scripts/release_baseline.py create mode 100644 scripts/tests/test_release_baseline.py diff --git a/.github/workflows/release-benchmarks.yml b/.github/workflows/release-benchmarks.yml index af4847c..d24c1f8 100644 --- a/.github/workflows/release-benchmarks.yml +++ b/.github/workflows/release-benchmarks.yml @@ -11,9 +11,11 @@ on: release: types: - published + # Exercise the full producer on a selected ref without publishing a release. + workflow_dispatch: concurrency: - group: release-benchmarks-${{ github.event.release.tag_name }} + group: release-benchmarks-${{ github.event.release.tag_name || github.ref }} cancel-in-progress: false env: @@ -23,14 +25,18 @@ env: jobs: release-baseline: runs-on: ubuntu-latest - timeout-minutes: 60 + # 30 min setup/discovery + 150 min comparative + 90 min exact + 15 min tail. + timeout-minutes: 285 + env: + RELEASE_TAG: ${{ github.event.release.tag_name || format('validation-{0}-{1}', github.run_id, github.run_attempt) }} + CRITERION_HOME: ${{ github.workspace }}/target/criterion outputs: release-asset: ${{ steps.package-baseline.outputs.asset }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.release.tag_name }} + ref: ${{ github.event.release.tag_name || github.sha }} persist-credentials: false - name: Install Rust toolchain @@ -44,57 +50,121 @@ jobs: with: cache: false - - name: Resolve cargo-nextest version - id: cargo_nextest_version + - name: Resolve tool versions + id: tool_versions shell: bash run: | set -euo pipefail version="$(just --evaluate cargo_nextest_version)" - if [[ -z "$version" ]]; then - echo "::error::Could not resolve cargo_nextest_version from justfile" + uv_version="$(just --evaluate uv_version)" + if [[ -z "$version" || -z "$uv_version" ]]; then + echo "::error::Could not resolve pinned tool versions from justfile" exit 1 fi echo "version=$version" >> "$GITHUB_OUTPUT" + echo "uv_version=$uv_version" >> "$GITHUB_OUTPUT" - name: Install cargo-nextest env: - CARGO_NEXTEST_VERSION: ${{ steps.cargo_nextest_version.outputs.version }} + CARGO_NEXTEST_VERSION: ${{ steps.tool_versions.outputs.version }} run: cargo install --locked cargo-nextest --version "$CARGO_NEXTEST_VERSION" + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version-file: ".python-version" + + - name: Install uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + version: ${{ steps.tool_versions.outputs.uv_version }} + enable-cache: false + - name: Validate benchmark inputs run: just test-bench-inputs - - name: Save release Criterion baseline - env: - RELEASE_TAG: ${{ github.event.release.tag_name }} + - name: Inventory full release suites + timeout-minutes: 20 + run: just bench-release-inventory + + - name: Save comparative Criterion baseline + id: comparative + timeout-minutes: 150 run: | set -euo pipefail + echo "[release-baseline] vs_linalg started $(date -u +%FT%TZ); budget 150 min" + date +%s > target/vs_linalg-started + just bench-save-baseline "$RELEASE_TAG" vs_linalg + date +%s > target/vs_linalg-finished + + - name: Save exact Criterion baseline + id: exact + timeout-minutes: 90 + run: | + set -euo pipefail + echo "[release-baseline] exact started $(date -u +%FT%TZ); budget 90 min" + date +%s > target/exact-started + just bench-save-baseline "$RELEASE_TAG" exact + date +%s > target/exact-finished - cargo bench --locked --features bench --bench vs_linalg -- --save-baseline "$RELEASE_TAG" - cargo bench --locked --features bench,exact --bench exact -- --save-baseline "$RELEASE_TAG" + - name: Validate complete release dataset + run: just bench-release-check "$RELEASE_TAG" - name: Package release Criterion baseline id: package-baseline - env: - RELEASE_TAG: ${{ github.event.release.tag_name }} run: | set -euo pipefail asset="la-stack-${RELEASE_TAG}-criterion-baseline.tar.gz" + cp target/release-benchmark-inventory.json target/criterion/release-benchmark-inventory.json tar -C target -czf "$asset" criterion echo "asset=$asset" >> "$GITHUB_OUTPUT" - name: Upload temporary baseline artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: bench-baseline-${{ github.event.release.tag_name }} + name: bench-baseline-${{ env.RELEASE_TAG }} path: ${{ steps.package-baseline.outputs.asset }} retention-days: 30 if-no-files-found: error + - name: Suite timing summary + if: ${{ always() }} + env: + COMPARATIVE_OUTCOME: ${{ steps.comparative.outcome }} + EXACT_OUTCOME: ${{ steps.exact.outcome }} + run: | + set -euo pipefail + { + echo "### Release benchmark timing" + echo "" + echo "| Suite | Outcome | Elapsed seconds | Budget minutes |" + echo "| --- | --- | --- | --- |" + for suite in vs_linalg exact; do + elapsed="not started" + if [[ -f "target/$suite-started" ]]; then + end="$(date +%s)" + if [[ -f "target/$suite-finished" ]]; then + end="$(cat "target/$suite-finished")" + fi + elapsed="$((end - $(cat "target/$suite-started")))" + fi + outcome="$COMPARATIVE_OUTCOME" + budget=150 + if [[ "$suite" == exact ]]; then + outcome="$EXACT_OUTCOME" + budget=90 + fi + echo "| $suite | $outcome | $elapsed | $budget |" + done + echo "" + echo "An unfinished suite reports elapsed time through this summary." + } | tee -a "$GITHUB_STEP_SUMMARY" + publish-baseline: + if: ${{ github.event_name == 'release' }} needs: release-baseline permissions: contents: write @@ -110,6 +180,7 @@ jobs: - name: Attach baseline to GitHub Release env: GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} RELEASE_TAG: ${{ github.event.release.tag_name }} RELEASE_ASSET: ${{ needs.release-baseline.outputs.release-asset }} run: | diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index aed21d8..396003a 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -19,6 +19,8 @@ the commands measure and where their outputs go. - [`vs_linalg` Methodology](#vs_linalg-methodology) - [Exact-Arithmetic Notes](#exact-arithmetic-notes) - [Release Notes](#release-notes) + - [Hosted Release Runtime Budget](#hosted-release-runtime-budget) + - [Validate The Release Workflow](#validate-the-release-workflow) ## Start Here @@ -684,3 +686,74 @@ The durable published baseline is the GitHub Release artifact created by `.github/workflows/release-benchmarks.yml`. That workflow runs the benchmark-input correctness gate before timing or packaging the artifact. The committed release comparison is `docs/performance.md`, created by `just performance-release`. + +### Hosted Release Runtime Budget + +The producer runs full `vs_linalg` and `exact` suites sequentially on one +`ubuntu-latest` runner. Separate steps allow 150 and 90 minutes respectively; +the outer job allows 285 minutes, reserving 30 minutes for cold setup, input +validation and inventory, plus 15 minutes for validation, packaging, upload, +and diagnostics. Discovery compiles both suites before measurement and has +its own 20-minute timeout. Dependency caches remain disabled. + +The [v0.4.5 run](https://github.com/acgetchell/la-stack/actions/runs/32444040827) +measured 304 comparative benchmarks in about 55m 34s, plus 2m 9s compilation +(57m 43s total). Its exact suite was cancelled during `exact_d2/det_exact`; +that run provides no complete exact-suite runtime. The old 60-minute outer +limit therefore could not accommodate even that smaller harness. + +Inventory on 2026-09-07 found the following current workloads. The planning +estimate uses 12 seconds per benchmark, allowing analysis/report overhead +above the configured 3-second warmup and 5-second measurement target. This is +a capacity estimate, not a measured runtime or a guaranteed upper bound; +Criterion can extend measurement for expensive iterations. + +| Suite | Benchmarks | Nominal warmup + measurement | Planning estimate | Step limit | Headroom above estimate | +| --- | --- | --- | --- | --- | --- | +| `vs_linalg` | 533 | 71.1 min | 106.6 min | 150 min | 43.4 min (41%) | +| `exact` | 264 | 35.2 min | 52.8 min | 90 min | 37.2 min (70%) | + +Both commands retain Criterion's release defaults: 100 samples, 100,000 +resamples, 95% confidence, 3-second warmup, and 5-second measurement target. +There are no quick-mode flags, sampling reductions, or benchmark filters. +Diagnostic families and all peer rows remain included. The workflow logs +the current inventory counts and planning estimates before timing; its final +summary records each suite's outcome, elapsed seconds, and budget even after +a step failure. A runner-level termination can still prevent that summary. + +`just bench-release-inventory` requires a fresh Criterion directory and obtains +the complete IDs from each compiled binary's `--list` output. It also checks +that the inventory includes the report registry and every canonical README +peer row. `just bench-release-check ` then requires every discovered ID, +valid mean/median estimates with 95% intervals, and 100 finite positive samples. +Each named baseline's four raw JSON files must match its `new` measurement. +Missing diagnostics, failed Criterion writes, stale baselines, and malformed +measurements all stop publication. Only successful validation permits packaging +the single `criterion/` archive, including the inventory manifest, and uploading +the temporary Actions artifact. The release-only publisher attaches that archive +as `la-stack-$TAG-criterion-baseline.tar.gz`. + +### Validate The Release Workflow + +After pushing a branch containing the workflow change, dispatch the producer +against that ref with the GitHub CLI: + +```bash +gh workflow run release-benchmarks.yml --ref +gh run list --workflow release-benchmarks.yml --event workflow_dispatch +gh run watch --exit-status +gh run download --name bench-baseline-validation--1 +``` + +This existing workflow is already registered by its release runs, so the CLI +can select a branch containing the manual trigger. The Actions page also offers +manual dispatch once the trigger is available on the default branch; see +[GitHub's dispatch documentation](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_dispatch). + +A manual run uses the selected commit and a `validation--` +baseline name. It performs full input validation, measurement, dataset checks, +packaging, and the 30-day temporary upload; its publisher is skipped. For a +rerun, substitute the actual attempt number in the artifact name. Record the +successful run URL and both elapsed suite times when validating a budget change. +The estimates above still require this representative hosted run; local tests +and archive fixtures do not establish GitHub-runner runtime or upload success. diff --git a/docs/RELEASING.md b/docs/RELEASING.md index d0d22ab..5e2ccdd 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -242,6 +242,15 @@ dependency caches, including tool binaries. It disables Rust toolchain and with `cargo install --locked`. Only the separate publisher job receives `contents: write` to attach the packaged baseline to the release. +The producer budgets 150 minutes for `vs_linalg` and 90 minutes for `exact` +within a 285-minute job. Both suites retain full release sampling. Inventory +and raw-data validation must succeed before the single complete archive is +packaged and uploaded; inspect the suite timing summary when a run fails. +See the [hosted runtime budget](BENCHMARKING.md#hosted-release-runtime-budget) +for measured history, capacity estimates, and headroom. A manual +[workflow validation run](BENCHMARKING.md#validate-the-release-workflow) +exercises packaging and temporary upload without attaching a release asset. + ### 7. Remove the merged release branch After publication and baseline verification succeed: diff --git a/docs/code_organization.md b/docs/code_organization.md index 0303099..5305a03 100644 --- a/docs/code_organization.md +++ b/docs/code_organization.md @@ -100,6 +100,11 @@ owns the Python script inventory and entry points for comparisons, plotting, release metadata, changelog generation/archiving, and tag preparation. The [justfile](../justfile) owns executable development workflows. +`scripts/release_baseline.py` owns release-suite inventory and complete raw +Criterion validation. The release workflow packages only datasets that pass +that gate; its regression and archive tests live in +`scripts/tests/test_release_baseline.py`. + ## Documentation owners Use [Documentation guidance](dev/docs.md) for README, references, mathematical diff --git a/justfile b/justfile index 73cae4a..baa4246 100644 --- a/justfile +++ b/justfile @@ -285,6 +285,14 @@ bench-latest: bench-vs-linalg-la-stack bench-exact bench-latest-vs-last baseline="last": bench-latest python-sync uv run --locked bench-compare {{ quote(baseline) }} +# Discover all release benchmarks and report their expected measurement budget. +bench-release-inventory: _ensure-uv + uv run --locked scripts/release_baseline.py inventory + +# Check every discovered benchmark before the release workflow packages it. +bench-release-check tag: _ensure-uv + uv run --locked scripts/release_baseline.py validate --baseline {{ quote(tag) }} + # Save a Criterion baseline. Defaults to all release-signal benchmark suites. bench-save-baseline tag suite="all": #!/usr/bin/env bash diff --git a/scripts/README.md b/scripts/README.md index b08cf1d..3bda321 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -287,6 +287,7 @@ validates SemVer, and handles GitHub's 125KB tag-annotation size limit. | `criterion_dim_plot.py` | Plot Criterion benchmark results (CSV + SVG + README table) | | `tag_release.py` | Create annotated git tags from CHANGELOG.md sections | | `postprocess_changelog.py` | Normalize and reflow generated git-cliff Markdown safely | +| `release_baseline.py` | Inventory full Criterion suites and validate complete raw release baselines before packaging | | `subprocess_utils.py` | Safe subprocess wrappers for git commands | | `update_cargo_tool_pins.py` | Reconcile repository-owned Cargo and active uv tool pins with installed versions | | `update_python_dev_pins.py` | Resolve and advance exact Python development-tool pins through uv | diff --git a/scripts/release_baseline.py b/scripts/release_baseline.py new file mode 100644 index 0000000..5c08c50 --- /dev/null +++ b/scripts/release_baseline.py @@ -0,0 +1,186 @@ +"""Inventory and validate complete release Criterion datasets before packaging.""" + +import argparse +import json +import math +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import cast + +from bench_compare import ( + EXACT_GROUPS, + VS_LINALG_CANONICAL_DIMS, + VS_LINALG_RELEASE_SIGNAL_BENCHES_BY_DIM, + VS_LINALG_STANDARD_BENCH_ORDER, +) +from performance_artifacts import TimingEstimate +from subprocess_utils import ExecutableNotFoundError, format_exception_diagnostics, run_cargo_command + +SUITES = {"vs_linalg": "bench", "exact": "bench,exact"} +RAW_FILES = ("benchmark.json", "estimates.json", "sample.json", "tukey.json") + + +def parse_benchmark_list(output: str) -> list[str]: + """Read Criterion's full IDs, rejecting empty or duplicate inventories.""" + ids = [line.removesuffix(": benchmark") for line in output.splitlines() if line.endswith(": benchmark")] + if not ids or len(ids) != len(set(ids)) or any("/" not in name for name in ids): + msg = "Criterion inventory must contain unique, nonempty benchmark IDs" + raise ValueError(msg) + return sorted(ids) + + +def required_report_ids() -> set[str]: + """Include every release-report row and all canonical README peer rows.""" + return { + *(f"{group}/{bench}" for group, benches in EXACT_GROUPS.items() for bench in benches), + *( + f"d{dimension}/{bench}" + for dimension in VS_LINALG_CANONICAL_DIMS + for bench in (*VS_LINALG_STANDARD_BENCH_ORDER, *VS_LINALG_RELEASE_SIGNAL_BENCHES_BY_DIM.get(dimension, [])) + ), + } + + +def inventory_ids(data: object) -> set[str]: + """Validate the manifest and its coverage of the report consumers.""" + if not isinstance(data, dict) or set(data) != set(SUITES): + msg = "inventory must contain exactly vs_linalg and exact suites" + raise ValueError(msg) + expected: set[str] = set() + for suite, ids in data.items(): + if not isinstance(ids, list) or not ids or any(not isinstance(name, str) or not name for name in ids): + raise ValueError(f"invalid benchmark inventory for {suite}") + if len(ids) != len(set(ids)) or expected.intersection(ids): + raise ValueError(f"duplicate benchmark IDs in {suite}") + expected.update(ids) + missing = required_report_ids() - expected + if missing: + raise ValueError(f"inventory omits report consumers: {', '.join(sorted(missing))}") + return expected + + +def discover(root: Path, manifest: Path, criterion: Path) -> None: + """Compile and list both unfiltered suites without collecting measurements.""" + if criterion.exists() and any(criterion.iterdir()): + raise ValueError(f"release inventory requires a fresh Criterion directory: {criterion}") + inventory: dict[str, list[str]] = {} + for suite, features in SUITES.items(): + print(f"[release-baseline] Discovering {suite}", flush=True) + result = run_cargo_command( + ["bench", "--locked", "--features", features, "--bench", suite, "--", "--list"], + cwd=root, + capture_output=False, + stdout=subprocess.PIPE, + timeout=None, # The workflow bounds compilation and discovery together. + ) + inventory[suite] = parse_benchmark_list(result.stdout) + count = len(inventory[suite]) + print( + f"[release-baseline] {suite}: {count} benchmarks; " + f"nominal warmup + measurement {count * 8 / 60:.1f} min; " + f"planning estimate at 12 s/benchmark {count * 12 / 60:.1f} min (not an upper bound)", + flush=True, + ) + inventory_ids(inventory) + manifest.parent.mkdir(parents=True, exist_ok=True) + manifest.write_text(json.dumps(inventory, indent=2) + "\n", encoding="utf-8", newline="\n") + + +def read_object(path: Path) -> dict[str, object]: + """Require a JSON object at the raw artifact boundary.""" + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise TypeError(f"expected a JSON object in {path}") + return cast("dict[str, object]", data) # JSON objects always have string keys. + + +def positive_number(value: object) -> float: + """Reject booleans, nonnumeric values, and invalid timing numbers.""" + if isinstance(value, bool) or not isinstance(value, (float, int)) or not math.isfinite(value) or value <= 0: + raise ValueError(f"expected a finite positive timing value, got {value!r}") + return float(value) + + +def validate_measurement(directory: Path) -> None: + """Check full default sampling and the estimates used by report consumers.""" + samples = read_object(directory / "sample.json") + for field in ("iters", "times"): + values = samples.get(field) + if not isinstance(values, list) or len(values) != 100: + raise ValueError(f"{directory}: {field} must contain 100 samples") + for value in values: + positive_number(value) + estimates = read_object(directory / "estimates.json") + for statistic in ("mean", "median"): + estimate = estimates.get(statistic) + if not isinstance(estimate, dict) or not isinstance(interval := estimate.get("confidence_interval"), dict): + raise TypeError(f"{directory}: missing {statistic} estimate or confidence interval") + if interval.get("confidence_level") != 0.95: + raise ValueError(f"{directory}: expected a 95% confidence interval") + TimingEstimate( + median_ns=positive_number(estimate.get("point_estimate")), + ci_lower_ns=positive_number(interval.get("lower_bound")), + ci_upper_ns=positive_number(interval.get("upper_bound")), + ) + + +def validate(criterion: Path, manifest: Path, baseline: str) -> int: + """Fail closed on missing, corrupt, stale, or unexpected benchmark output.""" + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", baseline) or baseline in {"new", "base", "change", "report"}: + msg = "baseline must be a safe, nonreserved directory name" + raise ValueError(msg) + expected = inventory_ids(json.loads(manifest.read_text(encoding="utf-8"))) + report_ids = required_report_ids() + observed: set[str] = set() + for metadata in sorted(criterion.glob("**/new/benchmark.json")): + benchmark = read_object(metadata).get("full_id") + if not isinstance(benchmark, str) or benchmark not in expected or benchmark in observed: + raise ValueError(f"unexpected or duplicate benchmark ID in {metadata}: {benchmark!r}") + directory = metadata.parent + if benchmark in report_ids and directory.parent != criterion / benchmark: + raise ValueError(f"report benchmark is outside its consumer path: {metadata}") + saved = directory.parent / baseline + for filename in RAW_FILES: + # Criterion can log a write/copy error and still exit successfully. + # Require every saved raw file to match the freshly measured sample. + source = directory / filename + destination = saved / filename + if source.read_bytes() != destination.read_bytes(): + raise ValueError(f"saved baseline differs from new measurement: {destination}") + json.loads(source.read_text(encoding="utf-8")) + validate_measurement(directory) + observed.add(benchmark) + missing = expected - observed + if missing: + raise ValueError(f"incomplete Criterion dataset: {', '.join(sorted(missing))}") + return len(observed) + + +def main(argv: list[str] | None = None) -> int: + """Provide the release workflow's inventory and publication gates.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("inventory", "validate")) + parser.add_argument("--baseline") + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--manifest", type=Path, default=Path("target/release-benchmark-inventory.json")) + parser.add_argument("--criterion-dir", type=Path, default=Path(os.environ.get("CRITERION_HOME", "target/criterion"))) + args = parser.parse_args(argv) + try: + if args.command == "inventory": + discover(args.root, args.manifest, args.criterion_dir) + else: + if not args.baseline: + parser.error("validate requires --baseline") + count = validate(args.criterion_dir, args.manifest, args.baseline) + print(f"[release-baseline] Validated {count} complete benchmarks in new and {args.baseline}") + return 0 + except (OSError, ValueError, TypeError, ExecutableNotFoundError, subprocess.SubprocessError) as error: + print(f"[release-baseline] {format_exception_diagnostics(error)}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_release_baseline.py b/scripts/tests/test_release_baseline.py new file mode 100644 index 0000000..50cce9b --- /dev/null +++ b/scripts/tests/test_release_baseline.py @@ -0,0 +1,253 @@ +"""Exercise complete release archives and the workflow's publication barriers.""" + +import json +import os +import shutil +import subprocess +import tarfile +import textwrap +from pathlib import Path + +import pytest + +import release_baseline + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = REPO_ROOT / ".github/workflows/release-benchmarks.yml" +IDS = {"vs_linalg": ["d2/la_stack_dot", "diagnostic/peer"], "exact": ["exact_d2/det_exact"]} +BASELINE = "v1.2.3" + + +def write_json(path: Path, data: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data) + "\n", encoding="utf-8", newline="\n") + + +@pytest.fixture +def dataset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path, Path]: + monkeypatch.setattr(release_baseline, "required_report_ids", lambda: {"d2/la_stack_dot", "exact_d2/det_exact"}) + criterion = tmp_path / "target/criterion" + manifest = tmp_path / "target/release-benchmark-inventory.json" + write_json(manifest, IDS) + estimate = {"point_estimate": 2.0, "confidence_interval": {"confidence_level": 0.95, "lower_bound": 1.0, "upper_bound": 3.0}} + for ids in IDS.values(): + for benchmark in ids: + for sample in ("new", BASELINE): + directory = criterion / benchmark / sample + write_json(directory / "benchmark.json", {"full_id": benchmark}) + write_json(directory / "sample.json", {"iters": [1.0] * 100, "times": [2.0] * 100}) + write_json(directory / "estimates.json", {"mean": estimate, "median": estimate}) + write_json(directory / "tukey.json", [1.0, 2.0, 3.0, 4.0]) + return criterion, manifest + + +def shell_step(name: str) -> str: + """Extract a literal run block so tests exercise the workflow's actual shell.""" + text = WORKFLOW.read_text(encoding="utf-8") + step = text.split(f" - name: {name}\n", 1)[1].split("\n - ", 1)[0] + lines: list[str] = [] + for line in step.split(" run: |\n", 1)[1].splitlines(): + if line.strip() and not line.startswith(" "): + break + lines.append(line) + return textwrap.dedent("\n".join(lines)).rstrip() + + +def run_shell(script: str, cwd: Path, extra_env: dict[str, str]) -> subprocess.CompletedProcess[str]: + bash = shutil.which("bash") + assert bash is not None + return subprocess.run( # noqa: S603 - run only checked-in workflow shell in an isolated fixture. + [bash, "--noprofile", "--norc", "-c", script], + cwd=cwd, + env={**os.environ, **extra_env}, + capture_output=True, + encoding="utf-8", + check=False, + timeout=30, + ) + + +def test_complete_dataset_round_trips_through_workflow_archive(dataset: tuple[Path, Path], tmp_path: Path) -> None: + criterion, manifest = dataset + assert release_baseline.validate(criterion, manifest, BASELINE) == 3 + output = tmp_path / "step outputs" + # Git Bash accepts forward-slash drive paths on Windows as well as POSIX paths. + result = run_shell(shell_step("Package release Criterion baseline"), tmp_path, {"RELEASE_TAG": BASELINE, "GITHUB_OUTPUT": output.as_posix()}) + assert result.returncode == 0, result.stderr + asset = f"la-stack-{BASELINE}-criterion-baseline.tar.gz" + assert output.read_text() == f"asset={asset}\n" + with tarfile.open(tmp_path / asset) as archive: + for ids in IDS.values(): + for benchmark in ids: + for sample in ("new", BASELINE): + for filename in release_baseline.RAW_FILES: + member = archive.extractfile(f"criterion/{benchmark}/{sample}/{filename}") + assert member is not None + assert member.read() == (criterion / benchmark / sample / filename).read_bytes() + member = archive.extractfile("criterion/release-benchmark-inventory.json") + assert member is not None + assert json.loads(member.read()) == IDS + + +@pytest.mark.parametrize("filename", release_baseline.RAW_FILES) +@pytest.mark.parametrize("sample", ["new", BASELINE]) +def test_missing_raw_file_blocks_publication(dataset: tuple[Path, Path], sample: str, filename: str) -> None: + criterion, manifest = dataset + missing = criterion / "diagnostic/peer" / sample / filename + missing.unlink() + if sample == "new" and filename == "benchmark.json": + with pytest.raises(ValueError, match="incomplete Criterion dataset: diagnostic/peer"): + release_baseline.validate(criterion, manifest, BASELINE) + else: + with pytest.raises(FileNotFoundError) as error: + release_baseline.validate(criterion, manifest, BASELINE) + assert error.value.filename == str(missing) + + +def test_missing_entire_suite_blocks_publication(dataset: tuple[Path, Path]) -> None: + criterion, manifest = dataset + shutil.rmtree(criterion / "exact_d2") + with pytest.raises(ValueError, match="incomplete Criterion dataset: exact_d2/det_exact"): + release_baseline.validate(criterion, manifest, BASELINE) + + +def test_saved_baseline_must_match_fresh_samples(dataset: tuple[Path, Path]) -> None: + criterion, manifest = dataset + write_json(criterion / "d2/la_stack_dot" / BASELINE / "sample.json", {"iters": [1.0] * 100, "times": [3.0] * 100}) + with pytest.raises(ValueError, match="saved baseline differs"): + release_baseline.validate(criterion, manifest, BASELINE) + + +def test_report_consumers_require_canonical_paths(dataset: tuple[Path, Path]) -> None: + criterion, manifest = dataset + (criterion / "d2/la_stack_dot").rename(criterion / "d2/misplaced") + with pytest.raises(ValueError, match="outside its consumer path"): + release_baseline.validate(criterion, manifest, BASELINE) + + +@pytest.mark.parametrize("corruption", ["short-samples", "nonfinite", "reversed-interval", "wrong-confidence", "malformed"]) +def test_invalid_measurement_blocks_publication(dataset: tuple[Path, Path], corruption: str) -> None: + criterion, manifest = dataset + for sample in ("new", BASELINE): + directory = criterion / "d2/la_stack_dot" / sample + if corruption == "malformed": + (directory / "estimates.json").write_text("{broken", encoding="utf-8", newline="\n") + elif corruption in {"short-samples", "nonfinite"}: + times = [2.0] * 99 if corruption == "short-samples" else [float("inf")] * 100 + write_json(directory / "sample.json", {"iters": [1.0] * 100, "times": times}) + else: + estimates = json.loads((directory / "estimates.json").read_text()) + interval = estimates["median"]["confidence_interval"] + interval["lower_bound" if corruption == "reversed-interval" else "confidence_level"] = 4.0 + write_json(directory / "estimates.json", estimates) + errors = { + "short-samples": "times must contain 100 samples", + "nonfinite": "finite positive timing value", + "reversed-interval": "confidence interval must be ordered", + "wrong-confidence": "expected a 95% confidence interval", + "malformed": "Expecting property name", + } + with pytest.raises(ValueError, match=errors[corruption]): + release_baseline.validate(criterion, manifest, BASELINE) + + +@pytest.mark.parametrize("baseline", ["../escape", "", "new", "base", "change", "report", "/absolute"]) +def test_invalid_baseline_is_rejected(dataset: tuple[Path, Path], baseline: str) -> None: + with pytest.raises(ValueError, match="safe, nonreserved"): + release_baseline.validate(*dataset, baseline) + + +def test_inventory_includes_release_consumers_and_peers() -> None: + expected = release_baseline.required_report_ids() + assert {"d64/faer_lu_solve", "d64/nalgebra_lu_solve", "exact_hilbert_5x5/solve_exact", "rational_input_d8/solve_big_rational_gaussian"} <= expected + with pytest.raises(ValueError, match="inventory omits report consumers"): + release_baseline.inventory_ids(IDS) + + +@pytest.mark.parametrize("output", ["", "hello", "a/b: benchmark\na/b: benchmark", "a: benchmark"]) +def test_invalid_listing_is_rejected(output: str) -> None: + with pytest.raises(ValueError, match="unique, nonempty"): + release_baseline.parse_benchmark_list(output) + + +@pytest.mark.parametrize("host_newline", ["\n", "\r\n"], ids=["posix", "windows"]) +def test_discovery_uses_full_suites_and_never_times_inputs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, host_newline: str) -> None: + calls: list[list[str]] = [] + write_text = Path.write_text + + def host_write_text(path: Path, data: str, encoding: str | None = None, errors: str | None = None, newline: str | None = None) -> int: + # Emulate Windows default text translation on every CI host. An explicit + # newline policy must preserve identical manifest bytes on both platforms. + return write_text(path, data, encoding=encoding, errors=errors, newline=host_newline if newline is None else newline) + + def run(args: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + calls.append(args) + suite = args[args.index("--bench") + 1] + return subprocess.CompletedProcess(args, 0, stdout="\n".join(f"{name}: benchmark" for name in IDS[suite])) + + monkeypatch.setattr(release_baseline, "run_cargo_command", run) + monkeypatch.setattr(release_baseline, "required_report_ids", lambda: {"d2/la_stack_dot", "exact_d2/det_exact"}) + monkeypatch.setattr(Path, "write_text", host_write_text) + manifest = tmp_path / "manifest.json" + release_baseline.discover(tmp_path, manifest, tmp_path / "criterion") + assert manifest.read_bytes() == (json.dumps(IDS, indent=2) + "\n").encode("utf-8") + assert calls == [ + ["bench", "--locked", "--features", "bench", "--bench", "vs_linalg", "--", "--list"], + ["bench", "--locked", "--features", "bench,exact", "--bench", "exact", "--", "--list"], + ] + + +def test_discovery_refuses_stale_measurements(dataset: tuple[Path, Path], tmp_path: Path) -> None: + criterion, manifest = dataset + before = manifest.read_bytes() + with pytest.raises(ValueError, match="fresh Criterion directory"): + release_baseline.discover(tmp_path, manifest, criterion) + assert manifest.read_bytes() == before + + +def test_workflow_fails_closed_and_manual_runs_cannot_publish() -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + steps = [ + "Validate benchmark inputs", + "Inventory full release suites", + "Save comparative Criterion baseline", + "Save exact Criterion baseline", + "Validate complete release dataset", + "Package release Criterion baseline", + "Upload temporary baseline artifact", + ] + positions = [workflow.index(f"- name: {step}") for step in steps] + assert positions == sorted(positions) + assert "\n if:" not in workflow[positions[0] : positions[-1]] + assert "continue-on-error" not in workflow + assert "workflow_dispatch:" in workflow + publisher = workflow.split(" publish-baseline:\n", 1)[1] + assert "if: ${{ github.event_name == 'release' }}" in publisher + assert "needs: release-baseline" in publisher + assert "GH_REPO: ${{ github.repository }}" in publisher + assert "contents: write" not in workflow.split(" publish-baseline:\n", 1)[0] + + +@pytest.mark.parametrize(("suite", "step"), [("vs_linalg", "Save comparative Criterion baseline"), ("exact", "Save exact Criterion baseline")]) +def test_failed_suite_retains_timing_and_does_not_mark_completion(tmp_path: Path, suite: str, step: str) -> None: + (tmp_path / "target").mkdir() + # A shell function substitutes only the benchmark command; timestamps and + # failure handling execute exactly as checked into the workflow. + script = "just() { return 7; }\n" + shell_step(step) + result = run_shell(script, tmp_path, {"RELEASE_TAG": BASELINE}) + assert result.returncode == 7 + assert (tmp_path / f"target/{suite}-started").is_file() + assert not (tmp_path / f"target/{suite}-finished").exists() + summary = tmp_path / "suite summary.md" + result = run_shell( + shell_step("Suite timing summary"), + tmp_path, + { + "GITHUB_STEP_SUMMARY": summary.as_posix(), + "COMPARATIVE_OUTCOME": "failure", + "EXACT_OUTCOME": "skipped", + }, + ) + assert result.returncode == 0, result.stderr + assert "| vs_linalg | failure |" in summary.read_text() + assert "| exact | skipped |" in summary.read_text() From d8382eac0b6b467ea390ea04a5ec04fb4600bc8a Mon Sep 17 00:00:00 2001 From: Adam Getchell Date: Mon, 7 Sep 2026 12:16:49 -0700 Subject: [PATCH 2/2] fix: enforce benchmark setup limits and improve README navigation - Limit checkout to 2 minutes and share a 28-minute timeout across tool installation, input validation, and benchmark inventory. - Align budget documentation with the enforced setup limits. - Place scalar types, API navigation, and features after Quickstart, move Examples before Benchmarks, and update the Contents list. - Sort feature flags, scalar domains, documentation links, examples, and their run commands lexicographically. --- .../prepare-release-benchmarks/action.yml | 57 ++++ .github/workflows/release-benchmarks.yml | 57 +--- README.md | 292 +++++++++--------- docs/BENCHMARKING.md | 12 +- docs/code_organization.md | 3 + scripts/tests/test_release_baseline.py | 34 +- 6 files changed, 253 insertions(+), 202 deletions(-) create mode 100644 .github/actions/prepare-release-benchmarks/action.yml diff --git a/.github/actions/prepare-release-benchmarks/action.yml b/.github/actions/prepare-release-benchmarks/action.yml new file mode 100644 index 0000000..3313def --- /dev/null +++ b/.github/actions/prepare-release-benchmarks/action.yml @@ -0,0 +1,57 @@ +name: Prepare release benchmarks +description: Install uncached tools, validate inputs, and inventory full release suites within the caller's shared timeout. + +runs: + using: composite + steps: + - name: Install Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + cache: false + cache-bin: false + + - name: Set up just + uses: ./.github/actions/setup-just # zizmor: ignore[self-repository] actionlint 1.7.12 does not accept $/... + with: + cache: false + + - name: Resolve tool versions + id: tool_versions + shell: bash + run: | + set -euo pipefail + + version="$(just --evaluate cargo_nextest_version)" + uv_version="$(just --evaluate uv_version)" + if [[ -z "$version" || -z "$uv_version" ]]; then + echo "::error::Could not resolve pinned tool versions from justfile" + exit 1 + fi + + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "uv_version=$uv_version" >> "$GITHUB_OUTPUT" + + - name: Install cargo-nextest + shell: bash + env: + CARGO_NEXTEST_VERSION: ${{ steps.tool_versions.outputs.version }} + run: cargo install --locked cargo-nextest --version "$CARGO_NEXTEST_VERSION" + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version-file: ".python-version" + + - name: Install uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + version: ${{ steps.tool_versions.outputs.uv_version }} + enable-cache: false + + - name: Validate benchmark inputs + shell: bash + run: just test-bench-inputs + + - name: Inventory full release suites + shell: bash + run: just bench-release-inventory diff --git a/.github/workflows/release-benchmarks.yml b/.github/workflows/release-benchmarks.yml index d24c1f8..5eff906 100644 --- a/.github/workflows/release-benchmarks.yml +++ b/.github/workflows/release-benchmarks.yml @@ -25,7 +25,8 @@ env: jobs: release-baseline: runs-on: ubuntu-latest - # 30 min setup/discovery + 150 min comparative + 90 min exact + 15 min tail. + # 2 min checkout + 28 min shared setup + 150 min comparative + 90 min exact + # + 15 min for the tail and runner overhead. timeout-minutes: 285 env: RELEASE_TAG: ${{ github.event.release.tag_name || format('validation-{0}-{1}', github.run_id, github.run_attempt) }} @@ -38,56 +39,12 @@ jobs: with: ref: ${{ github.event.release.tag_name || github.sha }} persist-credentials: false + timeout-minutes: 2 - - name: Install Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 - with: - cache: false - cache-bin: false - - - name: Set up just - uses: ./.github/actions/setup-just # zizmor: ignore[self-repository] actionlint 1.7.12 does not accept $/... - with: - cache: false - - - name: Resolve tool versions - id: tool_versions - shell: bash - run: | - set -euo pipefail - - version="$(just --evaluate cargo_nextest_version)" - uv_version="$(just --evaluate uv_version)" - if [[ -z "$version" || -z "$uv_version" ]]; then - echo "::error::Could not resolve pinned tool versions from justfile" - exit 1 - fi - - echo "version=$version" >> "$GITHUB_OUTPUT" - echo "uv_version=$uv_version" >> "$GITHUB_OUTPUT" - - - name: Install cargo-nextest - env: - CARGO_NEXTEST_VERSION: ${{ steps.tool_versions.outputs.version }} - run: cargo install --locked cargo-nextest --version "$CARGO_NEXTEST_VERSION" - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version-file: ".python-version" - - - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - with: - version: ${{ steps.tool_versions.outputs.uv_version }} - enable-cache: false - - - name: Validate benchmark inputs - run: just test-bench-inputs - - - name: Inventory full release suites - timeout-minutes: 20 - run: just bench-release-inventory + - name: Prepare release benchmarks + # One parent timeout bounds all nested installs, validation, and inventory. + timeout-minutes: 28 + uses: ./.github/actions/prepare-release-benchmarks # zizmor: ignore[self-repository] actionlint 1.7.12 does not accept $/... - name: Save comparative Criterion baseline id: comparative diff --git a/README.md b/README.md index 46cc1ae..f203612 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,8 @@ while keeping the API intentionally small and explicit. - [Introduction](#-introduction) - [Use this crate when](#-use-this-crate-when) - [Quickstart](#-quickstart) -- [Mathematical basis](#-mathematical-basis) -- [Design goals](#-design-goals) -- [Anti-goals](#-anti-goals) - [Scalar and bounded-value types](#-scalar-and-bounded-value-types) +- [API at a glance](#-api-at-a-glance) - [Features](#-features) - [Adaptive determinant filtering (D โ‰ค 4)](#adaptive-determinant-filtering-d--4) - [Certified dot products and affine differences](#certified-dot-products-and-affine-differences) @@ -35,10 +33,12 @@ while keeping the API intentionally small and explicit. - [LU solve](#lu-solve) - [Outward-rounded interval determinants](#outward-rounded-interval-determinants) - [Overflow-safe Euclidean norms](#overflow-safe-euclidean-norms) -- [API at a glance](#-api-at-a-glance) +- [Mathematical basis](#-mathematical-basis) +- [Design goals](#-design-goals) +- [Anti-goals](#-anti-goals) - [Documentation Map](#documentation-map) -- [Benchmarks](#-benchmarks-vs-nalgebrafaer) - [Examples](#-examples) +- [Benchmarks](#-benchmarks-vs-nalgebrafaer) - [Contributing](#-contributing) - [Citation](#-citation) - [References](#-references) @@ -127,93 +127,20 @@ provide a certified solution error bound. ### Feature flags +- `bench`: repository-development gate used only by benchmark targets and + benchmark-input tests; application crates should not enable it - `default`: no runtime dependencies; includes outward-rounded `Interval` and `IntervalMatrix` APIs - `exact`: exact determinant signs, determinant values, and solves over stored `f64` values or caller-supplied `BigRational` inputs -- `bench`: repository-development gate used only by benchmark targets and - benchmark-input tests; application crates should not enable it - -## ๐Ÿงฎ Mathematical basis - -`la-stack` operates on finite IEEE 754 binary64 values in small, fixed -dimensions. Its floating-point paths use [LU with partial pivoting][refs-lu], -[LDLT without pivoting][refs-ldlt] for exactly symmetric positive-definite matrices, and closed-form -determinants through D=4. These results remain subject to conditioning and -binary64 rounding; -factorization tolerances are rejection thresholds, not accuracy guarantees. For -Dโ‰ค4, direct determinants can be paired with a -[conservative absolute roundoff bound][refs-det-bound] when its range -preconditions hold. [Fixed-vector dot products and direct affine differences][refs-reductions] -can likewise return a paired estimate and certified absolute -roundoff bound without enabling arbitrary-precision dependencies. - -Derived binary64 expressions can instead be assembled with `Interval` -subtraction, addition, multiplication, negation, and square. The resulting -`IntervalMatrix` [determinant sign][refs-interval] is certified through D=7 when its enclosure -separates zero; the singleton `[0, 0]` also certifies exact zero. Every other -overlap with zero is explicitly inconclusive. This default-feature surface is -distinct from arbitrary-precision exact arithmetic. - -With `features = ["exact"]`, callers can either lift stored binary64 inputs -losslessly or supply already-exact rational inputs for -[exact determinant signs][refs-exact-sign], determinant values, and -[solves][refs-exact-solve]. Exactness over binary64 input starts at the -stored values and cannot recover information rounded away before construction. -See the -[mathematical basis](https://github.com/acgetchell/la-stack/blob/main/docs/mathematical_basis.md) -for the algorithms, validity boundaries, and supporting references. - -## ๐ŸŽฏ Design goals - -- โœ… `const fn` where possible (compile-time evaluation of determinants, dot products, etc.) -- โœ… Const-generic storage (no dynamically sized matrix or vector representation) -- โœ… `Copy` types where possible -- โœ… Defined binary64 arithmetic semantics: Rust's `f64::algebraic_*` - operations are forbidden because their unspecified reassociation, precision, - and special-value behavior is incompatible with the crate's error bounds, - non-finite classification, exact fallbacks, and reproducibility contract; - deliberate `f64::mul_add` remains allowed for its defined single-rounding - semantics -- โœ… Error-bounded f64 dot, affine-difference, and determinant filtering plus - optional exact signs (`dot_with_errbound`, `dot_difference_with_errbound`, - `det_errbound`, `det_sign_exact`) -- โœ… Overflow- and underflow-safe Euclidean vector norms (`norm`) -- โœ… Outward-rounded interval expressions and division-free determinant signs - through D=7, with explicit inconclusive evidence -- โœ… Exact determinant values and linear solves via optional arbitrary-precision - arithmetic (`det_exact`, `solve_exact`, strict/rounded f64 conversions) -- โœ… Explicit algorithms (LU, solve, determinant) -- โœ… Inline, stack-backed storage for core types; optional arbitrary-precision - exact values allocate as required -- โœ… No runtime dependencies by default (optional features may add deps) -- โœ… `unsafe` forbidden - -See [CHANGELOG.md](https://github.com/acgetchell/la-stack/blob/v0.4.5/CHANGELOG.md) -for release history and -[docs/roadmap.md](https://github.com/acgetchell/la-stack/blob/v0.4.5/docs/roadmap.md) -for current release planning. - -## ๐Ÿšซ Anti-goals - -- Alternate scalar families: `la-stack` deliberately supports finite `f64` and - optional exact `BigRational` input domains, not `f32`, `f16`, complex, or - generic scalar APIs -- Bare-metal performance: use [`blas`](https://crates.io/crates/blas) or - [`lapack`](https://crates.io/crates/lapack) with a native backend selected - through [`blas-src`](https://crates.io/crates/blas-src), - [`lapack-src`](https://crates.io/crates/lapack-src), or - [`openblas-src`](https://crates.io/crates/openblas-src) -- Broad general-purpose linear algebra: use [`nalgebra`](https://crates.io/crates/nalgebra) -- Large matrices/dimensions with parallelism: use [`faer`](https://crates.io/crates/faer) ## ๐Ÿ”ข Scalar and bounded-value types The public point-value scalar model deliberately has two input domains: -- finite `f64` through `Matrix` and `Vector` for floating-point work; - arbitrary-precision `BigRational` through `RationalMatrix` and - `RationalVector` behind the optional `"exact"` feature. + `RationalVector` behind the optional `"exact"` feature; +- finite `f64` through `Matrix` and `Vector` for floating-point work. `Interval` is a separate bounded-value layer over finite `f64` endpoints. It encloses exact-real values during a small set of outward-rounded operations and @@ -230,6 +157,47 @@ Lower-precision `f32` / `f16` throughput-oriented workloads are outside the crate's scope; they usually indicate large-matrix or accelerator-oriented use cases better served by broader linear-algebra libraries. +## ๐Ÿงฉ API at a glance + +Start with the capability you need; the [API reference][api-reference] lists +the complete public surface, and the [worked examples][api-guide] show how +to combine operations. + +| Capability | Main entry points | +|---|---| +| Vector operations and norms | [`Vector`][api-vector] | +| Floating-point determinants and solves | [`Matrix`][api-matrix], [`Lu`][api-lu], [`Ldlt`][api-ldlt] | +| Gram matrix construction | [`gram_matrix`][api-gram] | +| Certified dot, affine-difference, and determinant estimates | [`ScalarWithErrorBound`][api-scalar-bound], [`DeterminantWithErrorBound`][api-det-bound] | +| Interval expressions and determinant signs | [`Interval`][api-interval], [`IntervalMatrix`][api-interval-matrix] | +| Exact signs, determinants, solves, and output conversionยน | [Exact arithmetic examples][api-exact] | +| Runtime selection of a const-generic matrix dimension | [Dimension dispatch examples][api-dispatch] | + +[`Tolerance`][api-tolerance] validates numerical rejection thresholds. +[`LaError`][api-error] and its reason/location enums preserve structured +failure details; match non-exhaustive enums with a wildcard and struct-style +variants with `..`. See the [storage, access, and error guide][api-contracts] +for the full contracts. + +ยน Requires `features = ["exact"]`. + +[api-reference]: https://docs.rs/la-stack/latest/la_stack/ +[api-guide]: https://docs.rs/la-stack/latest/la_stack/guide/index.html +[api-vector]: https://docs.rs/la-stack/latest/la_stack/struct.Vector.html +[api-matrix]: https://docs.rs/la-stack/latest/la_stack/struct.Matrix.html +[api-lu]: https://docs.rs/la-stack/latest/la_stack/struct.Lu.html +[api-ldlt]: https://docs.rs/la-stack/latest/la_stack/struct.Ldlt.html +[api-gram]: https://docs.rs/la-stack/latest/la_stack/fn.gram_matrix.html +[api-scalar-bound]: https://docs.rs/la-stack/latest/la_stack/struct.ScalarWithErrorBound.html +[api-det-bound]: https://docs.rs/la-stack/latest/la_stack/struct.DeterminantWithErrorBound.html +[api-interval]: https://docs.rs/la-stack/latest/la_stack/struct.Interval.html +[api-interval-matrix]: https://docs.rs/la-stack/latest/la_stack/struct.IntervalMatrix.html +[api-exact]: https://docs.rs/la-stack/latest/la_stack/guide/exact/index.html +[api-dispatch]: https://docs.rs/la-stack/latest/la_stack/guide/index.html#dimension-dispatch +[api-tolerance]: https://docs.rs/la-stack/latest/la_stack/struct.Tolerance.html +[api-error]: https://docs.rs/la-stack/latest/la_stack/enum.LaError.html +[api-contracts]: https://docs.rs/la-stack/latest/la_stack/guide/index.html#storage-access-and-errors + ## โœจ Features ### Adaptive determinant filtering (D โ‰ค 4) @@ -315,58 +283,116 @@ the renames. `Matrix::norm_inf()` remains the maximum absolute row sum. [guide-certified]: https://docs.rs/la-stack/latest/la_stack/guide/certified/index.html [guide-adaptive]: https://docs.rs/la-stack/latest/la_stack/guide/adaptive/index.html -## ๐Ÿงฉ API at a glance +## ๐Ÿงฎ Mathematical basis -Start with the capability you need; the [API reference][api-reference] lists -the complete public surface, and the [worked examples][api-guide] show how -to combine operations. +`la-stack` operates on finite IEEE 754 binary64 values in small, fixed +dimensions. Its floating-point paths use [LU with partial pivoting][refs-lu], +[LDLT without pivoting][refs-ldlt] for exactly symmetric positive-definite matrices, and closed-form +determinants through D=4. These results remain subject to conditioning and +binary64 rounding; +factorization tolerances are rejection thresholds, not accuracy guarantees. For +Dโ‰ค4, direct determinants can be paired with a +[conservative absolute roundoff bound][refs-det-bound] when its range +preconditions hold. [Fixed-vector dot products and direct affine differences][refs-reductions] +can likewise return a paired estimate and certified absolute +roundoff bound without enabling arbitrary-precision dependencies. -| Capability | Main entry points | -|---|---| -| Vector operations and norms | [`Vector`][api-vector] | -| Floating-point determinants and solves | [`Matrix`][api-matrix], [`Lu`][api-lu], [`Ldlt`][api-ldlt] | -| Gram matrix construction | [`gram_matrix`][api-gram] | -| Certified dot, affine-difference, and determinant estimates | [`ScalarWithErrorBound`][api-scalar-bound], [`DeterminantWithErrorBound`][api-det-bound] | -| Interval expressions and determinant signs | [`Interval`][api-interval], [`IntervalMatrix`][api-interval-matrix] | -| Exact signs, determinants, solves, and output conversionยน | [Exact arithmetic examples][api-exact] | -| Runtime selection of a const-generic matrix dimension | [Dimension dispatch examples][api-dispatch] | +Derived binary64 expressions can instead be assembled with `Interval` +subtraction, addition, multiplication, negation, and square. The resulting +`IntervalMatrix` [determinant sign][refs-interval] is certified through D=7 when its enclosure +separates zero; the singleton `[0, 0]` also certifies exact zero. Every other +overlap with zero is explicitly inconclusive. This default-feature surface is +distinct from arbitrary-precision exact arithmetic. -[`Tolerance`][api-tolerance] validates numerical rejection thresholds. -[`LaError`][api-error] and its reason/location enums preserve structured -failure details; match non-exhaustive enums with a wildcard and struct-style -variants with `..`. See the [storage, access, and error guide][api-contracts] -for the full contracts. +With `features = ["exact"]`, callers can either lift stored binary64 inputs +losslessly or supply already-exact rational inputs for +[exact determinant signs][refs-exact-sign], determinant values, and +[solves][refs-exact-solve]. Exactness over binary64 input starts at the +stored values and cannot recover information rounded away before construction. +See the +[mathematical basis](https://github.com/acgetchell/la-stack/blob/main/docs/mathematical_basis.md) +for the algorithms, validity boundaries, and supporting references. -ยน Requires `features = ["exact"]`. +## ๐ŸŽฏ Design goals -[api-reference]: https://docs.rs/la-stack/latest/la_stack/ -[api-guide]: https://docs.rs/la-stack/latest/la_stack/guide/index.html -[api-vector]: https://docs.rs/la-stack/latest/la_stack/struct.Vector.html -[api-matrix]: https://docs.rs/la-stack/latest/la_stack/struct.Matrix.html -[api-lu]: https://docs.rs/la-stack/latest/la_stack/struct.Lu.html -[api-ldlt]: https://docs.rs/la-stack/latest/la_stack/struct.Ldlt.html -[api-gram]: https://docs.rs/la-stack/latest/la_stack/fn.gram_matrix.html -[api-scalar-bound]: https://docs.rs/la-stack/latest/la_stack/struct.ScalarWithErrorBound.html -[api-det-bound]: https://docs.rs/la-stack/latest/la_stack/struct.DeterminantWithErrorBound.html -[api-interval]: https://docs.rs/la-stack/latest/la_stack/struct.Interval.html -[api-interval-matrix]: https://docs.rs/la-stack/latest/la_stack/struct.IntervalMatrix.html -[api-exact]: https://docs.rs/la-stack/latest/la_stack/guide/exact/index.html -[api-dispatch]: https://docs.rs/la-stack/latest/la_stack/guide/index.html#dimension-dispatch -[api-tolerance]: https://docs.rs/la-stack/latest/la_stack/struct.Tolerance.html -[api-error]: https://docs.rs/la-stack/latest/la_stack/enum.LaError.html -[api-contracts]: https://docs.rs/la-stack/latest/la_stack/guide/index.html#storage-access-and-errors +- โœ… `const fn` where possible (compile-time evaluation of determinants, dot products, etc.) +- โœ… Const-generic storage (no dynamically sized matrix or vector representation) +- โœ… `Copy` types where possible +- โœ… Defined binary64 arithmetic semantics: Rust's `f64::algebraic_*` + operations are forbidden because their unspecified reassociation, precision, + and special-value behavior is incompatible with the crate's error bounds, + non-finite classification, exact fallbacks, and reproducibility contract; + deliberate `f64::mul_add` remains allowed for its defined single-rounding + semantics +- โœ… Error-bounded f64 dot, affine-difference, and determinant filtering plus + optional exact signs (`dot_with_errbound`, `dot_difference_with_errbound`, + `det_errbound`, `det_sign_exact`) +- โœ… Overflow- and underflow-safe Euclidean vector norms (`norm`) +- โœ… Outward-rounded interval expressions and division-free determinant signs + through D=7, with explicit inconclusive evidence +- โœ… Exact determinant values and linear solves via optional arbitrary-precision + arithmetic (`det_exact`, `solve_exact`, strict/rounded f64 conversions) +- โœ… Explicit algorithms (LU, solve, determinant) +- โœ… Inline, stack-backed storage for core types; optional arbitrary-precision + exact values allocate as required +- โœ… No runtime dependencies by default (optional features may add deps) +- โœ… `unsafe` forbidden + +See [CHANGELOG.md](https://github.com/acgetchell/la-stack/blob/v0.4.5/CHANGELOG.md) +for release history and +[docs/roadmap.md](https://github.com/acgetchell/la-stack/blob/v0.4.5/docs/roadmap.md) +for current release planning. + +## ๐Ÿšซ Anti-goals + +- Alternate scalar families: `la-stack` deliberately supports finite `f64` and + optional exact `BigRational` input domains, not `f32`, `f16`, complex, or + generic scalar APIs +- Bare-metal performance: use [`blas`](https://crates.io/crates/blas) or + [`lapack`](https://crates.io/crates/lapack) with a native backend selected + through [`blas-src`](https://crates.io/crates/blas-src), + [`lapack-src`](https://crates.io/crates/lapack-src), or + [`openblas-src`](https://crates.io/crates/openblas-src) +- Broad general-purpose linear algebra: use [`nalgebra`](https://crates.io/crates/nalgebra) +- Large matrices/dimensions with parallelism: use [`faer`](https://crates.io/crates/faer) ## ๐Ÿ—บ๏ธ Documentation Map - [API guide][api-guide] โ€” worked examples, API selection, storage, and error contracts. -- [Mathematical basis](https://github.com/acgetchell/la-stack/blob/main/docs/mathematical_basis.md) โ€” algorithms, numerical guarantees, and limitations. - [Benchmarking](https://github.com/acgetchell/la-stack/blob/v0.4.5/docs/BENCHMARKING.md) โ€” benchmark suites, comparison workflows, and measurement methodology. -- [Performance reports](https://github.com/acgetchell/la-stack/blob/main/docs/performance.md) โ€” release-to-release measurement results and provenance. - [Coverage](https://github.com/acgetchell/la-stack/blob/main/docs/MEASURING_COVERAGE.md) โ€” local and CI coverage commands and report locations. -- [Roadmap](https://github.com/acgetchell/la-stack/blob/v0.4.5/docs/roadmap.md) โ€” release planning, future directions, and non-goals. +- [Mathematical basis](https://github.com/acgetchell/la-stack/blob/main/docs/mathematical_basis.md) โ€” algorithms, numerical guarantees, and limitations. +- [Performance reports](https://github.com/acgetchell/la-stack/blob/main/docs/performance.md) โ€” release-to-release measurement results and provenance. - [Releasing](https://github.com/acgetchell/la-stack/blob/v0.4.5/docs/RELEASING.md) โ€” release preparation, validation, and publication. +- [Roadmap](https://github.com/acgetchell/la-stack/blob/v0.4.5/docs/roadmap.md) โ€” release planning, future directions, and non-goals. + +## ๐Ÿ“‹ Examples + +The `examples/` directory contains small, runnable programs: + +- **`const_det_4x4`** โ€” compile-time 4ร—4 determinant via `det_direct()` +- **`det_5x5`** โ€” determinant of a 5ร—5 matrix via LU +- **`exact_det_3x3`** โ€” exact determinant value of a near-singular 3ร—3 matrix (requires `exact` feature) +- **`exact_sign_3x3`** โ€” exact determinant sign of a near-singular 3ร—3 matrix (requires `exact` feature) +- **`exact_solve_3x3`** โ€” exact solve of a near-singular 3ร—3 system vs f64 LU (requires `exact` feature) +- **`ldlt_solve_3x3`** โ€” solve a 3ร—3 symmetric positive definite system via LDLT +- **`rational_input_5x5`** โ€” exact rational solve of a 5ร—5 system that becomes singular as f64 (requires `exact` feature) +- **`solve_5x5`** โ€” solve a 5ร—5 system via LU with partial pivoting + +```bash +just examples +# or individually: +cargo run --example const_det_4x4 +cargo run --example det_5x5 +cargo run --features exact --example exact_det_3x3 +cargo run --features exact --example exact_sign_3x3 +cargo run --features exact --example exact_solve_3x3 +cargo run --example ldlt_solve_3x3 +cargo run --features exact --example rational_input_5x5 +cargo run --example solve_5x5 +``` ## ๐Ÿ“ˆ Benchmarks (vs nalgebra/faer) @@ -430,32 +456,6 @@ expectations are validated outside the timed closures. -## ๐Ÿ“‹ Examples - -The `examples/` directory contains small, runnable programs: - -- **`solve_5x5`** โ€” solve a 5ร—5 system via LU with partial pivoting -- **`det_5x5`** โ€” determinant of a 5ร—5 matrix via LU -- **`ldlt_solve_3x3`** โ€” solve a 3ร—3 symmetric positive definite system via LDLT -- **`const_det_4x4`** โ€” compile-time 4ร—4 determinant via `det_direct()` -- **`exact_det_3x3`** โ€” exact determinant value of a near-singular 3ร—3 matrix (requires `exact` feature) -- **`exact_sign_3x3`** โ€” exact determinant sign of a near-singular 3ร—3 matrix (requires `exact` feature) -- **`exact_solve_3x3`** โ€” exact solve of a near-singular 3ร—3 system vs f64 LU (requires `exact` feature) -- **`rational_input_5x5`** โ€” exact rational solve of a 5ร—5 system that becomes singular as f64 (requires `exact` feature) - -```bash -just examples -# or individually: -cargo run --example solve_5x5 -cargo run --example det_5x5 -cargo run --example ldlt_solve_3x3 -cargo run --example const_det_4x4 -cargo run --features exact --example exact_det_3x3 -cargo run --features exact --example exact_sign_3x3 -cargo run --features exact --example exact_solve_3x3 -cargo run --features exact --example rational_input_5x5 -``` - ## ๐Ÿค Contributing A short contributor workflow: diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index 396003a..243c776 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -691,10 +691,14 @@ comparison is `docs/performance.md`, created by `just performance-release`. The producer runs full `vs_linalg` and `exact` suites sequentially on one `ubuntu-latest` runner. Separate steps allow 150 and 90 minutes respectively; -the outer job allows 285 minutes, reserving 30 minutes for cold setup, input -validation and inventory, plus 15 minutes for validation, packaging, upload, -and diagnostics. Discovery compiles both suites before measurement and has -its own 20-minute timeout. Dependency caches remain disabled. +the outer job allows 285 minutes. Checkout has a 2-minute timeout, followed +by a composite preparation step with one shared 28-minute timeout covering +all tool installation, input validation, and inventory. These two limits bound +setup execution to 30 minutes; exceeding either fails the producer before +benchmarking or publication. The remaining 15 minutes provide headroom for +dataset validation, packaging, upload, diagnostics, and runner overhead. +Discovery compiles both suites before measurement and shares the preparation +deadline with the preceding work. Dependency caches remain disabled. The [v0.4.5 run](https://github.com/acgetchell/la-stack/actions/runs/32444040827) measured 304 comparative benchmarks in about 55m 34s, plus 2m 9s compilation diff --git a/docs/code_organization.md b/docs/code_organization.md index 5305a03..d5ab553 100644 --- a/docs/code_organization.md +++ b/docs/code_organization.md @@ -105,6 +105,9 @@ Criterion validation. The release workflow packages only datasets that pass that gate; its regression and archive tests live in `scripts/tests/test_release_baseline.py`. +`.github/actions/prepare-release-benchmarks/action.yml` groups tool installation, +input validation, and inventory under the release workflow's shared setup timeout. + ## Documentation owners Use [Documentation guidance](dev/docs.md) for README, references, mathematical diff --git a/scripts/tests/test_release_baseline.py b/scripts/tests/test_release_baseline.py index 50cce9b..b840153 100644 --- a/scripts/tests/test_release_baseline.py +++ b/scripts/tests/test_release_baseline.py @@ -2,6 +2,7 @@ import json import os +import re import shutil import subprocess import tarfile @@ -14,6 +15,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] WORKFLOW = REPO_ROOT / ".github/workflows/release-benchmarks.yml" +PREPARATION = REPO_ROOT / ".github/actions/prepare-release-benchmarks/action.yml" IDS = {"vs_linalg": ["d2/la_stack_dot", "diagnostic/peer"], "exact": ["exact_d2/det_exact"]} BASELINE = "v1.2.3" @@ -207,9 +209,13 @@ def test_discovery_refuses_stale_measurements(dataset: tuple[Path, Path], tmp_pa def test_workflow_fails_closed_and_manual_runs_cannot_publish() -> None: workflow = WORKFLOW.read_text(encoding="utf-8") + preparation = PREPARATION.read_text(encoding="utf-8") + assert preparation.index("- name: Validate benchmark inputs") < preparation.index("- name: Inventory full release suites") + assert "run: just test-bench-inputs" in preparation + assert "run: just bench-release-inventory" in preparation + assert "continue-on-error" not in preparation steps = [ - "Validate benchmark inputs", - "Inventory full release suites", + "Prepare release benchmarks", "Save comparative Criterion baseline", "Save exact Criterion baseline", "Validate complete release dataset", @@ -228,6 +234,30 @@ def test_workflow_fails_closed_and_manual_runs_cannot_publish() -> None: assert "contents: write" not in workflow.split(" publish-baseline:\n", 1)[0] +def test_setup_limits_preserve_benchmark_and_tail_budgets() -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + producer = workflow.split(" release-baseline:\n", 1)[1].split(" publish-baseline:\n", 1)[0] + job_limit = re.search(r"(?m)^ timeout-minutes: (\d+)$", producer) + assert job_limit is not None + setup = producer.split(" steps:\n", 1)[1].split(" - name: Save comparative Criterion baseline", 1)[0] + setup_steps = re.findall(r"(?ms)^ - .*?(?=^ - |\Z)", setup) + assert setup_steps + setup_limits: list[int] = [] + for step in setup_steps: + limit = re.search(r"(?m)^ timeout-minutes: (\d+)$", step) + assert limit is not None, f"unbounded setup step: {step}" + setup_limits.append(int(limit[1])) + assert all(limit > 0 for limit in setup_limits) + assert sum(setup_limits) <= 30 + assert "uses: ./.github/actions/prepare-release-benchmarks" in setup + assert "using: composite" in PREPARATION.read_text(encoding="utf-8") + benchmark_limits = [ + int(limit) for limit in re.findall(r"(?ms)^ - name: Save (?:comparative|exact) Criterion baseline\n.*?^ timeout-minutes: (\d+)$", producer) + ] + assert benchmark_limits == [150, 90] + assert int(job_limit[1]) - sum(setup_limits) - sum(benchmark_limits) >= 15 + + @pytest.mark.parametrize(("suite", "step"), [("vs_linalg", "Save comparative Criterion baseline"), ("exact", "Save exact Criterion baseline")]) def test_failed_suite_retains_timing_and_does_not_mark_completion(tmp_path: Path, suite: str, step: str) -> None: (tmp_path / "target").mkdir()