diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6c80c53..195b930 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -69,11 +69,45 @@ jobs: run: ./tests/full_validate.sh --gate style - name: Validate docs, schemas, contracts, and generated files run: ./tests/full_validate.sh --gate contracts - - name: Enforce benchmark budget - run: ./tests/full_validate.sh --gate benchmark - name: Run dependency and static security checks run: ./tests/full_validate.sh --gate security + benchmarks: + name: Benchmark (${{ matrix.profile }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - profile: unix + os: ubuntu-latest + - profile: macos + os: macos-latest + - profile: windows + os: windows-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + - name: Install declared benchmark comparators + run: python -m pip install ".[typer,benchmark]" + - name: Run calibrated comparative benchmark + env: + BASE_CLI_BENCHMARK_PLATFORM: ${{ matrix.profile }} + SOURCE_REVISION: ${{ github.sha }} + run: python scripts/benchmark_runtime.py --check --iterations 31 --output benchmark-results.json + - name: Retain dated machine-readable benchmark evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: base-cli-benchmark-${{ matrix.profile }}-${{ github.run_id }} + path: benchmark-results.json + if-no-files-found: ignore + retention-days: 90 + linux-distributions: name: Validate (${{ matrix.name }}) runs-on: ubuntu-latest @@ -136,4 +170,13 @@ jobs: $drive = $env:GITHUB_WORKSPACE.Substring(0, 1).ToLowerInvariant() $path = $env:GITHUB_WORKSPACE.Substring(2).Replace('\', '/') $linuxWorkspace = "/mnt/$drive$path" - wsl --distribution Ubuntu --user root -- bash -lc "set -eu; cd '$linuxWorkspace'; sed -i 's/\r$//' tests/full_validate.sh tests/validate.sh; apt-get update -qq; apt-get install -y -qq nodejs npm python3-venv python3.14-venv; python3 -m venv /tmp/base-cli-venv; . /tmp/base-cli-venv/bin/activate; python -m pip install '.[dev,typer]'; bash tests/full_validate.sh --gate runtime" + $revision = $env:GITHUB_SHA + wsl --distribution Ubuntu --user root -- bash -lc "set -eu; cd '$linuxWorkspace'; sed -i 's/\r$//' tests/full_validate.sh tests/validate.sh; apt-get update -qq; apt-get install -y -qq nodejs npm python3-venv python3.14-venv; python3 -m venv /tmp/base-cli-venv; . /tmp/base-cli-venv/bin/activate; python -m pip install '.[dev,typer,benchmark]'; bash tests/full_validate.sh --gate runtime; BASE_CLI_BENCHMARK_PLATFORM=wsl SOURCE_REVISION='$revision' python scripts/benchmark_runtime.py --check --iterations 31 --output base-cli-benchmark-wsl.json" + - name: Retain WSL benchmark evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: base-cli-benchmark-wsl-${{ github.run_id }} + path: base-cli-benchmark-wsl.json + if-no-files-found: ignore + retention-days: 90 diff --git a/CHANGELOG.md b/CHANGELOG.md index ec20c2e..1a20b2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ and versions are tracked in the repo-root `VERSION` file. - Continue compatibility hardening and adoption work for the next release. +### Added + +- Publish versioned, comparative CLI benchmark reports with lifecycle and + feature scenarios, platform-specific regression gates, and retained CI evidence. + ### Fixed - Preserve explicit application identities losslessly while using diff --git a/docs/performance.md b/docs/performance.md index a46b881..ec883e1 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -1,42 +1,93 @@ # Performance and adversarial-regression contract `base-cli` treats startup and filesystem behavior as part of its public -quality contract. The checked benchmark is intentionally small and runs from -the source checkout: +quality contract. The benchmark is a comparative regression check, not a claim +that a lifecycle framework should outpace bare parsers. Its scenarios separate +interpreter/import cost, parser dispatch, the base-cli lifecycle, optional +features, and persistence. + +Install the complete local validation set, including every comparator: ```bash -python scripts/benchmark_runtime.py --check +python -m pip install '.[dev,typer,quality,benchmark]' +python scripts/benchmark_runtime.py --check --iterations 31 --output benchmark-results.json ``` -It records fresh-process import time and the cost of an isolated production -invocation through `base_cli.testing.invoke`. The comparison mode measures -equivalent no-op commands for base-cli, Click, Typer, and (when installed) -Cyclopts. Install the optional benchmark extra to include Cyclopts: +The benchmark is also part of the local aggregate: ```bash -python -m pip install 'base-cli[benchmark]' +./tests/full_validate.sh --gate benchmark ``` -The CI quality job checks the base-cli sample p95 against these budgets. The -benchmark records the selected platform profile in both text and JSON output; -set `BASE_CLI_BENCHMARK_PLATFORM` when a runner's filesystem or virtualization -boundary is not represented by the host operating system. Supported profiles -are `unix`, `macos`, `windows`, and `wsl`. - -| Measurement | Budget | -| --- | ---: | -| Fresh `import base_cli` (native Unix/macOS) | 750 ms | -| Fresh `import base_cli` (native Windows) | 1,000 ms | -| Fresh `import base_cli` (WSL2 on a Windows-mounted checkout) | 1,000 ms | -| Isolated invocation and runtime filesystem setup | 1,500 ms | - -The benchmark reports the median, p95, and maximum for seven samples. Pass -`--json` for a stable machine-readable result suitable for archiving or CI -comparison. These -budgets are intentionally broad enough for hosted runners while still -detecting accidental quadratic startup work, unbounded metadata scans, or -unexpected dependency imports. A performance improvement should preserve the -same lifecycle and persistence assertions covered by the adversarial tests. +## Scenario contract + +The comparative set is Click, Typer, Cyclopts, and base-cli. Each framework +registers an equivalent zero-argument no-op command. Fresh-process +measurements include Python startup, framework import, command construction, +and dispatch through the framework's normal entry point. Warm parser samples +reuse command objects; Click and Typer use Click's `CliRunner`, Cyclopts uses +its `App` call, and base-cli reports both a shared Click-runner lifecycle +sample and an end-to-end `base_cli.testing.invoke()` sample. The runner shape +for each value is recorded here so comparisons do not imply identical +mechanisms where framework APIs differ. + +Base-cli-only feature samples cover: + +- successful and failed JSON envelopes; +- debug diagnostics on the user stream; +- nested-command dispatch; +- persistence disabled versus enabled, with the same log event in both cases. + +These feature costs are reported separately from parser comparisons. JSON +success/error and nested dispatch use the public `App`/lifecycle API; persistence +samples differ only in whether file logging is enabled. Measurements are +in-process, warm, and use isolated temporary homes. + +## CI budgets and evidence + +CI collects 31 samples per scenario on Python 3.13 for each supported +benchmark profile: native Unix, macOS, Windows, and WSL2. `--check` fails if a +required comparator or scenario is missing, a p95 exceeds its profile budget, +or the measured base-cli lifecycle increment over Click exceeds its separate +profile budget. The lifecycle-to-Click ratio remains visible for interpretation, +but is not itself gated because Click's sub-millisecond baseline makes ratios +highly sensitive to timer granularity. Warm budgets apply to all non-persistence +base-cli feature scenarios; file persistence has a separate platform budget +because runner filesystems vary materially. Percentile gates catch practical +regressions while keeping noisy single maxima visible without making one +scheduler outlier block a change. + +| Budget (p95) | Unix | macOS | Windows | WSL2 | +| --- | ---: | ---: | ---: | ---: | +| Cold import, including interpreter startup | 750 ms | 750 ms | 1,000 ms | 1,000 ms | +| Cold no-op invocation, including startup and dispatch | 2,000 ms | 2,000 ms | 4,000 ms | 4,000 ms | +| Base-cli lifecycle increment over Click warm dispatch | 5 ms | 5 ms | 15 ms | 15 ms | +| Warm invocation and non-persistence feature scenarios | 50 ms | 50 ms | 100 ms | 100 ms | +| File-persistence-enabled scenario | 50 ms | 50 ms | 250 ms | 50 ms | + +An initial 31-sample local calibration on macOS (Python 3.14.6, Apple Silicon) +measured approximately 101 ms for base-cli cold import, 0.56 ms for warm +lifecycle dispatch, and 15.9 ms p95 for file-persisted logging. These are +development-host measurements, not adoption claims or release comparisons. +The first hosted 31-sample baseline measured file-persistence p95 at 22 ms on +Ubuntu, 21 ms on macOS, 158 ms on Windows, and 27 ms on WSL2. Windows also had +a high median absolute deviation (26 ms), so persistence has its own Windows +budget instead of weakening other warm-scenario gates. These measurements are +CI calibration evidence, not adoption claims or release comparisons; review +subsequent retained artifacts before tightening platform budgets. + +Each report is versioned as `base-cli.benchmark` schema version 1 and contains +the package version, source revision, UTC timestamp, platform profile, Python +version/ABI, OS release, architecture, CPU count, sample count, medians, p95, +maximum, median absolute deviation, and parser/lifecycle comparison values. +The Tests workflow retains a distinct JSON artifact for each platform profile +for 90 days. Download the artifact from the corresponding `Benchmark (...)` +or `Validate (WSL)` Actions job to compare dated runs. + +The profile can be selected explicitly with +`BASE_CLI_BENCHMARK_PLATFORM` when a runner's filesystem or virtualization +boundary is not represented by its host OS. Supported values are `unix`, +`macos`, `windows`, and `wsl`. ## Retention recovery work bounds diff --git a/docs/testing.md b/docs/testing.md index cac9b21..92d0a83 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -7,11 +7,11 @@ documentation/schema/contract validation, benchmark budgets, and security. Bandit and pip-audit are required; a missing tool is an error rather than a skipped check. -Run it from a clean checkout after installing the development and quality -extras: +Run it from a clean checkout after installing the development, quality, and +benchmark extras (the latter installs every declared framework comparator): ```bash -python -m pip install '.[dev,typer,quality]' +python -m pip install '.[dev,typer,quality,benchmark]' ./tests/full_validate.sh ``` @@ -30,15 +30,17 @@ validation gate. Individual gates can be selected for focused local work: ``` The Tests workflow runs the runtime suite across the OS/Python matrix and on -the supported Linux distributions/WSL. Its single quality job runs the -platform-independent coverage, typing, style, contract, benchmark, and -security gates once, with each group visible as a named Actions step. The -workflow validates feature branches through pull requests rather than -launching a second full run on every feature-branch push; direct pushes to -`main` and version tags remain validated. The -Package workflow focuses on release-boundary checks: building and validating -the wheel/sdist, checksums/SBOM, and clean installed-wheel smoke tests. It -does not repeat the source test, typing, lint, documentation, benchmark, or +the supported Linux distributions/WSL. Its quality job runs platform- +independent coverage, typing, style, contract, and security gates once, with +each group visible as a named Actions step. A separate comparative benchmark +matrix measures Click, Typer, Cyclopts, and base-cli on Unix, macOS, Windows, +and WSL; each job publishes an Actions summary and retains its versioned JSON +report as a dated artifact. The workflow validates feature branches through +pull requests rather than launching a second full run on every feature-branch +push; direct pushes to `main` and version tags remain validated. The Package +workflow focuses on release-boundary checks: building and validating the +wheel/sdist, checksums/SBOM, and clean installed-wheel smoke tests. It does +not repeat the source test, typing, lint, documentation, benchmark, or security suites. `./tests/full_validate.sh` remains the one-command local aggregate of all source gates. diff --git a/scripts/benchmark_runtime.py b/scripts/benchmark_runtime.py old mode 100644 new mode 100755 index a2b5c2d..24f3e78 --- a/scripts/benchmark_runtime.py +++ b/scripts/benchmark_runtime.py @@ -1,31 +1,72 @@ #!/usr/bin/env python3 -"""Track import and isolated invocation costs with stable, checked budgets.""" +"""Measure comparable Python CLI startup, invocation, and lifecycle scenarios.""" from __future__ import annotations import argparse +import datetime as dt import importlib.util import json import os +import platform import statistics import subprocess import sys +import sysconfig import tempfile import time from collections.abc import Callable +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as distribution_version from pathlib import Path from typing import Any, TypedDict, cast -# Fresh-process startup is materially slower on native Windows and on WSL -# when the checkout is on the Windows-mounted filesystem. CI can select the -# platform explicitly with BASE_CLI_BENCHMARK_PLATFORM; the fallback keeps -# local runs useful without requiring a setting. +# Budgets are intentionally profile-specific: hosted Windows and WSL runners +# have materially higher process and filesystem startup variance than POSIX. IMPORT_P95_BUDGETS_MS = { "unix": 750.0, "macos": 750.0, "windows": 1_000.0, "wsl": 1_000.0, } +COLD_INVOCATION_P95_BUDGETS_MS = { + "unix": 2_000.0, + "macos": 2_000.0, + "windows": 4_000.0, + "wsl": 4_000.0, +} +LIFECYCLE_OVERHEAD_P95_BUDGETS_MS = { + "unix": 5.0, + "macos": 5.0, + "windows": 15.0, + "wsl": 15.0, +} +WARM_INVOCATION_P95_BUDGETS_MS = { + "unix": 50.0, + "macos": 50.0, + "windows": 100.0, + "wsl": 100.0, +} +PERSISTENCE_ENABLED_P95_BUDGETS_MS = { + "unix": 50.0, + "macos": 50.0, + "windows": 250.0, + "wsl": 50.0, +} +DEFAULT_ITERATIONS = 31 +FRAMEWORKS = ("base-cli", "click", "typer", "cyclopts") +RESULT_SCHEMA = "base-cli.benchmark" +RESULT_SCHEMA_VERSION = 1 + + +class Summary(TypedDict): + median: float + p95: float + maximum: float + median_absolute_deviation: float + + +FrameworkMetrics = dict[str, Any] def _is_wsl() -> bool: @@ -53,22 +94,6 @@ def _benchmark_platform() -> str: BENCHMARK_PLATFORM = _benchmark_platform() -IMPORT_P95_BUDGET_MS = IMPORT_P95_BUDGETS_MS[BENCHMARK_PLATFORM] -INVOCATION_P95_BUDGET_MS = 1_500.0 -DEFAULT_ITERATIONS = 7 -FRAMEWORKS = ("base-cli", "click", "typer", "cyclopts") - - -class Summary(TypedDict): - median: float - p95: float - maximum: float - - -class FrameworkMetrics(TypedDict, total=False): - status: str - import_ms: Summary - invocation_ms: Summary def main() -> int: @@ -77,84 +102,346 @@ def main() -> int: "--iterations", type=int, default=DEFAULT_ITERATIONS, - help=f"number of samples per benchmark (default: {DEFAULT_ITERATIONS})", - ) - parser.add_argument( - "--check", - action="store_true", - help="fail when the documented p95 budgets are exceeded", - ) - parser.add_argument( - "--json", - action="store_true", - help="emit machine-readable benchmark results", + help=f"number of samples per scenario (default: {DEFAULT_ITERATIONS})", ) + parser.add_argument("--check", action="store_true", help="fail when a required comparator or budget is missing") + parser.add_argument("--json", action="store_true", help="emit the versioned machine-readable report to stdout") + parser.add_argument("--output", type=Path, help="write the versioned machine-readable report to this file") args = parser.parse_args() if args.iterations < 3: parser.error("--iterations must be at least 3") - metrics: dict[str, FrameworkMetrics] = {} + results: dict[str, FrameworkMetrics] = {} for framework in FRAMEWORKS: if importlib.util.find_spec(framework.replace("-", "_")) is None: - metrics[framework] = {"status": "unavailable"} + results[framework] = {"status": "unavailable"} continue - metrics[framework] = { - "import_ms": _summary(_measure_import(args.iterations, framework)), - "invocation_ms": _summary(_measure_invocations(args.iterations, framework)), + results[framework] = { + "cold_import_ms": _summary(_measure_import(args.iterations, framework)), + "warm_invocation_ms": _summary(_measure_invocations(args.iterations, framework)), + "cold_invocation_ms": _summary(_measure_cold_invocations(args.iterations, framework)), } - if args.json: - print( - json.dumps( - { - "iterations": args.iterations, - "platform": BENCHMARK_PLATFORM, - "import_p95_budget_ms": IMPORT_P95_BUDGET_MS, - "frameworks": metrics, - }, - sort_keys=True, - ) + + if results["base-cli"].get("status") != "unavailable": + results["base-cli"]["lifecycle_warm_invocation_ms"] = _summary(_measure_lifecycle_invocations(args.iterations)) + results["base-cli"]["production_warm_invocation_ms"] = _summary( + _measure_production_invocations(args.iterations) ) - else: - print(f"benchmark platform: {BENCHMARK_PLATFORM} (import p95 budget {IMPORT_P95_BUDGET_MS:.0f} ms)") - for framework, result in metrics.items(): - if result.get("status") == "unavailable": - print(f"{framework}: unavailable (install it to include this comparison)") - continue - assert "import_ms" in result and "invocation_ms" in result - print( - f"{framework} import_ms: median={{median:.2f}} p95={{p95:.2f}} max={{maximum:.2f}}".format( - **result["import_ms"] - ) - ) - print( - f"{framework} invocation_ms: median={{median:.2f}} p95={{p95:.2f}} max={{maximum:.2f}}".format( - **result["invocation_ms"] - ) - ) + results["base-cli"]["features"] = cast(Any, _measure_base_cli_features(args.iterations)) + + report = { + "schema": RESULT_SCHEMA, + "schema_version": RESULT_SCHEMA_VERSION, + "created_at_utc": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"), + "version": _package_version(), + "source_revision": _source_revision(), + "platform_profile": BENCHMARK_PLATFORM, + "iterations_per_scenario": args.iterations, + "environment": _environment_metadata(), + "framework_versions": _framework_versions(), + "results": results, + "comparisons": _comparisons(results), + "budgets_ms": _budgets_for_platform(BENCHMARK_PLATFORM), + } - if not args.check: - return 0 - failures = [] - base_metrics = metrics["base-cli"] - if base_metrics.get("status") == "unavailable": - failures.append("base-cli benchmark is unavailable") + failures = _check_results(results) if args.check else [] + _write_github_summary(report) + if args.json: + print(json.dumps(report, indent=2, sort_keys=True)) else: - assert "import_ms" in base_metrics and "invocation_ms" in base_metrics - if base_metrics["import_ms"]["p95"] > IMPORT_P95_BUDGET_MS: - failures.append(f"base-cli import p95 exceeded {IMPORT_P95_BUDGET_MS:.0f} ms") - if base_metrics["invocation_ms"]["p95"] > INVOCATION_P95_BUDGET_MS: - failures.append(f"base-cli invocation p95 exceeded {INVOCATION_P95_BUDGET_MS:.0f} ms") + _print_report(report) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"Benchmark report written to {args.output}") if failures: - print("Performance budget failure: " + "; ".join(failures), file=sys.stderr) + print("Benchmark contract failure: " + "; ".join(failures), file=sys.stderr) return 1 return 0 -def _measure_import(iterations: int, framework: str = "base-cli") -> list[float]: - package_root = Path(__file__).resolve().parents[1] / "lib" / "python" - environment = dict(os.environ) - existing_path = environment.get("PYTHONPATH") - environment["PYTHONPATH"] = f"{package_root}{os.pathsep}{existing_path}" if existing_path else str(package_root) +def _package_version() -> str: + version_file = Path(__file__).resolve().parents[1] / "VERSION" + try: + return version_file.read_text(encoding="utf-8").splitlines()[0].strip() + except (OSError, IndexError): + return "unknown" + + +def _source_revision() -> str | None: + configured = os.environ.get("SOURCE_REVISION") or os.environ.get("GITHUB_SHA") + if configured: + return configured + try: + completed = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=Path(__file__).resolve().parents[1], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError): + return None + revision = completed.stdout.strip() + return revision or None + + +def _environment_metadata() -> dict[str, Any]: + return { + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "processor": platform.processor() or None, + "python_implementation": platform.python_implementation(), + "python_version": platform.python_version(), + "python_abi": sysconfig.get_config_var("SOABI"), + "cpu_count": os.cpu_count(), + "github_actions": os.environ.get("GITHUB_ACTIONS", "").lower() == "true", + "github_run_id": os.environ.get("GITHUB_RUN_ID"), + "github_run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + } + + +def _framework_versions() -> dict[str, str | None]: + distributions = { + "base-cli": "base-cli", + "click": "click", + "typer": "typer", + "cyclopts": "cyclopts", + } + versions: dict[str, str | None] = {} + for framework, distribution in distributions.items(): + try: + versions[framework] = distribution_version(distribution) + except PackageNotFoundError: + versions[framework] = None + return versions + + +def _budgets_for_platform(platform_profile: str) -> dict[str, float]: + return { + "cold_import_p95": IMPORT_P95_BUDGETS_MS[platform_profile], + "cold_invocation_p95": COLD_INVOCATION_P95_BUDGETS_MS[platform_profile], + "lifecycle_increment_over_click_p95": LIFECYCLE_OVERHEAD_P95_BUDGETS_MS[platform_profile], + "warm_invocation_p95": WARM_INVOCATION_P95_BUDGETS_MS[platform_profile], + "persistence_enabled_p95": PERSISTENCE_ENABLED_P95_BUDGETS_MS[platform_profile], + } + + +def _comparisons(results: dict[str, FrameworkMetrics]) -> dict[str, float | None]: + base = results.get("base-cli", {}) + click = results.get("click", {}) + base_import = _metric_p95(base, "cold_import_ms") + click_import = _metric_p95(click, "cold_import_ms") + lifecycle = _metric_p95(base, "lifecycle_warm_invocation_ms") + click_warm = _metric_p95(click, "warm_invocation_ms") + return { + "base_cli_to_click_import_p95_ratio": _ratio(base_import, click_import), + "lifecycle_increment_over_click_warm_p95_ms": ( + max(0.0, lifecycle - click_warm) if lifecycle is not None and click_warm is not None else None + ), + "base_cli_lifecycle_to_click_warm_p95_ratio": _ratio(lifecycle, click_warm), + } + + +def _ratio(numerator: float | None, denominator: float | None) -> float | None: + if numerator is None or denominator is None or denominator <= 0: + return None + return numerator / denominator + + +def _metric_p95(metrics: FrameworkMetrics, key: str) -> float | None: + value = metrics.get(key) + if isinstance(value, dict): + p95 = value.get("p95") + return float(p95) if isinstance(p95, (int, float)) else None + return None + + +def _check_results(results: dict[str, FrameworkMetrics]) -> list[str]: + failures: list[str] = [] + for framework in FRAMEWORKS: + metrics = results.get(framework, {"status": "unavailable"}) + if metrics.get("status") == "unavailable": + failures.append(f"required comparator {framework} is unavailable; install the benchmark extras") + continue + for metric in ("cold_import_ms", "cold_invocation_ms", "warm_invocation_ms"): + if _metric_p95(metrics, metric) is None: + failures.append(f"required comparator {framework} is missing {metric} p95") + + base = results.get("base-cli", {}) + if base.get("status") == "unavailable": + return failures + + for metric in ("lifecycle_warm_invocation_ms", "production_warm_invocation_ms"): + if _metric_p95(base, metric) is None: + failures.append(f"base-cli benchmark is missing {metric} p95") + + features = base.get("features") + expected_features = { + "lifecycle_noop_ms", + "json_success_ms", + "json_error_ms", + "diagnostics_ms", + "nested_command_ms", + "persistence_disabled_ms", + "persistence_enabled_ms", + } + if not isinstance(features, dict): + failures.append("base-cli benchmark is missing feature scenarios") + else: + for feature in sorted(expected_features): + if _metric_p95(cast(FrameworkMetrics, {"metric": features.get(feature)}), "metric") is None: + failures.append(f"base-cli benchmark is missing {feature} p95") + + import_p95 = _metric_p95(base, "cold_import_ms") + if import_p95 is not None and import_p95 > IMPORT_P95_BUDGETS_MS[BENCHMARK_PLATFORM]: + failures.append(f"base-cli cold import p95 exceeded {IMPORT_P95_BUDGETS_MS[BENCHMARK_PLATFORM]:.0f} ms") + + cold_p95 = _metric_p95(base, "cold_invocation_ms") + if cold_p95 is not None and cold_p95 > COLD_INVOCATION_P95_BUDGETS_MS[BENCHMARK_PLATFORM]: + failures.append( + f"base-cli cold invocation p95 exceeded {COLD_INVOCATION_P95_BUDGETS_MS[BENCHMARK_PLATFORM]:.0f} ms" + ) + + warm_p95 = _metric_p95(base, "warm_invocation_ms") + if warm_p95 is not None and warm_p95 > WARM_INVOCATION_P95_BUDGETS_MS[BENCHMARK_PLATFORM]: + failures.append( + f"base-cli warm no-op invocation p95 exceeded {WARM_INVOCATION_P95_BUDGETS_MS[BENCHMARK_PLATFORM]:.0f} ms" + ) + + lifecycle_p95 = _metric_p95(base, "lifecycle_warm_invocation_ms") + click_p95 = _metric_p95(results.get("click", {}), "warm_invocation_ms") + if lifecycle_p95 is not None and click_p95 is not None: + if click_p95 <= 0: + failures.append("Click warm invocation p95 must be greater than zero for a valid comparison") + increment = max(0.0, lifecycle_p95 - click_p95) + budget = LIFECYCLE_OVERHEAD_P95_BUDGETS_MS[BENCHMARK_PLATFORM] + if increment > budget: + failures.append(f"base-cli lifecycle increment over Click p95 exceeded {budget:.0f} ms") + + production_p95 = _metric_p95(base, "production_warm_invocation_ms") + warm_budget = WARM_INVOCATION_P95_BUDGETS_MS[BENCHMARK_PLATFORM] + if production_p95 is not None and production_p95 > warm_budget: + failures.append(f"base-cli production-boundary warm invocation p95 exceeded {warm_budget:.0f} ms") + + if isinstance(features, dict): + for name, value in features.items(): + p95 = _metric_p95(cast(FrameworkMetrics, {"metric": value}), "metric") + feature_budget = _feature_budget_for_platform(name, BENCHMARK_PLATFORM) + if p95 is not None and p95 > feature_budget: + failures.append(f"base-cli {name} p95 exceeded {feature_budget:.0f} ms") + return failures + + +def _print_report(report: dict[str, Any]) -> None: + print( + f"benchmark platform: {report['platform_profile']} | Python {report['environment']['python_version']} " + f"| {report['iterations_per_scenario']} samples/scenario" + ) + for framework, metrics in report["results"].items(): + if metrics.get("status") == "unavailable": + print(f"{framework}: unavailable") + continue + for key, value in metrics.items(): + if isinstance(value, dict) and "p95" in value: + print( + f"{framework} {key}: median={value['median']:.2f} ms " + f"p95={value['p95']:.2f} ms max={value['maximum']:.2f} ms " + f"MAD={value['median_absolute_deviation']:.2f} ms" + ) + elif key == "features" and isinstance(value, dict): + for feature, summary in value.items(): + print( + f"base-cli {feature}: median={summary['median']:.2f} ms " + f"p95={summary['p95']:.2f} ms max={summary['maximum']:.2f} ms " + f"MAD={summary['median_absolute_deviation']:.2f} ms" + ) + print("comparisons: " + json.dumps(report["comparisons"], sort_keys=True)) + + +def _write_github_summary(report: dict[str, Any]) -> None: + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + + profile = report["platform_profile"] + lines = [ + f"## Base CLI performance — {profile}", + "", + f"Python {report['environment']['python_version']} · " + f"{report['iterations_per_scenario']} samples per scenario · " + f"base-cli {report['framework_versions'].get('base-cli') or 'unavailable'}", + "", + "### Framework comparison (p95, milliseconds)", + "", + "| Framework | Fresh-process import | Cold invocation | Warm invocation |", + "| --- | ---: | ---: | ---: |", + ] + for framework in FRAMEWORKS: + metrics = report["results"].get(framework, {}) + version = report["framework_versions"].get(framework) or "unavailable" + lines.append( + f"| {framework} {version} | {_summary_value(metrics, 'cold_import_ms')} | " + f"{_summary_value(metrics, 'cold_invocation_ms')} | {_summary_value(metrics, 'warm_invocation_ms')} |" + ) + + comparison = report["comparisons"] + lines.extend( + [ + "", + "### Lifecycle overhead versus parser dispatch", + "", + "| Measure | Result | Budget |", + "| --- | ---: | ---: |", + f"| base-cli lifecycle increment over Click warm p95 | " + f"{_format_number(comparison.get('lifecycle_increment_over_click_warm_p95_ms'))} ms | " + f"{_format_number(report['budgets_ms']['lifecycle_increment_over_click_p95'])} ms |", + f"| base-cli / Click lifecycle warm p95 | " + f"{_format_number(comparison.get('base_cli_lifecycle_to_click_warm_p95_ratio'))}× | informational |", + "", + "### Base CLI feature scenarios (p95, milliseconds)", + "", + "| Scenario | p95 | Budget |", + "| --- | ---: | ---: |", + ] + ) + features = report["results"].get("base-cli", {}).get("features", {}) + for feature, summary in sorted(features.items()): + feature_budget = _feature_budget_for_platform(feature, profile) + lines.append( + f"| {feature.removesuffix('_ms').replace('_', ' ')} | {_format_number(summary.get('p95'))} ms | " + f"{_format_number(feature_budget)} ms |" + ) + lines.append("") + try: + with Path(summary_path).open("a", encoding="utf-8") as stream: + stream.write("\n".join(lines)) + except OSError as exc: + print(f"Could not write GitHub Actions summary: {exc}", file=sys.stderr) + + +def _summary_value(metrics: FrameworkMetrics, name: str) -> str: + summary = metrics.get(name) + if not isinstance(summary, dict): + return "—" + return f"{_format_number(summary.get('p95'))} ms" + + +def _feature_budget_for_platform(feature: str, platform_profile: str) -> float: + if feature == "persistence_enabled_ms": + return PERSISTENCE_ENABLED_P95_BUDGETS_MS[platform_profile] + return WARM_INVOCATION_P95_BUDGETS_MS[platform_profile] + + +def _format_number(value: object) -> str: + if not isinstance(value, (int, float)): + return "—" + return f"{value:.2f}" + + +def _measure_import(iterations: int, framework: str) -> list[float]: + environment = _subprocess_environment() samples: list[float] = [] for _ in range(iterations): started = time.perf_counter_ns() @@ -169,34 +456,114 @@ def _measure_import(iterations: int, framework: str = "base-cli") -> list[float] return samples -def _measure_invocations(iterations: int, framework: str = "base-cli") -> list[float]: +def _measure_cold_invocations(iterations: int, framework: str) -> list[float]: + environment = _subprocess_environment() + programs = { + "base-cli": ( + "import base_cli\n" + "app = base_cli.App(name='benchmark-cold', log_to_file=False)\n" + "@app.command()\n" + "def command(ctx):\n del ctx\n" + "raise SystemExit(base_cli.run_app(app, []))\n" + ), + "click": ( + "import click\n" + "@click.command()\n" + "def command():\n return None\n" + "command.main([], prog_name='benchmark-cold', standalone_mode=False)\n" + ), + "typer": ( + "import typer\n" + "from typer.main import get_command\n" + "app = typer.Typer()\n" + "@app.command()\n" + "def command():\n return None\n" + "get_command(app).main([], prog_name='benchmark-cold', standalone_mode=False)\n" + ), + "cyclopts": ("import cyclopts\napp = cyclopts.App()\n@app.default\ndef command():\n return None\napp([])\n"), + } + samples: list[float] = [] + with tempfile.TemporaryDirectory(prefix="base-cli-cold-benchmark-") as tmpdir: + environment["HOME"] = tmpdir + environment["XDG_CACHE_HOME"] = str(Path(tmpdir) / ".cache") + environment["BASE_CLI_CACHE_DIR"] = str(Path(tmpdir) / ".cache") + if os.name == "nt": + environment["USERPROFILE"] = tmpdir + environment["LOCALAPPDATA"] = str(Path(tmpdir) / "AppData" / "Local") + for _ in range(iterations): + started = time.perf_counter_ns() + subprocess.run( + [sys.executable, "-c", programs[framework]], + check=True, + cwd=Path(__file__).resolve().parents[1], + env=environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + samples.append(_elapsed_ms(started)) + return samples + + +def _measure_invocations(iterations: int, framework: str) -> list[float]: if framework == "click": return _measure_click_invocations(iterations) if framework == "typer": return _measure_typer_invocations(iterations) if framework == "cyclopts": return _measure_cyclopts_invocations(iterations) + return _measure_production_invocations(iterations) + + +def _measure_production_invocations(iterations: int) -> list[float]: import base_cli from base_cli.testing import invoke - app = base_cli.App(name="benchmark-runtime") + app = base_cli.App(name="benchmark-runtime", log_to_file=False) @app.command() - def main(ctx: base_cli.Context[Any, Any, Any]) -> None: + def main(ctx: Any) -> None: del ctx + _command = app.click_command samples: list[float] = [] with tempfile.TemporaryDirectory(prefix="base-cli-benchmark-") as tmpdir: home = Path(tmpdir) for _ in range(iterations): started = time.perf_counter_ns() result = invoke(app, [], home=home) + elapsed = _elapsed_ms(started) if result.exit_code != 0: raise RuntimeError(f"benchmark invocation failed: {result.output}") - samples.append(_elapsed_ms(started)) + samples.append(elapsed) return samples +def _measure_lifecycle_invocations(iterations: int) -> list[float]: + import base_cli + from click.testing import CliRunner + + app = base_cli.App(name="benchmark-lifecycle", log_to_file=False) + + @app.command() + def main(ctx: Any) -> None: + del ctx + + command = cast(Any, app.click_command) + runner = CliRunner() + # Keep this probe in the same ambient Click runner environment as the + # comparator. The isolated HOME used by production scenarios measures a + # different contract and would make the lifecycle delta incomparable. + return _measure_runner(iterations, lambda: runner.invoke(command, []).exit_code) + + +def _subprocess_environment() -> dict[str, str]: + package_root = Path(__file__).resolve().parents[1] / "lib" / "python" + environment = dict(os.environ) + existing_path = environment.get("PYTHONPATH") + environment["PYTHONPATH"] = f"{package_root}{os.pathsep}{existing_path}" if existing_path else str(package_root) + return environment + + def _measure_click_invocations(iterations: int) -> list[float]: import click from click.testing import CliRunner @@ -237,6 +604,99 @@ def callback() -> None: return _measure_runner(iterations, lambda: cast(Any, app)([])) +def _measure_base_cli_features(iterations: int) -> dict[str, Summary]: + import base_cli + import click + from base_cli.testing import invoke + + def make_app(name: str, *, log_to_file: bool = False) -> base_cli.App: + return base_cli.App(name=name, log_to_file=log_to_file) + + lifecycle = make_app("benchmark-lifecycle-noop") + + @lifecycle.command() + def lifecycle_noop(ctx: Any) -> None: + del ctx + + json_success = make_app("benchmark-json-success") + json_success.lifecycle_options = base_cli.LifecycleOptions( + json=base_cli.LifecycleOption("--json"), + ) + + @json_success.command() + def success(ctx: Any) -> None: + del ctx + print("ok") + + json_error = make_app("benchmark-json-error") + json_error.lifecycle_options = base_cli.LifecycleOptions( + json=base_cli.LifecycleOption("--json"), + ) + + @json_error.command() + def failure(ctx: Any) -> None: + del ctx + raise click.ClickException("expected benchmark error") + + diagnostics = make_app("benchmark-diagnostics") + + @diagnostics.command() + def diagnose(ctx: Any) -> None: + ctx.log.debug("benchmark diagnostic") + + nested = make_app("benchmark-nested") + + @nested.subcommand("noop") + def nested_noop(ctx: Any) -> None: + del ctx + + persistence_disabled = make_app("benchmark-persistence-disabled", log_to_file=False) + + @persistence_disabled.command() + def log_without_file(ctx: Any) -> None: + ctx.log.info("benchmark persistence probe") + + persistence_enabled = make_app("benchmark-persistence-enabled", log_to_file=True) + + @persistence_enabled.command() + def log_with_file(ctx: Any) -> None: + ctx.log.info("benchmark persistence probe") + + scenarios = { + "lifecycle_noop_ms": (lifecycle, [], 0, None), + "json_success_ms": (json_success, ["--json"], 0, "success"), + "json_error_ms": (json_error, ["--json"], 1, "error"), + "diagnostics_ms": (diagnostics, ["--debug"], 0, None), + "nested_command_ms": (nested, ["noop"], 0, None), + "persistence_disabled_ms": (persistence_disabled, [], 0, None), + "persistence_enabled_ms": (persistence_enabled, [], 0, None), + } + for app, _args, _expected_exit, _json_type in scenarios.values(): + _command = app.click_command + + results: dict[str, Summary] = {} + for metric_name, (app, args, expected_exit, json_type) in scenarios.items(): + samples: list[float] = [] + with tempfile.TemporaryDirectory(prefix=f"{app.name}-") as tmpdir: + home = Path(tmpdir) + for _ in range(iterations): + started = time.perf_counter_ns() + result = invoke(app, args, home=home) + elapsed = _elapsed_ms(started) + if result.exit_code != expected_exit: + raise RuntimeError(f"{metric_name} benchmark failed: {result.output}") + if json_type is not None: + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError(f"{metric_name} did not emit a JSON envelope") from exc + if payload.get("type") != json_type: + raise RuntimeError(f"{metric_name} emitted unexpected JSON envelope type") + samples.append(elapsed) + results[metric_name] = _summary(samples) + return results + + def _measure_runner(iterations: int, callback: Callable[[], Any]) -> list[float]: samples: list[float] = [] for _ in range(iterations): @@ -253,11 +713,15 @@ def _elapsed_ms(started_ns: int) -> float: def _summary(samples: list[float]) -> Summary: + if len(samples) < 3: + raise ValueError("at least three benchmark samples are required") + median = statistics.median(samples) p95 = statistics.quantiles(samples, n=20, method="inclusive")[18] return { - "median": statistics.median(samples), + "median": median, "p95": p95, "maximum": max(samples), + "median_absolute_deviation": statistics.median(abs(sample - median) for sample in samples), } diff --git a/tests/test_benchmark_runtime.py b/tests/test_benchmark_runtime.py index 4ba4032..8528e5e 100644 --- a/tests/test_benchmark_runtime.py +++ b/tests/test_benchmark_runtime.py @@ -1,8 +1,11 @@ from __future__ import annotations import importlib.util +import os +import tempfile import unittest from pathlib import Path +from unittest import mock _SCRIPT_PATH = Path(__file__).parents[1] / "scripts" / "benchmark_runtime.py" _SPEC = importlib.util.spec_from_file_location("benchmark_runtime", _SCRIPT_PATH) @@ -13,12 +16,17 @@ class BenchmarkSummaryTests(unittest.TestCase): - def test_summary_reports_interpolated_p95_separately_from_maximum(self) -> None: + def test_summary_reports_percentile_and_variance_separately(self) -> None: summary = benchmark_runtime._summary([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0]) self.assertEqual(summary["median"], 4.0) self.assertAlmostEqual(summary["p95"], 6.7) self.assertEqual(summary["maximum"], 7.0) + self.assertEqual(summary["median_absolute_deviation"], 2.0) + + def test_summary_requires_enough_samples_for_percentiles(self) -> None: + with self.assertRaisesRegex(ValueError, "at least three"): + benchmark_runtime._summary([1.0, 2.0]) def test_framework_comparison_has_stable_public_set(self) -> None: self.assertEqual( @@ -26,8 +34,156 @@ def test_framework_comparison_has_stable_public_set(self) -> None: ("base-cli", "click", "typer", "cyclopts"), ) - def test_platform_profiles_have_explicit_import_budgets(self) -> None: - self.assertEqual(benchmark_runtime.IMPORT_P95_BUDGETS_MS["unix"], 750.0) - self.assertEqual(benchmark_runtime.IMPORT_P95_BUDGETS_MS["macos"], 750.0) - self.assertEqual(benchmark_runtime.IMPORT_P95_BUDGETS_MS["windows"], 1_000.0) - self.assertEqual(benchmark_runtime.IMPORT_P95_BUDGETS_MS["wsl"], 1_000.0) + def test_platform_profiles_have_explicit_budget_vectors(self) -> None: + for profile in ("unix", "macos", "windows", "wsl"): + budgets = benchmark_runtime._budgets_for_platform(profile) + self.assertEqual( + set(budgets), + { + "cold_import_p95", + "cold_invocation_p95", + "lifecycle_increment_over_click_p95", + "warm_invocation_p95", + "persistence_enabled_p95", + }, + ) + self.assertGreater(budgets["cold_invocation_p95"], budgets["cold_import_p95"]) + + def test_environment_metadata_has_reproduction_fields(self) -> None: + metadata = benchmark_runtime._environment_metadata() + + for key in ( + "system", + "release", + "machine", + "python_implementation", + "python_version", + "python_abi", + "cpu_count", + ): + self.assertIn(key, metadata) + + def test_comparison_separates_click_parser_from_lifecycle_increment(self) -> None: + metrics = self._complete_results(lifecycle_p95=4.0, click_p95=1.5) + + comparisons = benchmark_runtime._comparisons(metrics) + + self.assertEqual(comparisons["lifecycle_increment_over_click_warm_p95_ms"], 2.5) + self.assertAlmostEqual(comparisons["base_cli_lifecycle_to_click_warm_p95_ratio"], 4.0 / 1.5) + + def test_check_requires_every_declared_comparator(self) -> None: + metrics = self._complete_results() + metrics["cyclopts"] = {"status": "unavailable"} + + with mock.patch.object(benchmark_runtime, "BENCHMARK_PLATFORM", "macos"): + failures = benchmark_runtime._check_results(metrics) + + self.assertTrue(any("required comparator cyclopts is unavailable" in failure for failure in failures)) + + def test_check_rejects_realistic_lifecycle_regression(self) -> None: + metrics = self._complete_results(lifecycle_p95=5.5, click_p95=0.2) + + with mock.patch.object(benchmark_runtime, "BENCHMARK_PLATFORM", "macos"): + failures = benchmark_runtime._check_results(metrics) + + self.assertTrue(any("lifecycle increment over Click p95 exceeded 5 ms" in failure for failure in failures)) + + def test_check_rejects_cold_start_budget_regression(self) -> None: + metrics = self._complete_results(base_import_p95=751.0) + + with mock.patch.object(benchmark_runtime, "BENCHMARK_PLATFORM", "macos"): + failures = benchmark_runtime._check_results(metrics) + + self.assertTrue(any("cold import p95 exceeded 750 ms" in failure for failure in failures)) + + def test_check_rejects_a_missing_required_feature_scenario(self) -> None: + metrics = self._complete_results() + metrics["base-cli"]["features"] = {} + + with mock.patch.object(benchmark_runtime, "BENCHMARK_PLATFORM", "macos"): + failures = benchmark_runtime._check_results(metrics) + + self.assertTrue(any("missing diagnostics_ms p95" in failure for failure in failures)) + + def test_windows_persistence_budget_does_not_weaken_other_warm_scenarios(self) -> None: + metrics = self._complete_results() + metrics["base-cli"]["features"]["persistence_enabled_ms"] = self._summary(201.0) + metrics["base-cli"]["features"]["diagnostics_ms"] = self._summary(101.0) + + with mock.patch.object(benchmark_runtime, "BENCHMARK_PLATFORM", "windows"): + failures = benchmark_runtime._check_results(metrics) + + self.assertFalse(any("persistence_enabled_ms" in failure for failure in failures)) + self.assertTrue(any("diagnostics_ms p95 exceeded 100 ms" in failure for failure in failures)) + + def test_windows_persistence_budget_rejects_material_regressions(self) -> None: + metrics = self._complete_results() + metrics["base-cli"]["features"]["persistence_enabled_ms"] = self._summary(251.0) + + with mock.patch.object(benchmark_runtime, "BENCHMARK_PLATFORM", "windows"): + failures = benchmark_runtime._check_results(metrics) + + self.assertTrue(any("persistence_enabled_ms p95 exceeded 250 ms" in failure for failure in failures)) + + def test_github_summary_separates_lifecycle_overhead_from_parser(self) -> None: + metrics = self._complete_results(lifecycle_p95=4.0, click_p95=1.5) + report = { + "platform_profile": "macos", + "environment": {"python_version": "3.13.0"}, + "iterations_per_scenario": 31, + "framework_versions": {framework: "1.0" for framework in benchmark_runtime.FRAMEWORKS}, + "results": metrics, + "comparisons": benchmark_runtime._comparisons(metrics), + "budgets_ms": benchmark_runtime._budgets_for_platform("macos"), + } + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "summary.md" + with mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(path)}): + benchmark_runtime._write_github_summary(report) + summary = path.read_text(encoding="utf-8") + + self.assertIn("Framework comparison (p95, milliseconds)", summary) + self.assertIn("Lifecycle overhead versus parser dispatch", summary) + self.assertIn("base-cli lifecycle increment over Click warm p95", summary) + self.assertIn("Base CLI feature scenarios", summary) + + @staticmethod + def _summary(p95: float) -> dict[str, float]: + return { + "median": p95 * 0.8, + "p95": p95, + "maximum": p95 * 1.1, + "median_absolute_deviation": p95 * 0.05, + } + + @classmethod + def _complete_results( + cls, + *, + base_import_p95: float = 10.0, + lifecycle_p95: float = 2.0, + click_p95: float = 1.0, + ) -> dict[str, dict[str, object]]: + results: dict[str, dict[str, object]] = {} + for framework in benchmark_runtime.FRAMEWORKS: + results[framework] = { + "cold_import_ms": cls._summary(base_import_p95 if framework == "base-cli" else 8.0), + "cold_invocation_ms": cls._summary(20.0), + "warm_invocation_ms": cls._summary(click_p95 if framework == "click" else 1.0), + } + results["base-cli"].update( + { + "lifecycle_warm_invocation_ms": cls._summary(lifecycle_p95), + "production_warm_invocation_ms": cls._summary(4.0), + "features": { + "lifecycle_noop_ms": cls._summary(1.0), + "json_success_ms": cls._summary(1.0), + "json_error_ms": cls._summary(1.0), + "diagnostics_ms": cls._summary(1.0), + "nested_command_ms": cls._summary(1.0), + "persistence_disabled_ms": cls._summary(1.0), + "persistence_enabled_ms": cls._summary(1.0), + }, + } + ) + return results