From 406acf64664f49928bc2b467a1b8442c9ee3100e Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Sun, 20 Sep 2026 14:24:02 +0300 Subject: [PATCH 1/2] fix(benchmarks): accept the multi-scenario verify.py invocation its docs advertise --- benchmarks/README.md | 2 +- benchmarks/verify.py | 52 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index f38751e..152c28f 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -285,7 +285,7 @@ cd benchmarks ../.venv/bin/python run_http.py stack async bare,full,full_all_tuned # what a scenario gives up, and where the time goes inside one -../.venv/bin/python verify.py errors_only_no_txn +../.venv/bin/python verify.py errors_only errors_only_no_txn errors_only_logging_lean errors_only_skip_txn ../.venv/bin/python micro.py ../.venv/bin/python profile_one.py sentry errors_only ../.venv/bin/python profile_one.py sentry errors_only --callers 'Random.seed' diff --git a/benchmarks/verify.py b/benchmarks/verify.py index 663f5d9..dba10c0 100644 --- a/benchmarks/verify.py +++ b/benchmarks/verify.py @@ -5,6 +5,8 @@ the transaction name, whether the incoming distributed trace was continued, and the breadcrumbs. python verify.py errors_only errors_only_no_txn errors_only_skip_txn + +Each scenario runs in its own process, because `sentry_sdk.init` cannot be undone between them. """ import argparse @@ -13,6 +15,8 @@ import io import json import logging +import pathlib +import subprocess import sys import typing @@ -30,6 +34,9 @@ logger = logging.getLogger("bench") +HERE: typing.Final = pathlib.Path(__file__).resolve().parent + + class CapturingTransport(Transport): def __init__(self, options: dict[str, typing.Any] | None = None) -> None: super().__init__(options) @@ -82,29 +89,52 @@ def describe(scenario: str, event: dict[str, typing.Any]) -> dict[str, typing.An } -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("scenario") - args = parser.parse_args() - - logging.basicConfig(level=logging.INFO, stream=io.StringIO()) - +def describe_one(scenario: str) -> None: transport = CapturingTransport() - kwargs = sentry_scenarios.SCENARIOS[args.scenario] + kwargs = sentry_scenarios.SCENARIOS[scenario] if kwargs is None: - parser.error("scenario 'off' captures nothing") + msg = f"scenario '{scenario}' captures nothing" + raise ValueError(msg) init_kwargs = kwargs() init_kwargs["transport"] = transport sentry_sdk.init(**init_kwargs) - for patch in sentry_scenarios.PATCHES.get(args.scenario, ()): + for patch in sentry_scenarios.PATCHES.get(scenario, ()): patch() asyncio.run(drive(make_app())) for event in transport.events: - json.dump(describe(args.scenario, event), sys.stdout) + json.dump(describe(scenario, event), sys.stdout) sys.stdout.write("\n") +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("scenario", nargs="+") + args = parser.parse_args() + + # Validate every scenario before running any, so a typo in the last one does not + # surface as a subprocess traceback after the earlier ones have already run. + for scenario in args.scenario: + if scenario not in sentry_scenarios.SCENARIOS: + parser.error(f"unknown scenario: {scenario}") + if sentry_scenarios.SCENARIOS[scenario] is None: + parser.error(f"scenario '{scenario}' captures nothing") + + logging.basicConfig(level=logging.INFO, stream=io.StringIO()) + + # Each extra scenario gets its own process: `sentry_sdk.init` and the PATCHES + # monkeypatches cannot be undone, so a loop here would report later scenarios + # through the earlier one's patches. + if len(args.scenario) > 1: + for scenario in args.scenario: + subprocess.run( # noqa: S603 + [sys.executable, str(HERE / "verify.py"), scenario], check=True, cwd=HERE + ) + return + + describe_one(args.scenario[0]) + + if __name__ == "__main__": main() From 216eeec527ef95b7ba00cd49a4395674092ac3c5 Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Sun, 20 Sep 2026 14:28:01 +0300 Subject: [PATCH 2/2] review: resolve scenarios once, drop the duplicated guard --- benchmarks/verify.py | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/benchmarks/verify.py b/benchmarks/verify.py index dba10c0..06ec2a0 100644 --- a/benchmarks/verify.py +++ b/benchmarks/verify.py @@ -4,7 +4,7 @@ raises, with an incoming `sentry-trace` header, and prints what survived on the captured event: the transaction name, whether the incoming distributed trace was continued, and the breadcrumbs. - python verify.py errors_only errors_only_no_txn errors_only_skip_txn + python verify.py errors_only errors_only_no_txn errors_only_logging_lean errors_only_skip_txn Each scenario runs in its own process, because `sentry_sdk.init` cannot be undone between them. """ @@ -34,7 +34,7 @@ logger = logging.getLogger("bench") -HERE: typing.Final = pathlib.Path(__file__).resolve().parent +HERE = pathlib.Path(__file__).resolve().parent class CapturingTransport(Transport): @@ -89,13 +89,18 @@ def describe(scenario: str, event: dict[str, typing.Any]) -> dict[str, typing.An } -def describe_one(scenario: str) -> None: +def resolve(parser: argparse.ArgumentParser, name: str) -> typing.Callable[[], dict[str, typing.Any]]: + if name not in sentry_scenarios.SCENARIOS: + parser.error(f"unknown scenario: {name}") + build = sentry_scenarios.SCENARIOS[name] + if build is None: + parser.error(f"scenario '{name}' captures nothing") + return build + + +def run_scenario(scenario: str, build: typing.Callable[[], dict[str, typing.Any]]) -> None: transport = CapturingTransport() - kwargs = sentry_scenarios.SCENARIOS[scenario] - if kwargs is None: - msg = f"scenario '{scenario}' captures nothing" - raise ValueError(msg) - init_kwargs = kwargs() + init_kwargs = build() init_kwargs["transport"] = transport sentry_sdk.init(**init_kwargs) for patch in sentry_scenarios.PATCHES.get(scenario, ()): @@ -110,30 +115,21 @@ def describe_one(scenario: str) -> None: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("scenario", nargs="+") + parser.add_argument("scenarios", nargs="+") args = parser.parse_args() - - # Validate every scenario before running any, so a typo in the last one does not - # surface as a subprocess traceback after the earlier ones have already run. - for scenario in args.scenario: - if scenario not in sentry_scenarios.SCENARIOS: - parser.error(f"unknown scenario: {scenario}") - if sentry_scenarios.SCENARIOS[scenario] is None: - parser.error(f"scenario '{scenario}' captures nothing") + builds = [resolve(parser, name) for name in args.scenarios] logging.basicConfig(level=logging.INFO, stream=io.StringIO()) - # Each extra scenario gets its own process: `sentry_sdk.init` and the PATCHES - # monkeypatches cannot be undone, so a loop here would report later scenarios - # through the earlier one's patches. - if len(args.scenario) > 1: - for scenario in args.scenario: + if len(args.scenarios) > 1: + # One process each; inprocess.py explains why none of this can be undone between scenarios. + for scenario in args.scenarios: subprocess.run( # noqa: S603 [sys.executable, str(HERE / "verify.py"), scenario], check=True, cwd=HERE ) return - describe_one(args.scenario[0]) + run_scenario(args.scenarios[0], builds[0]) if __name__ == "__main__":