fix(benchmarks): accept the multi-scenario verify.py invocation its docs advertise

benchmarks/README.md

@@ -285,7 +285,7 @@ cd benchmarks
 ../.venv/bin/python run_http.py stack async bare,full,full_all_tuned
 
 # what a scenario gives up, and where the time goes inside one
-../.venv/bin/python verify.py errors_only_no_txn
+../.venv/bin/python verify.py errors_only errors_only_no_txn errors_only_logging_lean errors_only_skip_txn
 ../.venv/bin/python micro.py
 ../.venv/bin/python profile_one.py sentry errors_only
 ../.venv/bin/python profile_one.py sentry errors_only --callers 'Random.seed'

benchmarks/verify.py

@@ -4,7 +4,9 @@
 raises, with an incoming `sentry-trace` header, and prints what survived on the captured event:
 the transaction name, whether the incoming distributed trace was continued, and the breadcrumbs.
 
-    python verify.py errors_only errors_only_no_txn errors_only_skip_txn
+    python verify.py errors_only errors_only_no_txn errors_only_logging_lean errors_only_skip_txn
+
+Each scenario runs in its own process, because `sentry_sdk.init` cannot be undone between them.
 """
 
 import argparse
@@ -13,6 +15,8 @@
 import io
 import json
 import logging
+import pathlib
+import subprocess
 import sys
 import typing
 
@@ -30,6 +34,9 @@
 logger = logging.getLogger("bench")
 
 
+HERE = pathlib.Path(__file__).resolve().parent
+
+
 class CapturingTransport(Transport):
     def __init__(self, options: dict[str, typing.Any] | None = None) -> None:
         super().__init__(options)
@@ -82,29 +89,48 @@ def describe(scenario: str, event: dict[str, typing.Any]) -> dict[str, typing.An
     }
 
 
-def main() -> None:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("scenario")
-    args = parser.parse_args()
+def resolve(parser: argparse.ArgumentParser, name: str) -> typing.Callable[[], dict[str, typing.Any]]:
+    if name not in sentry_scenarios.SCENARIOS:
+        parser.error(f"unknown scenario: {name}")
+    build = sentry_scenarios.SCENARIOS[name]
+    if build is None:
+        parser.error(f"scenario '{name}' captures nothing")
+    return build
 
-    logging.basicConfig(level=logging.INFO, stream=io.StringIO())
 
+def run_scenario(scenario: str, build: typing.Callable[[], dict[str, typing.Any]]) -> None:
     transport = CapturingTransport()
-    kwargs = sentry_scenarios.SCENARIOS[args.scenario]
-    if kwargs is None:
-        parser.error("scenario 'off' captures nothing")
-    init_kwargs = kwargs()
+    init_kwargs = build()
     init_kwargs["transport"] = transport
     sentry_sdk.init(**init_kwargs)
-    for patch in sentry_scenarios.PATCHES.get(args.scenario, ()):
+    for patch in sentry_scenarios.PATCHES.get(scenario, ()):
         patch()
 
     asyncio.run(drive(make_app()))
 
     for event in transport.events:
-        json.dump(describe(args.scenario, event), sys.stdout)
+        json.dump(describe(scenario, event), sys.stdout)
         sys.stdout.write("\n")
 
 
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("scenarios", nargs="+")
+    args = parser.parse_args()
+    builds = [resolve(parser, name) for name in args.scenarios]
+
+    logging.basicConfig(level=logging.INFO, stream=io.StringIO())
+
+    if len(args.scenarios) > 1:
+        # One process each; inprocess.py explains why none of this can be undone between scenarios.
+        for scenario in args.scenarios:
+            subprocess.run(  # noqa: S603
+                [sys.executable, str(HERE / "verify.py"), scenario], check=True, cwd=HERE
+            )
+        return
+
+    run_scenario(args.scenarios[0], builds[0])
+
+
 if __name__ == "__main__":
     main()